Exemple #1
0
    def on_btnPreProccess_clicked(self):
        """
        Slot documentation goes here.
        """
        source = self.edit_source_preprocessing_tab.text()
        target = self.edit_target_preprocessing_tab.text()
        lm_text = self.edit_lm_text_preprocessing_tab.text()
        output = self.edit_output_preprocessing_tab.text()
        source_language = self.preprocessing_source_language
        target_language = self.preprocessing_target_language

        if not source:
            doAlert("Please choose a source text first.")
            return
        elif not target:
            doAlert("Please choose a target text first.")
            return
        elif not output:
            doAlert("Please choose an output directory first.")
            return
        elif not source_language:
            doAlert(
                "Please choose an preprocessing_source_language directory first."
            )
            return
        elif not target_language:
            doAlert(
                "Please choose an preprocessing_target_language directory first."
            )
            return
        else:
            text = self.core._prepare_corpus(output, source_language,
                                             target_language, source, target,
                                             lm_text)
            self.results_preprocessing.setText(text)
    def on_buttonBox_accepted(self):
        """
        Slot documentation goes here.
        """
        def checkEmpty(mystr):
            return len(str(mystr).strip()) <= 0

        # Check everything.
        self.modelName = self.editName.text()
        if checkEmpty(self.modelName):
            doAlert("Please provide non-empty Model Name")
            return
        if self.grpBoxInternet.isChecked():
            self.source = self.editInternet.text()
            self.sourceMode = "Internet"
        elif self.grpBoxLocal.isChecked():
            self.source = self.editLocal.text()
            self.sourceMode = "Local"
            if not os.path.exists(str(self.source)):
                doAlert("Please provide valid local file as source")
                return
        else:
            doAlert("Please provide Install Source")
            return
        if checkEmpty(self.source):
            doAlert("Please provide non-empty Install Source")
            return
        self.dest = self.editPath.text()
        if checkEmpty(self.dest):
            doAlert("Please provide non-empty Install Destination Folder")
            return
        self.accept()
Exemple #3
0
 def detect(self):
     pf = platform.system()
     if pf == 'Windows':
         self.mosesPath = self.findRegistryPath()
         if self.mosesPath:
             return self.checkMosesInstall()
         else:
             if not doQuestion(
                 'Cannot find Moses installation, click "Yes" to '
                 'manually set the Moses path, click "No" to exit.'):
                 return False
         # If not found, use a dialog.
         startdir = 'C:\\'
         if "ProgramFiles(x86)" in os.environ:
             startdir = os.environ["ProgramFiles(x86)"]
         elif "ProgramFiles" in os.environ:
             startdir = os.environ["ProgramFiles"]
         else:
             pass
         dialog = QFileDialog(None, directory=startdir)
         dialog.setFileMode(QFileDialog.Directory)
         dialog.setViewMode(QFileDialog.Detail)
         dialog.setOptions(QFileDialog.ShowDirsOnly)
         if dialog.exec_():
             self.mosesPath = str(dialog.selectedFiles()[0])
             return self.checkMosesInstall()
         else:
             doAlert("Failed to find Moses Installation path, exit.")
             return False
     else:
         doAlert("Platform %s not supported yet" % pf)
         return False
    def on_buttonBox_accepted(self):
        """
        Slot documentation goes here.
        """
        def checkEmpty(mystr):
            return len(str(mystr).strip()) <= 0

        # Check everything.
        self.modelName = self.editName.text()
        if checkEmpty(self.modelName):
            doAlert("Please provide non-empty Model Name")
            return
        if self.grpBoxInternet.isChecked():
            self.source = self.editInternet.text()
            self.sourceMode = "Internet"
        elif self.grpBoxLocal.isChecked():
            self.source = self.editLocal.text()
            self.sourceMode = "Local"
            if not os.path.exists(str(self.source)):
                doAlert("Please provide valid local file as source")
                return
        else:
            doAlert("Please provide Install Source")
            return
        if checkEmpty(self.source):
            doAlert("Please provide non-empty Install Source")
            return
        self.dest = self.editPath.text()
        if checkEmpty(self.dest):
            doAlert("Please provide non-empty Install Destination Folder")
            return
        self.accept()
Exemple #5
0
 def detect(self):
     pf = platform.system()
     if pf == 'Windows':
         self.mosesPath = self.findRegistryPath()
         if self.mosesPath:
             return self.checkMosesInstall()
         else:
             if not doQuestion(
                     'Cannot find Moses installation, click "Yes" to '
                     'manually set the Moses path, click "No" to exit.'):
                 return False
         # If not found, use a dialog.
         startdir = 'C:\\'
         if "ProgramFiles(x86)" in os.environ:
             startdir = os.environ["ProgramFiles(x86)"]
         elif "ProgramFiles" in os.environ:
             startdir = os.environ["ProgramFiles"]
         else:
             pass
         dialog = QFileDialog(None, directory=startdir)
         dialog.setFileMode(QFileDialog.Directory)
         dialog.setViewMode(QFileDialog.Detail)
         dialog.setOptions(QFileDialog.ShowDirsOnly)
         if dialog.exec_():
             self.mosesPath = str(dialog.selectedFiles()[0])
             return self.checkMosesInstall()
         else:
             doAlert("Failed to find Moses Installation path, exit.")
             return False
     else:
         doAlert("Platform %s not supported yet" % pf)
         return False
 def newEntry(self):
     import random
     rec = self.record()
     for i in xrange(1, 10):
         rec.setValue(i,  QVariant(str(random.random())))
     self.insertRecord(-1,  rec)
     doAlert(self.query().lastInsertId().toString())
 def newEntry(self):
     import random
     rec = self.record()
     for i in xrange(1, 10):
         rec.setValue(i, QVariant(str(random.random())))
     self.insertRecord(-1, rec)
     doAlert(self.query().lastInsertId().toString())
Exemple #8
0
 def checkMosesInstall(self):
     for func in (self.getMosesCmd, self.getTokenizer, self.getDetokenizer, self.getTruecase, self.getDetruecase):
         if not os.path.exists(func()):
             doAlert(
                 "Missing executables in Moses installation path [%s], "
                 "exit." % self.mosesPath)
             return False
     return True
Exemple #9
0
 def checkMosesInstall(self):
     for func in (self.getMosesCmd, self.getTokenizer, self.getDetokenizer,
                  self.getTruecase, self.getDetruecase):
         if not os.path.exists(func()):
             doAlert("Missing executables in Moses installation path [%s], "
                     "exit." % self.mosesPath)
             return False
     return True
    def on_btnMachineTranslation_clicked(self):

        source = self.edit_source_machine_translation_tab.text()
        if not source:
            doAlert("Please choose a source text first.")
            return
        text = self.migrated_backend_main._machine_translation(
            source, self.chooseModel).decode('utf8')
        self.results_machine_translation.setText(text)
Exemple #11
0
    def on_btnMachineTranslation_clicked(self):

        source = self.edit_source_machine_translation_tab.text()
        if not source:
            doAlert("Please choose a source text.")
            return
        self.results_machine_translation.setText(
            "Running decoder, please wait\n\n")
        text = self.core._machine_translation(source,
                                              self.chooseModel).decode('utf8')
        self.results_machine_translation.setText(text)
Exemple #12
0
    def on_btnTraining_clicked(self):
        """
        Slot documentation goes here.
        """

        text = self.core._train()

        if text == "ERR":
            text = "ERROR: missing corpus to train model"
            doAlert("Please provide corpus to train model")
            self.tabWidget.setCurrentIndex(0)
            return
        else:
            self.results_training.setText(text)
Exemple #13
0
 def on_buttonBox_accepted(self):
     """
     Slot documentation goes here.
     """
     current = self.selTableView.currentIndex()
     if not current:
         doAlert("Please choose a model to start.")
         return
     record = self.model.record(current.row())
     self.ID = record.value("ID").toString()
     self.modelName = record.value("name").toString()
     self.srcLang = record.value('srclang').toString()
     self.trgLang = record.value('trglang').toString()
     self.path = record.value("path").toString()
     self.mosesini = record.value("mosesini").toString()
     self.accept()
 def on_buttonBox_accepted(self):
     """
     Slot documentation goes here.
     """
     current = self.selTableView.currentIndex()
     if not current:
         doAlert("Please choose a model to start.")
         return
     record = self.model.record(current.row())
     self.ID = record.value("ID").toString()
     self.modelName = record.value("name").toString()
     self.srcLang = record.value('srclang').toString()
     self.trgLang = record.value('trglang').toString()
     self.path = record.value("path").toString()
     self.mosesini = record.value("mosesini").toString()
     self.accept()
 def detect(self):
     doQuestion('Cannot find Moses installation, click "Yes" to '
                'manually set the Moses path, click "No" to exit.')
     # If not found, use a dialog.
     startdir = 'C:\\'
     if "ProgramFiles(x86)" in os.environ:
         startdir = os.environ["ProgramFiles(x86)"]
     elif "ProgramFiles" in os.environ:
         startdir = os.environ["ProgramFiles"]
     else:
         pass
     dialog = QFileDialog(None, directory=startdir)
     dialog.setFileMode(QFileDialog.Directory)
     dialog.setViewMode(QFileDialog.Detail)
     dialog.setOptions(QFileDialog.ShowDirsOnly)
     if dialog.exec_():
         self.mosesPath = str(dialog.selectedFiles()[0])
         return self.mosesPath
     else:
         doAlert("Failed to find Moses Installation path, exit.")
         return "Failed to find Moses Installation path, exit."
    def on_btnEvaluation_clicked(self):
        """
        Slot documentation goes here.
        """
        source = self.edit_source_evaluation_tab.text()
        target = self.edit_target_evaluation_tab.text()
        output = self.edit_output_evaluation_tab.text()

        if not source:
            doAlert("Please choose a source text first.")
            return
        elif not target:
            doAlert("Please choose a target text first.")
            return
        elif not output:
            doAlert("Please choose an output directory first.")
            return

        checkbox_indexes = [
        ]  #[False] * 8 #checkbox_indexes["WER","PER","HTER", "BLEU","BLEU2GRAM","BLEU3GRAM"]
        checkbox_indexes.append(self.btn_check_WER.isChecked())
        checkbox_indexes.append(self.btn_check_PER.isChecked())
        checkbox_indexes.append(self.btn_check_HTER.isChecked())
        checkbox_indexes.append(self.btn_check_BLEU.isChecked())
        checkbox_indexes.append(self.btn_check_BLEU2GRAM.isChecked())
        checkbox_indexes.append(self.btn_check_BLEU3GRAM.isChecked())
        checkbox_indexes.append(self.btn_check_BLEU4GRAM.isChecked())

        text = self.migrated_backend_main._evaluate(checkbox_indexes, source,
                                                    target)
        self.results_evaluation.setText(text)
Exemple #17
0
 def __init__(self, model, moses):
     super(Engine, self).__init__()
     self.model = model
     self.moses = moses
     self.check(self.model)
     # Determine how many steps by num of modules in the model directory
     # (moses + tok/detok + ...) + 1
     modelini = model['path'] + os.sep + 'model' + os.sep + 'model.ini'
     if not os.path.exists(modelini):
         raise Exception(
             "Model ini file doesn't exist, please check model dir %s" %
             modelini)
     cp = ConfigParser.RawConfigParser()
     cp.readfp(open(modelini))
     truemodel = None
     try:
         truemodel = cp.get("Preprocess", 'Truemodel')
         truemodel_path = os.path.join(model['path'], 'model', truemodel)
         if not os.path.exists(truemodel_path):
             doAlert("Truemodel doesn't exist, ignore %s" % truemodel)
             truemodel = None
     except:
         # doAlert("Truemodel not configured")
         truemodel = None
     self.cmds = []
     # tok
     self.cmds.append(
         '"%s" -q -l %s -noxml' %
         (self.moses.getTokenizer(), str(model['srclang']).lower()))
     if truemodel is not None:
         self.cmds.append('"%s" -model "%s"' %
                          (self.moses.getTruecase(), truemodel))
     self.cmds.append('"%s" -f moses.ini' % self.moses.getMosesCmd())
     self.cmds.append('"%s"' % self.moses.getDetruecase())
     self.cmds.append(
         '"%s" -q -noxml -l %s' %
         (self.moses.getDetokenizer(), str(model['trglang']).lower()))
     self.process = []
Exemple #18
0
 def __init__(self, model, moses):
     super(Engine, self).__init__()
     self.model = model
     self.moses = moses
     self.check(self.model)
     # Determine how many steps by num of modules in the model directory
     # (moses + tok/detok + ...) + 1
     modelini = model['path'] + os.sep + 'model' + os.sep + 'model.ini'
     if not os.path.exists(modelini):
         raise Exception(
             "Model ini file doesn't exist, please check model dir %s"
             % modelini)
     cp = ConfigParser.RawConfigParser()
     cp.readfp(open(modelini))
     truemodel = None
     try:
         truemodel = cp.get("Preprocess", 'Truemodel')
         truemodel_path = os.path.join(model['path'], 'model', truemodel)
         if not os.path.exists(truemodel_path):
             doAlert("Truemodel doesn't exist, ignore %s" % truemodel)
             truemodel = None
     except:
         # doAlert("Truemodel not configured")
         truemodel = None
     self.cmds = []
     # tok
     self.cmds.append(
         '"%s" -q -l %s -noxml'
         % (self.moses.getTokenizer(), str(model['srclang']).lower()))
     if truemodel is not None:
         self.cmds.append(
             '"%s" -model "%s"' % (self.moses.getTruecase(), truemodel))
     self.cmds.append('"%s" -f moses.ini' % self.moses.getMosesCmd())
     self.cmds.append('"%s"' % self.moses.getDetruecase())
     self.cmds.append(
         '"%s" -q -noxml -l %s'
         % (self.moses.getDetokenizer(), str(model['trglang']).lower()))
     self.process = []
Exemple #19
0
 def on_btnTranslate_clicked(self):
     """
     Slot documentation goes here.
     """
     if self.engine is None:
         doAlert("Please load MT model first.")
         return
     self.btnTranslate.setEnabled(False)
     self.editTrg.setText("")
     try:
         texts = str(self.editSrc.toPlainText().toUtf8()).split('\n')
         trans = []
         for text in texts:
             if text.strip() == "":
                 trans.append(text)
             else:
                 trans.append(
                     self.engine.translate(text.replace(
                         '\r', ' ').strip()).decode('utf8'))
         self.editTrg.setText('\n'.join(trans))
     except Exception, e:
         print >> sys.stderr, str(e)
         doAlert("Translation failed!")
Exemple #20
0
    def installModel(self,  installParam):
        dest = installParam['dest']
        # Make dir.
        if not os.path.exists(dest):
            try:
                os.makedirs(str(dest))
            except:
                doAlert("Failed to create install directory: %s" % dest)
                return
        # Create entry in db.
        rec = self.record()
        rec.setValue('name',  installParam['modelName'])
        rec.setValue('status',  'Fetching Source...')
        rec.setValue('path',  dest)
        rec.setValue('origin',  installParam['source'])
        rec.setValue('originMode',  installParam['sourceMode'])
        rec.setValue('date',  QDateTime.currentDateTime())
        rec.setValue('deleted', 'False')
        self.insertRecord(-1,  rec)
        rowid = self.query().lastInsertId()

        # Start thread.
        def installThread(irowid):

            # Find the current row in model.
            def updateRecord(keyvalues):
                curRow = None
                # TODO: use binary search instead of linear
                for i in xrange(0,  self.rowCount()):
                    if self.record(i).value("ID") == rowid:
                        curRow = i
                        break
                if curRow is not None:
                    record = self.record(curRow)
                    for key in keyvalues:
                        record.setValue(key,  keyvalues[key])
                    self.changeRecord(curRow, record)
                return curRow

            def checkExit():
                # Check thread is ok to run.
                if irowid not in self.installThreads or not self.installThreads[irowid][1]:
                    return True
                else:
                    return False

            def markExit():
                if irowid in self.installThreads:  # Set thread to dead.
                    self.installThreads[irowid][1] = False

            def statusMessageLogMarkExit(status=None, message=None,
                                         exception=None):
                if status is not None:
                    updateRecord({'status': status})
                if message is not None:
                    self.emit(SIGNAL("messageBox(QString)"), message)
                    print >> sys.stderr, message
                if exception is not None:
                    print >> sys.stderr, str(exception)
                markExit()

            # 1. Download or copy from local.
            # Where the downloaded/copied zip file is:
            destFile = os.path.join(str(dest),  "model.zip")
            # Where the unzipped contents are:
            destDir = os.path.join(str(dest), "model")

            if installParam['sourceMode'] == 'Local':
                fin = fout = None
                try:
                    inFile = str(installParam['source'])
                    total_size = os.path.getsize(inFile)
                    fin = open(inFile, 'rb')
                    chunk_size = 52428800  # 50MB as chunk size
                    fout = open(destFile, 'wb')
                    content = fin.read(chunk_size)
                    download_size = content_size = len(content)
                    lastMsg = ""
                    while content_size > 0:
                        # Check if thread is notified as exit.
                        if checkExit():
                            return statusMessageLogMarkExit()
                        fout.write(content)
                        if total_size > 0:
                            msg = 'COPY %.0f%%' % (
                                download_size * 100.0 / total_size)
                        else:
                            msg = 'COPY %d MB' % (download_size / 1048576)
                        if msg != lastMsg:
                            updateRecord({'status': msg})
                            lastMsg = msg
                        content = fin.read(chunk_size)
                        content_size = len(content)
                        download_size += content_size
                except Exception as e:
                    return statusMessageLogMarkExit(
                        status=(
                            'Failed copying from: %s'
                            % installParam['source']),
                        message=(
                            "Failed copy model: %s"
                            % installParam['modelName']),
                        exception=e)
                finally:
                    if fin:
                        fin.close()
                    if fout:
                        fout.close()

            elif installParam['sourceMode'] == 'Internet':
                conn = fout = None
                try:
                    conn = urllib2.urlopen(str(installParam['source']))
                    total_size = int(conn.headers['Content-Length'])
                    chunk_size = 1048576  # 1MB as chunk size
                    fout = open(destFile, 'wb')
                    content = conn.read(chunk_size)
                    download_size = content_size = len(content)
                    lastMsg = ""
                    while content_size > 0:
                        # Check if thread is notified as exit.
                        if checkExit():
                            return statusMessageLogMarkExit()
                        fout.write(content)
                        if total_size > 0:
                            msg = 'DOWNLOAD %.0f%%' % (
                                download_size * 100.0 / total_size)
                        else:
                            msg = 'DOWNLOAD %d MB' % (download_size / 1048576)
                        if msg != lastMsg:
                            updateRecord({'status': msg})
                            lastMsg = msg
                        content = conn.read(chunk_size)
                        content_size = len(content)
                        download_size += content_size
                except Exception as e:
                    return statusMessageLogMarkExit(
                        status=(
                            'Failed downloading from: %s'
                            % installParam['source']),
                        message=(
                            "Failed download model: %s"
                            % installParam['modelName']),
                        exception=e)
                finally:
                    if conn:
                        conn.close()
                    if fout:
                        fout.close()
            else:
                return statusMessageLogMarkExit(
                    status='Unsupported source mode: %s'
                    % installParam['sourceMode'])

            # 2. Unzip.
            zfile = fout = None
            try:
                zfile = zipfile.ZipFile(destFile)
                # Check property files.
                if "model.ini" not in zfile.namelist():
                    return statusMessageLogMarkExit(
                        status=(
                            'Missing model.ini in model file: %s'
                            % installParam['sourceMode']),
                        message=(
                            "Invalid modle file format because model.ini "
                            "is missing in the zipped model file, exit "
                            "installation for model %s"
                            % installParam['modelName']))
                chunk_size = 52428800  # 50MB as chunk size
                # Get file size uncompressed.
                total_size = 0
                for name in zfile.namelist():
                    total_size += zfile.getinfo(name).file_size
                download_size = 0
                lastMsg = ""
                for i, name in enumerate(zfile.namelist()):
                    (dirname, filename) = os.path.split(name)
                    outDir = os.path.join(destDir, dirname)
                    if not os.path.exists(outDir):
                        os.makedirs(outDir)
                    if filename:
                        fin = zfile.open(name, 'r')
                        outFile = os.path.join(destDir,  name)
                        fout = open(outFile, 'wb')
                        content = fin.read(chunk_size)
                        content_size = len(content)
                        download_size += content_size
                        while content_size > 0:
                            # Check if thread is notified as exit.
                            if checkExit():
                                return statusMessageLogMarkExit()
                            fout.write(content)
                            if total_size > 0:
                                msg = 'UNZIP %.0f%%' % (
                                    download_size * 100.0 / total_size)
                            else:
                                msg = 'UNZIP %d MB' % (
                                    download_size / 1048576)
                            if msg != lastMsg:
                                updateRecord({'status': msg})
                                lastMsg = msg
                            content = fin.read(chunk_size)
                            content_size = len(content)
                            download_size += content_size
                        fin.close()
                        fout.close()
            except Exception as e:
                return statusMessageLogMarkExit(
                    status=(
                        'Failed unzipping from: %s' % installParam['source']),
                    message=(
                        "Failed unzip model: %s" % installParam['modelName']),
                    exception=e)
            finally:
                if zfile:
                    zfile.close()
                if fin:
                    fin.close()
                if fout:
                    fout.close()

            # 3 Post process and check data validity.
            try:
                modelini = os.path.join(destDir, "model.ini")
                cp = ConfigParser.RawConfigParser()
                cp.read(modelini)
                mosesini = os.path.join(destDir, 'moses.ini')
                if not os.path.exists(mosesini):
                    raise Exception("Moses ini doesn't exist")
                updateRecord({
                    'srclang': cp.get("Language", 'Source Language').upper(),
                    'trglang': cp.get("Language", 'Target Language').upper(),
                    'mosesini': mosesini},
                    )
            except Exception as e:
                return statusMessageLogMarkExit(
                    status='ERROR model format %s' % installParam['source'],
                    message=(
                        "Unspported model format: %s"
                        % installParam['modelName']),
                    exception=e)

            statusMessageLogMarkExit(
                status='READY',
                message="Model %s Installed!" % installParam['modelName'])
            # Send new model signal.
            self.emit(SIGNAL("modelInstalled()"))  # Record selection.
            return
            # End of thread func.

        # Start the thread.
        irowid, _ = rowid.toInt()
        t = threading.Thread(target=installThread,  args=(irowid, ))
        if irowid in self.installThreads:  # If old thread is there.
            print >> sys.stderr, (
                "table rowid %d already has a thread running, stop it"
                % irowid)
            self.installThreads[irowid][1] = False
        self.installThreads[irowid] = [t,  True]
        t.start()
    def on_btnPostEditing_clicked(self):
        source = self.edit_source_post_editing.text()
        #print source
        target = self.edit_target_post_editing.text()
        #print target
        output = self.edit_output_post_editing.text()
        if not source and self.btn_bilingual_post_edition.isChecked():
            doAlert("Please choose a source text first.")
            return
        if not target:
            doAlert("Please choose a target text first.")
            return
        if not output:
            doAlert("Please choose an output directory first.")
            return

        self.source_text = []
        self.unchanged_target_text = []
        self.target_text = []

        with open(target) as fp:
            for line in fp:
                line = line.decode("utf-8")
                if line != '\n':
                    self.unchanged_target_text.append(line)
                    self.target_text.append(line)
        if self.btn_bilingual_post_edition.isChecked():
            with open(source) as fp:
                for line in fp:
                    line = line.decode("utf-8")
                    if line != '\n':
                        self.source_text.append(line)
        else:
            self.source_text = self.target_text

        self.post_editing_data["source"] = self.source_text
        self.post_editing_data["target"] = self.target_text
        self.table_offset_PostEdition = 0
        self.table_rows_PostEdition = 100  #delete these vars if nav butons not used <<,>>,+,-
        self.table_rows_Differences = 50
        self.table_offset_Differences = 0
        self.lastChangedSegmentIndex = -1

        self.modified_references_indices = []
        self.modified_table_items_coordinates = []
        self.lastChangedTableItemCoordinates = (-1, -1)
        #CHECK WHICH OF THE FOLLOWING SHOULD BE CLEARED
        #TO SHOW DIFFERENCES CORRECTLY AFTER REOADING FILES FOR PE
        #AGREGAR LIMPIAR BUSQEUDAS
        self.saved_modified_references = []
        self.differences = None
        #self.post_editing_data = {}
        self.differences_data = {}
        self.unmodified_target = []
        self.modified_target = []
        self.last_changed_item_in_post_edition = None
        self.last_selected_search = None
        self.log = {}
        self.update_table_PostEdition()
        #THE FOLLOWING CODE CREATES A NICE SIMPLE TABLE WITH WRAPPED
        #TEXT IN THE CELLS, COULD BE USED AS REFERENCE TO REFACTOR CODE
        #P self.table_post_editing.clear()
        #Pfor y, key in enumerate(sorted(self.post_editing_data.keys())):
        #P  for x, item in enumerate(self.post_editing_data[key]):
        #P
        #P  newitem = QTableWidgetItem(textwrap.fill(item, width=70))
        #Pif key != "Post-edited":
        #P  newitem.setReadOnly(True)

        #P  self.table_post_editing.setItem(x, y, newitem)

        #P self.table_post_editing.setTextElideMode(QtCore.Qt.ElideNone)
        #P self.table_post_editing.resizeColumnsToContents()
        #P self.table_post_editing.resizeRowsToContents()
        #P self.table_post_editing.horizontalHeader().setResizeMode(QtGui.QHeaderView.Stretch)
        self.PE_table_controls_groupBox.show()
        self.PE_search_groupBox.show()
        self.table_post_editing.show()
Exemple #22
0
    def on_btnEvaluation_clicked(self):
        """
        Slot documentation goes here.
        """
        hyp = self.edit_hyp_evaluation_tab.text()
        ref = self.edit_ref_evaluation_tab.text()
        output = ""  #self.edit_output_evaluation_tab.text()

        if not hyp:
            doAlert("Please choose MTed text.")
            return
        elif not ref:
            doAlert("Please choose reference text.")
            return
        #elif not output:
        ##   doAlert("Please choose an output directory first.")
        #   return

        if self.btn_check_WER.isChecked():
            pathEval = (os.path.dirname(os.path.realpath(__file__)))
            cmd = "perl " + pathEval + "/evaluation_scripts" + "/WER.pl" + " -t " + hyp + " -r " + ref
            proc = subprocess.Popen([cmd],
                                    stdout=subprocess.PIPE,
                                    stderr=subprocess.PIPE,
                                    shell=True)

            out, err = proc.communicate()

            output += "WER: %s (Execution errors: %s)\n" % (out, err)
            #print cmd

        if self.btn_check_PER.isChecked():
            cmd = "perl " + pathEval + "/evaluation_scripts" + "/PER.pl" + " -t " + hyp + " -r " + ref
            proc = subprocess.Popen([cmd],
                                    stdout=subprocess.PIPE,
                                    stderr=subprocess.PIPE,
                                    shell=True)

            out, err = proc.communicate()
            output += "PER: %s (Execution errors: %s)\n" % (out, err)
            #print cmd

        if self.btn_check_HTER.isChecked():
            cmd = "perl " + pathEval + "/evaluation_scripts" + "/tercom_v6b.pl" + " -h " + hyp + " -r " + ref + " -o sum_doc "
            proc = subprocess.Popen([cmd],
                                    stdout=subprocess.PIPE,
                                    stderr=subprocess.PIPE,
                                    shell=True)
            out, err = proc.communicate()
            output += "HTER: %s (Execution errors: %s)\n" % (out, err)
            #print cmd

        if self.btn_check_BLEU.isChecked():
            cmd = "perl " + pathEval + "/evaluation_scripts" + "/BLEU.pl " + ref + " < " + hyp
            proc = subprocess.Popen([cmd],
                                    stdout=subprocess.PIPE,
                                    stderr=subprocess.PIPE,
                                    shell=True)
            out, err = proc.communicate()
            output += "BLEU: %s (Execution errors: %s)\n" % (out, err)
            #print cmd

        self.results_evaluation.setText(output)
Exemple #23
0
    def on_btnPostEditing_clicked(self):
        self.post_editing_data = {'Source/MT': [], 'Post-edited': []}
        self.log = {}
        self.modified_target = []
        self.timestamps = []

        self.table_post_editing.clear()
        self.table_post_editing.blockSignals(True)

        source = self.edit_source_post_editing.text()
        # print source
        target = self.edit_target_post_editing.text()
        #print target
        output = self.edit_output_post_editing.text()
        if not source and self.btn_bilingual_post_edition.isChecked():
            doAlert("Please choose a source text first.")
            return
        if not target:
            doAlert("Please choose a target text first.")
            return
        if not output:
            doAlert("Please choose an output directory first.")
            return

        with open(target) as fp:
            for line in fp:
                line = line.decode("utf-8")
                if line != '\n':
                    self.post_editing_data['Post-edited'].append(line)
                    self.timestamps.append([(0, 0)
                                            ])  #initial time for PE timestamps
                    self.modified_target.append(None)
        if self.btn_bilingual_post_edition.isChecked():
            with open(source) as fp:
                for line in fp:
                    line = line.decode("utf-8")
                    if line != '\n':
                        self.post_editing_data['Source/MT'].append(line)
        else:
            self.post_editing_data['Source/MT'] = self.post_editing_data[
                'Post-edited']

        self.table_post_editing.setRowCount(
            len(self.post_editing_data['Post-edited']))
        if len(self.post_editing_data['Source/MT']) != len(
                self.post_editing_data['Post-edited']):
            doAlert("Please choose an output directory first.")
            return

        for y, key in enumerate(['Source/MT', 'Post-edited']):
            for x, item in enumerate(self.post_editing_data[key]):
                newitem = QTableWidgetItem(textwrap.fill(item, width=70))
                #print x, y
                if key == 'Source/MT':
                    newitem.setFlags(QtCore.Qt.ItemIsEditable)
                self.table_post_editing.setItem(x, y, newitem)

        self.table_post_editing.setHorizontalHeaderLabels(
            ["Source/MT", "Post-edited"])
        self.table_post_editing.setTextElideMode(QtCore.Qt.ElideNone)
        self.table_post_editing.resizeColumnsToContents()
        self.table_post_editing.resizeRowsToContents()
        self.table_post_editing.horizontalHeader().setResizeMode(
            QtGui.QHeaderView.Stretch)
        self.table_post_editing.setMouseTracking(True)
        #use the events below to calculate time spent per segment
        self.table_post_editing.show()
        self.table_post_editing.cellChanged.connect(self.endPESeg)
        self.table_post_editing.cellEntered.connect(self.startPESeg)
        self.table_post_editing.blockSignals(False)
    def installModel(self, installParam):
        dest = installParam['dest']
        # Make dir.
        if not os.path.exists(dest):
            try:
                os.makedirs(str(dest))
            except:
                doAlert("Failed to create install directory: %s" % dest)
                return
        # Create entry in db.
        rec = self.record()
        rec.setValue('name', installParam['modelName'])
        rec.setValue('status', 'Fetching Source...')
        rec.setValue('path', dest)
        rec.setValue('origin', installParam['source'])
        rec.setValue('originMode', installParam['sourceMode'])
        rec.setValue('date', QDateTime.currentDateTime())
        rec.setValue('deleted', 'False')
        self.insertRecord(-1, rec)
        rowid = self.query().lastInsertId()

        # Start thread.
        def installThread(irowid):

            # Find the current row in model.
            def updateRecord(keyvalues):
                curRow = None
                # TODO: use binary search instead of linear
                for i in xrange(0, self.rowCount()):
                    if self.record(i).value("ID") == rowid:
                        curRow = i
                        break
                if curRow is not None:
                    record = self.record(curRow)
                    for key in keyvalues:
                        record.setValue(key, keyvalues[key])
                    self.changeRecord(curRow, record)
                return curRow

            def checkExit():
                # Check thread is ok to run.
                if irowid not in self.installThreads or not self.installThreads[
                        irowid][1]:
                    return True
                else:
                    return False

            def markExit():
                if irowid in self.installThreads:  # Set thread to dead.
                    self.installThreads[irowid][1] = False

            def statusMessageLogMarkExit(status=None,
                                         message=None,
                                         exception=None):
                if status is not None:
                    updateRecord({'status': status})
                if message is not None:
                    self.emit(SIGNAL("messageBox(QString)"), message)
                    print >> sys.stderr, message
                if exception is not None:
                    print >> sys.stderr, str(exception)
                markExit()

            # 1. Download or copy from local.
            # Where the downloaded/copied zip file is:
            destFile = os.path.join(str(dest), "model.zip")
            # Where the unzipped contents are:
            destDir = os.path.join(str(dest), "model")

            if installParam['sourceMode'] == 'Local':
                fin = fout = None
                try:
                    inFile = str(installParam['source'])
                    total_size = os.path.getsize(inFile)
                    fin = open(inFile, 'rb')
                    chunk_size = 52428800  # 50MB as chunk size
                    fout = open(destFile, 'wb')
                    content = fin.read(chunk_size)
                    download_size = content_size = len(content)
                    lastMsg = ""
                    while content_size > 0:
                        # Check if thread is notified as exit.
                        if checkExit():
                            return statusMessageLogMarkExit()
                        fout.write(content)
                        if total_size > 0:
                            msg = 'COPY %.0f%%' % (download_size * 100.0 /
                                                   total_size)
                        else:
                            msg = 'COPY %d MB' % (download_size / 1048576)
                        if msg != lastMsg:
                            updateRecord({'status': msg})
                            lastMsg = msg
                        content = fin.read(chunk_size)
                        content_size = len(content)
                        download_size += content_size
                except Exception as e:
                    return statusMessageLogMarkExit(
                        status=('Failed copying from: %s' %
                                installParam['source']),
                        message=("Failed copy model: %s" %
                                 installParam['modelName']),
                        exception=e)
                finally:
                    if fin:
                        fin.close()
                    if fout:
                        fout.close()

            elif installParam['sourceMode'] == 'Internet':
                conn = fout = None
                try:
                    conn = urllib2.urlopen(str(installParam['source']))
                    total_size = int(conn.headers['Content-Length'])
                    chunk_size = 1048576  # 1MB as chunk size
                    fout = open(destFile, 'wb')
                    content = conn.read(chunk_size)
                    download_size = content_size = len(content)
                    lastMsg = ""
                    while content_size > 0:
                        # Check if thread is notified as exit.
                        if checkExit():
                            return statusMessageLogMarkExit()
                        fout.write(content)
                        if total_size > 0:
                            msg = 'DOWNLOAD %.0f%%' % (download_size * 100.0 /
                                                       total_size)
                        else:
                            msg = 'DOWNLOAD %d MB' % (download_size / 1048576)
                        if msg != lastMsg:
                            updateRecord({'status': msg})
                            lastMsg = msg
                        content = conn.read(chunk_size)
                        content_size = len(content)
                        download_size += content_size
                except Exception as e:
                    return statusMessageLogMarkExit(
                        status=('Failed downloading from: %s' %
                                installParam['source']),
                        message=("Failed download model: %s" %
                                 installParam['modelName']),
                        exception=e)
                finally:
                    if conn:
                        conn.close()
                    if fout:
                        fout.close()
            else:
                return statusMessageLogMarkExit(
                    status='Unsupported source mode: %s' %
                    installParam['sourceMode'])

            # 2. Unzip.
            zfile = fout = None
            try:
                zfile = zipfile.ZipFile(destFile)
                # Check property files.
                if "model.ini" not in zfile.namelist():
                    return statusMessageLogMarkExit(
                        status=('Missing model.ini in model file: %s' %
                                installParam['sourceMode']),
                        message=("Invalid modle file format because model.ini "
                                 "is missing in the zipped model file, exit "
                                 "installation for model %s" %
                                 installParam['modelName']))
                chunk_size = 52428800  # 50MB as chunk size
                # Get file size uncompressed.
                total_size = 0
                for name in zfile.namelist():
                    total_size += zfile.getinfo(name).file_size
                download_size = 0
                lastMsg = ""
                for i, name in enumerate(zfile.namelist()):
                    (dirname, filename) = os.path.split(name)
                    outDir = os.path.join(destDir, dirname)
                    if not os.path.exists(outDir):
                        os.makedirs(outDir)
                    if filename:
                        fin = zfile.open(name, 'r')
                        outFile = os.path.join(destDir, name)
                        fout = open(outFile, 'wb')
                        content = fin.read(chunk_size)
                        content_size = len(content)
                        download_size += content_size
                        while content_size > 0:
                            # Check if thread is notified as exit.
                            if checkExit():
                                return statusMessageLogMarkExit()
                            fout.write(content)
                            if total_size > 0:
                                msg = 'UNZIP %.0f%%' % (download_size * 100.0 /
                                                        total_size)
                            else:
                                msg = 'UNZIP %d MB' % (download_size / 1048576)
                            if msg != lastMsg:
                                updateRecord({'status': msg})
                                lastMsg = msg
                            content = fin.read(chunk_size)
                            content_size = len(content)
                            download_size += content_size
                        fin.close()
                        fout.close()
            except Exception as e:
                return statusMessageLogMarkExit(
                    status=('Failed unzipping from: %s' %
                            installParam['source']),
                    message=("Failed unzip model: %s" %
                             installParam['modelName']),
                    exception=e)
            finally:
                if zfile:
                    zfile.close()
                if fin:
                    fin.close()
                if fout:
                    fout.close()

            # 3 Post process and check data validity.
            try:
                modelini = os.path.join(destDir, "model.ini")
                cp = ConfigParser.RawConfigParser()
                cp.read(modelini)
                mosesini = os.path.join(destDir, 'moses.ini')
                if not os.path.exists(mosesini):
                    raise Exception("Moses ini doesn't exist")
                updateRecord(
                    {
                        'srclang': cp.get("Language",
                                          'Source Language').upper(),
                        'trglang': cp.get("Language",
                                          'Target Language').upper(),
                        'mosesini': mosesini
                    }, )
            except Exception as e:
                return statusMessageLogMarkExit(
                    status='ERROR model format %s' % installParam['source'],
                    message=("Unspported model format: %s" %
                             installParam['modelName']),
                    exception=e)

            statusMessageLogMarkExit(status='READY',
                                     message="Model %s Installed!" %
                                     installParam['modelName'])
            # Send new model signal.
            self.emit(SIGNAL("modelInstalled()"))  # Record selection.
            return
            # End of thread func.

        # Start the thread.
        irowid, _ = rowid.toInt()
        t = threading.Thread(target=installThread, args=(irowid, ))
        if irowid in self.installThreads:  # If old thread is there.
            print >> sys.stderr, (
                "table rowid %d already has a thread running, stop it" %
                irowid)
            self.installThreads[irowid][1] = False
        self.installThreads[irowid] = [t, True]
        t.start()