def on_btnPreProccess_clicked(self):
"""
Slot documentation goes here.
"""
source = self.edit_source_preprocessing_tab.text()
target = self.edit_target_preprocessing_tab.text()
lm_text = self.edit_lm_text_preprocessing_tab.text()
output = self.edit_output_preprocessing_tab.text()
source_language = self.preprocessing_source_language
target_language = self.preprocessing_target_language
if not source:
doAlert("Please choose a source text first.")
return
elif not target:
doAlert("Please choose a target text first.")
return
elif not output:
doAlert("Please choose an output directory first.")
return
elif not source_language:
doAlert(
"Please choose an preprocessing_source_language directory first."
)
return
elif not target_language:
doAlert(
"Please choose an preprocessing_target_language directory first."
)
return
else:
text = self.core._prepare_corpus(output, source_language,
target_language, source, target,
lm_text)
self.results_preprocessing.setText(text)
def on_buttonBox_accepted(self):
"""
Slot documentation goes here.
"""
def checkEmpty(mystr):
return len(str(mystr).strip()) <= 0
# Check everything.
self.modelName = self.editName.text()
if checkEmpty(self.modelName):
doAlert("Please provide non-empty Model Name")
return
if self.grpBoxInternet.isChecked():
self.source = self.editInternet.text()
self.sourceMode = "Internet"
elif self.grpBoxLocal.isChecked():
self.source = self.editLocal.text()
self.sourceMode = "Local"
if not os.path.exists(str(self.source)):
doAlert("Please provide valid local file as source")
return
else:
doAlert("Please provide Install Source")
return
if checkEmpty(self.source):
doAlert("Please provide non-empty Install Source")
return
self.dest = self.editPath.text()
if checkEmpty(self.dest):
doAlert("Please provide non-empty Install Destination Folder")
return
self.accept()
def detect(self):
pf = platform.system()
if pf == 'Windows':
self.mosesPath = self.findRegistryPath()
if self.mosesPath:
return self.checkMosesInstall()
else:
if not doQuestion(
'Cannot find Moses installation, click "Yes" to '
'manually set the Moses path, click "No" to exit.'):
return False
# If not found, use a dialog.
startdir = 'C:\\'
if "ProgramFiles(x86)" in os.environ:
startdir = os.environ["ProgramFiles(x86)"]
elif "ProgramFiles" in os.environ:
startdir = os.environ["ProgramFiles"]
else:
pass
dialog = QFileDialog(None, directory=startdir)
dialog.setFileMode(QFileDialog.Directory)
dialog.setViewMode(QFileDialog.Detail)
dialog.setOptions(QFileDialog.ShowDirsOnly)
if dialog.exec_():
self.mosesPath = str(dialog.selectedFiles()[0])
return self.checkMosesInstall()
else:
doAlert("Failed to find Moses Installation path, exit.")
return False
else:
doAlert("Platform %s not supported yet" % pf)
return False
def on_buttonBox_accepted(self):
"""
Slot documentation goes here.
"""
def checkEmpty(mystr):
return len(str(mystr).strip()) <= 0
# Check everything.
self.modelName = self.editName.text()
if checkEmpty(self.modelName):
doAlert("Please provide non-empty Model Name")
return
if self.grpBoxInternet.isChecked():
self.source = self.editInternet.text()
self.sourceMode = "Internet"
elif self.grpBoxLocal.isChecked():
self.source = self.editLocal.text()
self.sourceMode = "Local"
if not os.path.exists(str(self.source)):
doAlert("Please provide valid local file as source")
return
else:
doAlert("Please provide Install Source")
return
if checkEmpty(self.source):
doAlert("Please provide non-empty Install Source")
return
self.dest = self.editPath.text()
if checkEmpty(self.dest):
doAlert("Please provide non-empty Install Destination Folder")
return
self.accept()
def detect(self):
pf = platform.system()
if pf == 'Windows':
self.mosesPath = self.findRegistryPath()
if self.mosesPath:
return self.checkMosesInstall()
else:
if not doQuestion(
'Cannot find Moses installation, click "Yes" to '
'manually set the Moses path, click "No" to exit.'):
return False
# If not found, use a dialog.
startdir = 'C:\\'
if "ProgramFiles(x86)" in os.environ:
startdir = os.environ["ProgramFiles(x86)"]
elif "ProgramFiles" in os.environ:
startdir = os.environ["ProgramFiles"]
else:
pass
dialog = QFileDialog(None, directory=startdir)
dialog.setFileMode(QFileDialog.Directory)
dialog.setViewMode(QFileDialog.Detail)
dialog.setOptions(QFileDialog.ShowDirsOnly)
if dialog.exec_():
self.mosesPath = str(dialog.selectedFiles()[0])
return self.checkMosesInstall()
else:
doAlert("Failed to find Moses Installation path, exit.")
return False
else:
doAlert("Platform %s not supported yet" % pf)
return False
def newEntry(self):
import random
rec = self.record()
for i in xrange(1, 10):
rec.setValue(i, QVariant(str(random.random())))
self.insertRecord(-1, rec)
doAlert(self.query().lastInsertId().toString())
def newEntry(self):
import random
rec = self.record()
for i in xrange(1, 10):
rec.setValue(i, QVariant(str(random.random())))
self.insertRecord(-1, rec)
doAlert(self.query().lastInsertId().toString())
def checkMosesInstall(self):
for func in (self.getMosesCmd, self.getTokenizer, self.getDetokenizer, self.getTruecase, self.getDetruecase):
if not os.path.exists(func()):
doAlert(
"Missing executables in Moses installation path [%s], "
"exit." % self.mosesPath)
return False
return True
def checkMosesInstall(self):
for func in (self.getMosesCmd, self.getTokenizer, self.getDetokenizer,
self.getTruecase, self.getDetruecase):
if not os.path.exists(func()):
doAlert("Missing executables in Moses installation path [%s], "
"exit." % self.mosesPath)
return False
return True
def on_btnMachineTranslation_clicked(self):
source = self.edit_source_machine_translation_tab.text()
if not source:
doAlert("Please choose a source text first.")
return
text = self.migrated_backend_main._machine_translation(
source, self.chooseModel).decode('utf8')
self.results_machine_translation.setText(text)
def on_btnMachineTranslation_clicked(self):
source = self.edit_source_machine_translation_tab.text()
if not source:
doAlert("Please choose a source text.")
return
self.results_machine_translation.setText(
"Running decoder, please wait\n\n")
text = self.core._machine_translation(source,
self.chooseModel).decode('utf8')
self.results_machine_translation.setText(text)
def on_btnTraining_clicked(self):
"""
Slot documentation goes here.
"""
text = self.core._train()
if text == "ERR":
text = "ERROR: missing corpus to train model"
doAlert("Please provide corpus to train model")
self.tabWidget.setCurrentIndex(0)
return
else:
self.results_training.setText(text)
def on_buttonBox_accepted(self):
"""
Slot documentation goes here.
"""
current = self.selTableView.currentIndex()
if not current:
doAlert("Please choose a model to start.")
return
record = self.model.record(current.row())
self.ID = record.value("ID").toString()
self.modelName = record.value("name").toString()
self.srcLang = record.value('srclang').toString()
self.trgLang = record.value('trglang').toString()
self.path = record.value("path").toString()
self.mosesini = record.value("mosesini").toString()
self.accept()
def on_buttonBox_accepted(self):
"""
Slot documentation goes here.
"""
current = self.selTableView.currentIndex()
if not current:
doAlert("Please choose a model to start.")
return
record = self.model.record(current.row())
self.ID = record.value("ID").toString()
self.modelName = record.value("name").toString()
self.srcLang = record.value('srclang').toString()
self.trgLang = record.value('trglang').toString()
self.path = record.value("path").toString()
self.mosesini = record.value("mosesini").toString()
self.accept()
def detect(self):
doQuestion('Cannot find Moses installation, click "Yes" to '
'manually set the Moses path, click "No" to exit.')
# If not found, use a dialog.
startdir = 'C:\\'
if "ProgramFiles(x86)" in os.environ:
startdir = os.environ["ProgramFiles(x86)"]
elif "ProgramFiles" in os.environ:
startdir = os.environ["ProgramFiles"]
else:
pass
dialog = QFileDialog(None, directory=startdir)
dialog.setFileMode(QFileDialog.Directory)
dialog.setViewMode(QFileDialog.Detail)
dialog.setOptions(QFileDialog.ShowDirsOnly)
if dialog.exec_():
self.mosesPath = str(dialog.selectedFiles()[0])
return self.mosesPath
else:
doAlert("Failed to find Moses Installation path, exit.")
return "Failed to find Moses Installation path, exit."
def on_btnEvaluation_clicked(self):
"""
Slot documentation goes here.
"""
source = self.edit_source_evaluation_tab.text()
target = self.edit_target_evaluation_tab.text()
output = self.edit_output_evaluation_tab.text()
if not source:
doAlert("Please choose a source text first.")
return
elif not target:
doAlert("Please choose a target text first.")
return
elif not output:
doAlert("Please choose an output directory first.")
return
checkbox_indexes = [
] #[False] * 8 #checkbox_indexes["WER","PER","HTER", "BLEU","BLEU2GRAM","BLEU3GRAM"]
checkbox_indexes.append(self.btn_check_WER.isChecked())
checkbox_indexes.append(self.btn_check_PER.isChecked())
checkbox_indexes.append(self.btn_check_HTER.isChecked())
checkbox_indexes.append(self.btn_check_BLEU.isChecked())
checkbox_indexes.append(self.btn_check_BLEU2GRAM.isChecked())
checkbox_indexes.append(self.btn_check_BLEU3GRAM.isChecked())
checkbox_indexes.append(self.btn_check_BLEU4GRAM.isChecked())
text = self.migrated_backend_main._evaluate(checkbox_indexes, source,
target)
self.results_evaluation.setText(text)
def __init__(self, model, moses):
super(Engine, self).__init__()
self.model = model
self.moses = moses
self.check(self.model)
# Determine how many steps by num of modules in the model directory
# (moses + tok/detok + ...) + 1
modelini = model['path'] + os.sep + 'model' + os.sep + 'model.ini'
if not os.path.exists(modelini):
raise Exception(
"Model ini file doesn't exist, please check model dir %s" %
modelini)
cp = ConfigParser.RawConfigParser()
cp.readfp(open(modelini))
truemodel = None
try:
truemodel = cp.get("Preprocess", 'Truemodel')
truemodel_path = os.path.join(model['path'], 'model', truemodel)
if not os.path.exists(truemodel_path):
doAlert("Truemodel doesn't exist, ignore %s" % truemodel)
truemodel = None
except:
# doAlert("Truemodel not configured")
truemodel = None
self.cmds = []
# tok
self.cmds.append(
'"%s" -q -l %s -noxml' %
(self.moses.getTokenizer(), str(model['srclang']).lower()))
if truemodel is not None:
self.cmds.append('"%s" -model "%s"' %
(self.moses.getTruecase(), truemodel))
self.cmds.append('"%s" -f moses.ini' % self.moses.getMosesCmd())
self.cmds.append('"%s"' % self.moses.getDetruecase())
self.cmds.append(
'"%s" -q -noxml -l %s' %
(self.moses.getDetokenizer(), str(model['trglang']).lower()))
self.process = []
def __init__(self, model, moses):
super(Engine, self).__init__()
self.model = model
self.moses = moses
self.check(self.model)
# Determine how many steps by num of modules in the model directory
# (moses + tok/detok + ...) + 1
modelini = model['path'] + os.sep + 'model' + os.sep + 'model.ini'
if not os.path.exists(modelini):
raise Exception(
"Model ini file doesn't exist, please check model dir %s"
% modelini)
cp = ConfigParser.RawConfigParser()
cp.readfp(open(modelini))
truemodel = None
try:
truemodel = cp.get("Preprocess", 'Truemodel')
truemodel_path = os.path.join(model['path'], 'model', truemodel)
if not os.path.exists(truemodel_path):
doAlert("Truemodel doesn't exist, ignore %s" % truemodel)
truemodel = None
except:
# doAlert("Truemodel not configured")
truemodel = None
self.cmds = []
# tok
self.cmds.append(
'"%s" -q -l %s -noxml'
% (self.moses.getTokenizer(), str(model['srclang']).lower()))
if truemodel is not None:
self.cmds.append(
'"%s" -model "%s"' % (self.moses.getTruecase(), truemodel))
self.cmds.append('"%s" -f moses.ini' % self.moses.getMosesCmd())
self.cmds.append('"%s"' % self.moses.getDetruecase())
self.cmds.append(
'"%s" -q -noxml -l %s'
% (self.moses.getDetokenizer(), str(model['trglang']).lower()))
self.process = []
def on_btnTranslate_clicked(self):
"""
Slot documentation goes here.
"""
if self.engine is None:
doAlert("Please load MT model first.")
return
self.btnTranslate.setEnabled(False)
self.editTrg.setText("")
try:
texts = str(self.editSrc.toPlainText().toUtf8()).split('\n')
trans = []
for text in texts:
if text.strip() == "":
trans.append(text)
else:
trans.append(
self.engine.translate(text.replace(
'\r', ' ').strip()).decode('utf8'))
self.editTrg.setText('\n'.join(trans))
except Exception, e:
print >> sys.stderr, str(e)
doAlert("Translation failed!")
def installModel(self, installParam):
dest = installParam['dest']
# Make dir.
if not os.path.exists(dest):
try:
os.makedirs(str(dest))
except:
doAlert("Failed to create install directory: %s" % dest)
return
# Create entry in db.
rec = self.record()
rec.setValue('name', installParam['modelName'])
rec.setValue('status', 'Fetching Source...')
rec.setValue('path', dest)
rec.setValue('origin', installParam['source'])
rec.setValue('originMode', installParam['sourceMode'])
rec.setValue('date', QDateTime.currentDateTime())
rec.setValue('deleted', 'False')
self.insertRecord(-1, rec)
rowid = self.query().lastInsertId()
# Start thread.
def installThread(irowid):
# Find the current row in model.
def updateRecord(keyvalues):
curRow = None
# TODO: use binary search instead of linear
for i in xrange(0, self.rowCount()):
if self.record(i).value("ID") == rowid:
curRow = i
break
if curRow is not None:
record = self.record(curRow)
for key in keyvalues:
record.setValue(key, keyvalues[key])
self.changeRecord(curRow, record)
return curRow
def checkExit():
# Check thread is ok to run.
if irowid not in self.installThreads or not self.installThreads[irowid][1]:
return True
else:
return False
def markExit():
if irowid in self.installThreads: # Set thread to dead.
self.installThreads[irowid][1] = False
def statusMessageLogMarkExit(status=None, message=None,
exception=None):
if status is not None:
updateRecord({'status': status})
if message is not None:
self.emit(SIGNAL("messageBox(QString)"), message)
print >> sys.stderr, message
if exception is not None:
print >> sys.stderr, str(exception)
markExit()
# 1. Download or copy from local.
# Where the downloaded/copied zip file is:
destFile = os.path.join(str(dest), "model.zip")
# Where the unzipped contents are:
destDir = os.path.join(str(dest), "model")
if installParam['sourceMode'] == 'Local':
fin = fout = None
try:
inFile = str(installParam['source'])
total_size = os.path.getsize(inFile)
fin = open(inFile, 'rb')
chunk_size = 52428800 # 50MB as chunk size
fout = open(destFile, 'wb')
content = fin.read(chunk_size)
download_size = content_size = len(content)
lastMsg = ""
while content_size > 0:
# Check if thread is notified as exit.
if checkExit():
return statusMessageLogMarkExit()
fout.write(content)
if total_size > 0:
msg = 'COPY %.0f%%' % (
download_size * 100.0 / total_size)
else:
msg = 'COPY %d MB' % (download_size / 1048576)
if msg != lastMsg:
updateRecord({'status': msg})
lastMsg = msg
content = fin.read(chunk_size)
content_size = len(content)
download_size += content_size
except Exception as e:
return statusMessageLogMarkExit(
status=(
'Failed copying from: %s'
% installParam['source']),
message=(
"Failed copy model: %s"
% installParam['modelName']),
exception=e)
finally:
if fin:
fin.close()
if fout:
fout.close()
elif installParam['sourceMode'] == 'Internet':
conn = fout = None
try:
conn = urllib2.urlopen(str(installParam['source']))
total_size = int(conn.headers['Content-Length'])
chunk_size = 1048576 # 1MB as chunk size
fout = open(destFile, 'wb')
content = conn.read(chunk_size)
download_size = content_size = len(content)
lastMsg = ""
while content_size > 0:
# Check if thread is notified as exit.
if checkExit():
return statusMessageLogMarkExit()
fout.write(content)
if total_size > 0:
msg = 'DOWNLOAD %.0f%%' % (
download_size * 100.0 / total_size)
else:
msg = 'DOWNLOAD %d MB' % (download_size / 1048576)
if msg != lastMsg:
updateRecord({'status': msg})
lastMsg = msg
content = conn.read(chunk_size)
content_size = len(content)
download_size += content_size
except Exception as e:
return statusMessageLogMarkExit(
status=(
'Failed downloading from: %s'
% installParam['source']),
message=(
"Failed download model: %s"
% installParam['modelName']),
exception=e)
finally:
if conn:
conn.close()
if fout:
fout.close()
else:
return statusMessageLogMarkExit(
status='Unsupported source mode: %s'
% installParam['sourceMode'])
# 2. Unzip.
zfile = fout = None
try:
zfile = zipfile.ZipFile(destFile)
# Check property files.
if "model.ini" not in zfile.namelist():
return statusMessageLogMarkExit(
status=(
'Missing model.ini in model file: %s'
% installParam['sourceMode']),
message=(
"Invalid modle file format because model.ini "
"is missing in the zipped model file, exit "
"installation for model %s"
% installParam['modelName']))
chunk_size = 52428800 # 50MB as chunk size
# Get file size uncompressed.
total_size = 0
for name in zfile.namelist():
total_size += zfile.getinfo(name).file_size
download_size = 0
lastMsg = ""
for i, name in enumerate(zfile.namelist()):
(dirname, filename) = os.path.split(name)
outDir = os.path.join(destDir, dirname)
if not os.path.exists(outDir):
os.makedirs(outDir)
if filename:
fin = zfile.open(name, 'r')
outFile = os.path.join(destDir, name)
fout = open(outFile, 'wb')
content = fin.read(chunk_size)
content_size = len(content)
download_size += content_size
while content_size > 0:
# Check if thread is notified as exit.
if checkExit():
return statusMessageLogMarkExit()
fout.write(content)
if total_size > 0:
msg = 'UNZIP %.0f%%' % (
download_size * 100.0 / total_size)
else:
msg = 'UNZIP %d MB' % (
download_size / 1048576)
if msg != lastMsg:
updateRecord({'status': msg})
lastMsg = msg
content = fin.read(chunk_size)
content_size = len(content)
download_size += content_size
fin.close()
fout.close()
except Exception as e:
return statusMessageLogMarkExit(
status=(
'Failed unzipping from: %s' % installParam['source']),
message=(
"Failed unzip model: %s" % installParam['modelName']),
exception=e)
finally:
if zfile:
zfile.close()
if fin:
fin.close()
if fout:
fout.close()
# 3 Post process and check data validity.
try:
modelini = os.path.join(destDir, "model.ini")
cp = ConfigParser.RawConfigParser()
cp.read(modelini)
mosesini = os.path.join(destDir, 'moses.ini')
if not os.path.exists(mosesini):
raise Exception("Moses ini doesn't exist")
updateRecord({
'srclang': cp.get("Language", 'Source Language').upper(),
'trglang': cp.get("Language", 'Target Language').upper(),
'mosesini': mosesini},
)
except Exception as e:
return statusMessageLogMarkExit(
status='ERROR model format %s' % installParam['source'],
message=(
"Unspported model format: %s"
% installParam['modelName']),
exception=e)
statusMessageLogMarkExit(
status='READY',
message="Model %s Installed!" % installParam['modelName'])
# Send new model signal.
self.emit(SIGNAL("modelInstalled()")) # Record selection.
return
# End of thread func.
# Start the thread.
irowid, _ = rowid.toInt()
t = threading.Thread(target=installThread, args=(irowid, ))
if irowid in self.installThreads: # If old thread is there.
print >> sys.stderr, (
"table rowid %d already has a thread running, stop it"
% irowid)
self.installThreads[irowid][1] = False
self.installThreads[irowid] = [t, True]
t.start()
def on_btnPostEditing_clicked(self):
source = self.edit_source_post_editing.text()
#print source
target = self.edit_target_post_editing.text()
#print target
output = self.edit_output_post_editing.text()
if not source and self.btn_bilingual_post_edition.isChecked():
doAlert("Please choose a source text first.")
return
if not target:
doAlert("Please choose a target text first.")
return
if not output:
doAlert("Please choose an output directory first.")
return
self.source_text = []
self.unchanged_target_text = []
self.target_text = []
with open(target) as fp:
for line in fp:
line = line.decode("utf-8")
if line != '\n':
self.unchanged_target_text.append(line)
self.target_text.append(line)
if self.btn_bilingual_post_edition.isChecked():
with open(source) as fp:
for line in fp:
line = line.decode("utf-8")
if line != '\n':
self.source_text.append(line)
else:
self.source_text = self.target_text
self.post_editing_data["source"] = self.source_text
self.post_editing_data["target"] = self.target_text
self.table_offset_PostEdition = 0
self.table_rows_PostEdition = 100 #delete these vars if nav butons not used <<,>>,+,-
self.table_rows_Differences = 50
self.table_offset_Differences = 0
self.lastChangedSegmentIndex = -1
self.modified_references_indices = []
self.modified_table_items_coordinates = []
self.lastChangedTableItemCoordinates = (-1, -1)
#CHECK WHICH OF THE FOLLOWING SHOULD BE CLEARED
#TO SHOW DIFFERENCES CORRECTLY AFTER REOADING FILES FOR PE
#AGREGAR LIMPIAR BUSQEUDAS
self.saved_modified_references = []
self.differences = None
#self.post_editing_data = {}
self.differences_data = {}
self.unmodified_target = []
self.modified_target = []
self.last_changed_item_in_post_edition = None
self.last_selected_search = None
self.log = {}
self.update_table_PostEdition()
#THE FOLLOWING CODE CREATES A NICE SIMPLE TABLE WITH WRAPPED
#TEXT IN THE CELLS, COULD BE USED AS REFERENCE TO REFACTOR CODE
#P self.table_post_editing.clear()
#Pfor y, key in enumerate(sorted(self.post_editing_data.keys())):
#P for x, item in enumerate(self.post_editing_data[key]):
#P
#P newitem = QTableWidgetItem(textwrap.fill(item, width=70))
#Pif key != "Post-edited":
#P newitem.setReadOnly(True)
#P self.table_post_editing.setItem(x, y, newitem)
#P self.table_post_editing.setTextElideMode(QtCore.Qt.ElideNone)
#P self.table_post_editing.resizeColumnsToContents()
#P self.table_post_editing.resizeRowsToContents()
#P self.table_post_editing.horizontalHeader().setResizeMode(QtGui.QHeaderView.Stretch)
self.PE_table_controls_groupBox.show()
self.PE_search_groupBox.show()
self.table_post_editing.show()
def on_btnEvaluation_clicked(self):
"""
Slot documentation goes here.
"""
hyp = self.edit_hyp_evaluation_tab.text()
ref = self.edit_ref_evaluation_tab.text()
output = "" #self.edit_output_evaluation_tab.text()
if not hyp:
doAlert("Please choose MTed text.")
return
elif not ref:
doAlert("Please choose reference text.")
return
#elif not output:
## doAlert("Please choose an output directory first.")
# return
if self.btn_check_WER.isChecked():
pathEval = (os.path.dirname(os.path.realpath(__file__)))
cmd = "perl " + pathEval + "/evaluation_scripts" + "/WER.pl" + " -t " + hyp + " -r " + ref
proc = subprocess.Popen([cmd],
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
shell=True)
out, err = proc.communicate()
output += "WER: %s (Execution errors: %s)\n" % (out, err)
#print cmd
if self.btn_check_PER.isChecked():
cmd = "perl " + pathEval + "/evaluation_scripts" + "/PER.pl" + " -t " + hyp + " -r " + ref
proc = subprocess.Popen([cmd],
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
shell=True)
out, err = proc.communicate()
output += "PER: %s (Execution errors: %s)\n" % (out, err)
#print cmd
if self.btn_check_HTER.isChecked():
cmd = "perl " + pathEval + "/evaluation_scripts" + "/tercom_v6b.pl" + " -h " + hyp + " -r " + ref + " -o sum_doc "
proc = subprocess.Popen([cmd],
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
shell=True)
out, err = proc.communicate()
output += "HTER: %s (Execution errors: %s)\n" % (out, err)
#print cmd
if self.btn_check_BLEU.isChecked():
cmd = "perl " + pathEval + "/evaluation_scripts" + "/BLEU.pl " + ref + " < " + hyp
proc = subprocess.Popen([cmd],
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
shell=True)
out, err = proc.communicate()
output += "BLEU: %s (Execution errors: %s)\n" % (out, err)
#print cmd
self.results_evaluation.setText(output)
def on_btnPostEditing_clicked(self):
self.post_editing_data = {'Source/MT': [], 'Post-edited': []}
self.log = {}
self.modified_target = []
self.timestamps = []
self.table_post_editing.clear()
self.table_post_editing.blockSignals(True)
source = self.edit_source_post_editing.text()
# print source
target = self.edit_target_post_editing.text()
#print target
output = self.edit_output_post_editing.text()
if not source and self.btn_bilingual_post_edition.isChecked():
doAlert("Please choose a source text first.")
return
if not target:
doAlert("Please choose a target text first.")
return
if not output:
doAlert("Please choose an output directory first.")
return
with open(target) as fp:
for line in fp:
line = line.decode("utf-8")
if line != '\n':
self.post_editing_data['Post-edited'].append(line)
self.timestamps.append([(0, 0)
]) #initial time for PE timestamps
self.modified_target.append(None)
if self.btn_bilingual_post_edition.isChecked():
with open(source) as fp:
for line in fp:
line = line.decode("utf-8")
if line != '\n':
self.post_editing_data['Source/MT'].append(line)
else:
self.post_editing_data['Source/MT'] = self.post_editing_data[
'Post-edited']
self.table_post_editing.setRowCount(
len(self.post_editing_data['Post-edited']))
if len(self.post_editing_data['Source/MT']) != len(
self.post_editing_data['Post-edited']):
doAlert("Please choose an output directory first.")
return
for y, key in enumerate(['Source/MT', 'Post-edited']):
for x, item in enumerate(self.post_editing_data[key]):
newitem = QTableWidgetItem(textwrap.fill(item, width=70))
#print x, y
if key == 'Source/MT':
newitem.setFlags(QtCore.Qt.ItemIsEditable)
self.table_post_editing.setItem(x, y, newitem)
self.table_post_editing.setHorizontalHeaderLabels(
["Source/MT", "Post-edited"])
self.table_post_editing.setTextElideMode(QtCore.Qt.ElideNone)
self.table_post_editing.resizeColumnsToContents()
self.table_post_editing.resizeRowsToContents()
self.table_post_editing.horizontalHeader().setResizeMode(
QtGui.QHeaderView.Stretch)
self.table_post_editing.setMouseTracking(True)
#use the events below to calculate time spent per segment
self.table_post_editing.show()
self.table_post_editing.cellChanged.connect(self.endPESeg)
self.table_post_editing.cellEntered.connect(self.startPESeg)
self.table_post_editing.blockSignals(False)
def installModel(self, installParam):
dest = installParam['dest']
# Make dir.
if not os.path.exists(dest):
try:
os.makedirs(str(dest))
except:
doAlert("Failed to create install directory: %s" % dest)
return
# Create entry in db.
rec = self.record()
rec.setValue('name', installParam['modelName'])
rec.setValue('status', 'Fetching Source...')
rec.setValue('path', dest)
rec.setValue('origin', installParam['source'])
rec.setValue('originMode', installParam['sourceMode'])
rec.setValue('date', QDateTime.currentDateTime())
rec.setValue('deleted', 'False')
self.insertRecord(-1, rec)
rowid = self.query().lastInsertId()
# Start thread.
def installThread(irowid):
# Find the current row in model.
def updateRecord(keyvalues):
curRow = None
# TODO: use binary search instead of linear
for i in xrange(0, self.rowCount()):
if self.record(i).value("ID") == rowid:
curRow = i
break
if curRow is not None:
record = self.record(curRow)
for key in keyvalues:
record.setValue(key, keyvalues[key])
self.changeRecord(curRow, record)
return curRow
def checkExit():
# Check thread is ok to run.
if irowid not in self.installThreads or not self.installThreads[
irowid][1]:
return True
else:
return False
def markExit():
if irowid in self.installThreads: # Set thread to dead.
self.installThreads[irowid][1] = False
def statusMessageLogMarkExit(status=None,
message=None,
exception=None):
if status is not None:
updateRecord({'status': status})
if message is not None:
self.emit(SIGNAL("messageBox(QString)"), message)
print >> sys.stderr, message
if exception is not None:
print >> sys.stderr, str(exception)
markExit()
# 1. Download or copy from local.
# Where the downloaded/copied zip file is:
destFile = os.path.join(str(dest), "model.zip")
# Where the unzipped contents are:
destDir = os.path.join(str(dest), "model")
if installParam['sourceMode'] == 'Local':
fin = fout = None
try:
inFile = str(installParam['source'])
total_size = os.path.getsize(inFile)
fin = open(inFile, 'rb')
chunk_size = 52428800 # 50MB as chunk size
fout = open(destFile, 'wb')
content = fin.read(chunk_size)
download_size = content_size = len(content)
lastMsg = ""
while content_size > 0:
# Check if thread is notified as exit.
if checkExit():
return statusMessageLogMarkExit()
fout.write(content)
if total_size > 0:
msg = 'COPY %.0f%%' % (download_size * 100.0 /
total_size)
else:
msg = 'COPY %d MB' % (download_size / 1048576)
if msg != lastMsg:
updateRecord({'status': msg})
lastMsg = msg
content = fin.read(chunk_size)
content_size = len(content)
download_size += content_size
except Exception as e:
return statusMessageLogMarkExit(
status=('Failed copying from: %s' %
installParam['source']),
message=("Failed copy model: %s" %
installParam['modelName']),
exception=e)
finally:
if fin:
fin.close()
if fout:
fout.close()
elif installParam['sourceMode'] == 'Internet':
conn = fout = None
try:
conn = urllib2.urlopen(str(installParam['source']))
total_size = int(conn.headers['Content-Length'])
chunk_size = 1048576 # 1MB as chunk size
fout = open(destFile, 'wb')
content = conn.read(chunk_size)
download_size = content_size = len(content)
lastMsg = ""
while content_size > 0:
# Check if thread is notified as exit.
if checkExit():
return statusMessageLogMarkExit()
fout.write(content)
if total_size > 0:
msg = 'DOWNLOAD %.0f%%' % (download_size * 100.0 /
total_size)
else:
msg = 'DOWNLOAD %d MB' % (download_size / 1048576)
if msg != lastMsg:
updateRecord({'status': msg})
lastMsg = msg
content = conn.read(chunk_size)
content_size = len(content)
download_size += content_size
except Exception as e:
return statusMessageLogMarkExit(
status=('Failed downloading from: %s' %
installParam['source']),
message=("Failed download model: %s" %
installParam['modelName']),
exception=e)
finally:
if conn:
conn.close()
if fout:
fout.close()
else:
return statusMessageLogMarkExit(
status='Unsupported source mode: %s' %
installParam['sourceMode'])
# 2. Unzip.
zfile = fout = None
try:
zfile = zipfile.ZipFile(destFile)
# Check property files.
if "model.ini" not in zfile.namelist():
return statusMessageLogMarkExit(
status=('Missing model.ini in model file: %s' %
installParam['sourceMode']),
message=("Invalid modle file format because model.ini "
"is missing in the zipped model file, exit "
"installation for model %s" %
installParam['modelName']))
chunk_size = 52428800 # 50MB as chunk size
# Get file size uncompressed.
total_size = 0
for name in zfile.namelist():
total_size += zfile.getinfo(name).file_size
download_size = 0
lastMsg = ""
for i, name in enumerate(zfile.namelist()):
(dirname, filename) = os.path.split(name)
outDir = os.path.join(destDir, dirname)
if not os.path.exists(outDir):
os.makedirs(outDir)
if filename:
fin = zfile.open(name, 'r')
outFile = os.path.join(destDir, name)
fout = open(outFile, 'wb')
content = fin.read(chunk_size)
content_size = len(content)
download_size += content_size
while content_size > 0:
# Check if thread is notified as exit.
if checkExit():
return statusMessageLogMarkExit()
fout.write(content)
if total_size > 0:
msg = 'UNZIP %.0f%%' % (download_size * 100.0 /
total_size)
else:
msg = 'UNZIP %d MB' % (download_size / 1048576)
if msg != lastMsg:
updateRecord({'status': msg})
lastMsg = msg
content = fin.read(chunk_size)
content_size = len(content)
download_size += content_size
fin.close()
fout.close()
except Exception as e:
return statusMessageLogMarkExit(
status=('Failed unzipping from: %s' %
installParam['source']),
message=("Failed unzip model: %s" %
installParam['modelName']),
exception=e)
finally:
if zfile:
zfile.close()
if fin:
fin.close()
if fout:
fout.close()
# 3 Post process and check data validity.
try:
modelini = os.path.join(destDir, "model.ini")
cp = ConfigParser.RawConfigParser()
cp.read(modelini)
mosesini = os.path.join(destDir, 'moses.ini')
if not os.path.exists(mosesini):
raise Exception("Moses ini doesn't exist")
updateRecord(
{
'srclang': cp.get("Language",
'Source Language').upper(),
'trglang': cp.get("Language",
'Target Language').upper(),
'mosesini': mosesini
}, )
except Exception as e:
return statusMessageLogMarkExit(
status='ERROR model format %s' % installParam['source'],
message=("Unspported model format: %s" %
installParam['modelName']),
exception=e)
statusMessageLogMarkExit(status='READY',
message="Model %s Installed!" %
installParam['modelName'])
# Send new model signal.
self.emit(SIGNAL("modelInstalled()")) # Record selection.
return
# End of thread func.
# Start the thread.
irowid, _ = rowid.toInt()
t = threading.Thread(target=installThread, args=(irowid, ))
if irowid in self.installThreads: # If old thread is there.
print >> sys.stderr, (
"table rowid %d already has a thread running, stop it" %
irowid)
self.installThreads[irowid][1] = False
self.installThreads[irowid] = [t, True]
t.start()
