def test_ec(self):
logging.info("test_")
line = '我们现今所使用的大部分舒学符号' # ，你们用的什么婊点符号
logging.info('input sentence is: %s', line)
corrected_sent, correct_ranges = correct(line)
logging.info('corrected_sent: %s', corrected_sent)
logging.info('correct_ranges: %s', correct_ranges)
def eval_bcmi_data(data_path, verbose=False):
sentence_size = 1
right_count = 0
right_result = dict()
wrong_result = dict()
with open(data_path, 'r', encoding='utf-8') as f:
for line in f:
line = line.strip()
error_sentence, right_sentence = get_bcmi_corpus(line)
if not error_sentence:
continue
pred_sentence, pred_detail = corrector.correct(error_sentence)
if verbose:
print('input sentence:', error_sentence)
print('pred sentence:', pred_sentence)
print('right sentence:', right_sentence)
sentence_size += 1
if right_sentence == pred_sentence:
right_count += 1
right_result[error_sentence] = [right_sentence, pred_sentence]
else:
wrong_result[error_sentence] = [right_sentence, pred_sentence]
if verbose:
print('right count:', right_count, ';sentence size:', sentence_size)
return right_count / sentence_size, right_result, wrong_result
def run(image_path):
image_path_ = image_path.rsplit('.', 1)[0]
preprocess_image_path = image_path_ + '_bw.jpg'
audio_path = image_path_ + '.wav'
image = cv2.imread(image_path)
image_bw = segmenter.preprocess(image)
cv2.imwrite(preprocess_image_path, image_bw)
lines = segmenter.segment_lines(image_bw)
#classified_text = "රජගහා විහාරෆ හඤූතර තරඔක් කදූභැටියරී වර පසින් ඇහ් අ"
classified_text = ""
for i, line in enumerate(lines):
character_images = segmenter.segment_line(line, i)
for character_image in character_images:
classified_text += classifier.classify(character_image)
#classified_text += "2"
#classified_text = "රජගහා විහාරෆ හඤූතර තරඔක් කදූභැටියරී වර පසින් ඇහ්"
# remove extra spaces
#classified_text = ""
classified_text = classified_text.strip()
classified_text = " ".join(classified_text.split())
#join modifiers
classified_text = classifier.join_modifiers(classified_text)
print classified_text
corrected_words = corrector.correct(classified_text)
corrected_text = ""
for words in corrected_words:
corrected_text += words[0].encode("utf-8") + " "
print corrected_text
synthesized_data = synthesizer.synthesize(corrected_text)
audio_outfile = wave.open(audio_path, 'wb')
audio_outfile.setparams(synthesized_data[0][0])
for i in range(0, len(synthesized_data), 1):
audio_outfile.writeframes(synthesized_data[i][1])
# classified_text = "රජගහා විහාරෆ හඤූතර තරඔක් කදූභැටියරී වර පසින් ඇහ් අ"
# corrected_words = [[1111111,2,"old1"],[3233333,4,"old2"]]
return classified_text, corrected_words, image_path_, audio_path
def ocr(warp_id):
'''
Input -
{
[crop: {
[left: int]
[right: int]
[top: int]
[bottom: int]
}]
}
Output -
content
translated
'''
img_path = get_warp_image_path(warp_id)
img = cv2.imread(img_path, flags=cv2.IMREAD_GRAYSCALE)
if img is None:
fl.abort(404)
json = request.get_json()
try:
if "crop" in json:
crop_obj = json["crop"]
left = none_or_int(crop_obj.get("left"))
right = none_or_int(crop_obj.get("right"))
top = none_or_int(crop_obj.get("top"))
bottom = none_or_int(crop_obj.get("bottom"))
img = img[top:bottom, left:right]
except (KeyError, ValueError):
fl.abort(400)
with NamedTemporaryFile(suffix='.ppm') as f:
cv2.imwrite(f.name, img)
proc = Popen([os.path.join(CURRENT_DIRECTORY, "nhocr"),
f.name,
"-o",
"-",
"-block"],
stdout=PIPE)
buf = u"".join([line.decode("utf-8")
for line in proc.stdout])
proc.wait()
buf = u"\n".join(buf.splitlines())
content = correct(buf)
print content.encode("utf-8")
annotated, translated = ja_ko_translator(content)
return fl.jsonify(
content=annotated,
translated=translated
)
def eval_sighan_corpus(pkl_path, verbose=False):
sighan_data = load_pkl(pkl_path)
total_count = 1
right_count = 0
right_result = dict()
wrong_result = dict()
for error_sentence, right_detail in sighan_data:
pred_sentence, pred_detail = corrector.correct(error_sentence)
if verbose:
print('input sentence:', error_sentence)
print('pred sentence:', pred_sentence)
for (right_loc, right_w, right_r) in right_detail:
total_count += 1
# if right_r == pred_r:
# right_count += 1
# right_result[error_sentence] = [right_r, pred_r]
# else:
# wrong_result[error_sentence] = [right_r, pred_r]
if verbose:
print('right: {} => {} , index: {}'.format(
right_w, right_r, right_loc))
# if verbose:
# print('right count:', right_count, ';total count:', total_count)
return right_count / total_count, right_result, wrong_result
("xsel --version", 'You need to install xsel'),
("xdotool --help", 'You need to install xdotool'),
('xte -help > /dev/null', 'You need to install xautomation'),
("xvkbd -no-sync -no-repeat -xsendevent -text '\C' > /dev/null",
'You need to install xvkbd'),
)
try:
for command, error_text in commands:
ret_code = call(command, shell=True)
if ret_code != 0:
raise OSError(error_text)
except OSError as e:
print('\n' + '!' * 30)
print("Test failed:\n" + str(e))
print('!' * 30 + '\n')
exit(1)
if __name__ == '__main__':
print('Keyboard layout corrector. \r\nTesting system environment...')
check_env()
print('Test example:')
test_str = 'ghbdtn'
corrected_str = correct(test_str)
print('"%s" -> "%s"' % (test_str, corrected_str))
if corrected_str == 'привет':
print('Test passed.')
#!/usr/bin/env python
# -*- coding: utf-8 -*-
from subprocess import call
from corrector import correct
__author__ = 'Odarchenko N.D.'
if __name__ == '__main__':
print('Keyboard layout corrector. \r\nTesting system environment...')
commands = (
("xsel --clear", 'You need to install xsel'),
('setxkbmap > /dev/null', 'You need to install setxkbmap'),
("xvkbd -xsendevent -text '\C'>/dev/null", 'You need to install xvkbd'),
)
try:
for command, error_text in commands:
ret_code = call(command, shell=True)
if ret_code != 0:
raise OSError(error_text)
except OSError as e:
print('\n' + '!' * 30)
print("Test failed:\n" + str(e))
print('!' * 30 + '\n')
exit(1)
print('Test example:')
test = 'ghbdtn'
print('"%s" -> "%s"' % (test, correct(test)))
print('Test passed.')
# -*- coding: utf-8 -*-
import numpy as np
# import segmenter
# import classifier
import corrector
import codecs
# import synthesizer
file = open('corrector/input.txt', 'r')
input_text = file.read()
# input_text = raw_text[3:]
print input_text
corrected_text = corrector.correct(input_text)
print '%s' % ''.join([' , '.join('%s' % ' '.join(e) for e in corrected_text)])
# corrected_text = corrector.correct(classified_text)
#print "corrected text : " + corrected_text
# synthesized_voice = synthesizer.synthesize(corrected_text)
obj = json.loads(resp_payload)
resultData = obj["resultData"]
annotated = content
hurigana = obj.get("hurigana", [])
for item in hurigana:
from_ = item["z"]
to = item["h"]
annotated = annotated.replace(from_, from_ + "(" + to + ")")
return (annotated, resultData)
return trans
if __name__ == '__main__':
trans = build_trans()
src = sys.stdin.read().decode("utf-8")
print "[Original]"
print src.encode("UTF-8")
src = correct(src)
orig, result = trans(src)
if result is None:
sys.exit(1)
result = result.encode("utf-8")
print "[Translated]"
sys.stdout.write(orig)
sys.stdout.write("\n")
sys.stdout.write(result)
#!/usr/bin/env python
# -*- coding: utf-8 -*-
from subprocess import call
from corrector import correct
__author__ = 'Odarchenko N.D.'
if __name__ == '__main__':
print('Keyboard layout corrector. \r\nTesting system environment...')
commands = (
("xsel --clear", 'You need to install xsel'),
('setxkbmap > /dev/null', 'You need to install setxkbmap'),
("xvkbd -xsendevent -text '\C'>/dev/null",
'You need to install xvkbd'),
)
try:
for command, error_text in commands:
ret_code = call(command, shell=True)
if ret_code != 0:
raise OSError(error_text)
except OSError as e:
print('\n' + '!' * 30)
print("Test failed:\n" + str(e))
print('!' * 30 + '\n')
exit(1)
print('Test example:')
test = 'ghbdtn'
print('"%s" -> "%s"' % (test, correct(test)))
print('Test passed.')
def correct_content(self, content, language):
# TODO to be moved to LT processes class
# Segments and sends the content to LT according to the
# public api rate limits
# http://wiki.languagetool.org/public-http-api
if os.path.isfile(self.outpath):
msg = 'title exists in cache: %s'%self.title
print(self.outpath)
print(msg)
logging.info(msg)
with open(self.outpath) as f:
responses = json.load(f)
return responses
else:
responses = {'title': self.title, 'results': []}
if self.online:
per_req_size_limit = 6e3 # KB
sentences = content.split('. ')
requests = []
test_chunks = []
chunk = []
for sentence in sentences:
chunk.append(sentence)
total_chunk = '. '.join(chunk)
if sys.getsizeof(total_chunk) > per_req_size_limit:
requests.append(total_chunk)
test_chunks.append((chunk[0], chunk[-1]))
chunk = []
if chunk:
# add last chunk
requests.append('. '.join(chunk))
test_chunks.append((chunk[0], chunk[-1]))
# send requests to api
# TODO smarter rate limit control needed
total_requests = len(requests)
for i, request in enumerate(requests):
try:
response = api.check(request,
api_url=self.languagetool,
lang=language)
# TODO check language, if confidence lower than 0.90 resend
except Exception as e:
msg = "%s language error. Trying to detect the language."\
""%language
logging.warning(msg)
response = api.check(test_chunks[i][1],
api_url=self.languagetool,
lang=language)
language_bottom = response['language']['detectedLanguage']['code']
response = api.check(test_chunks[i][0],
api_url=self.languagetool,
lang=language_bottom)
language_top = response['language']['detectedLanguage']['code']
if language != language_top:
language = language_top
else:
language = language_bottom
msg = "%s detected as new language"%language
logging.info(msg)
response = api.check(request,
api_url=self.languagetool,
lang=language)
message = '%i/%i response sent'%(i+1, total_requests)
print(message)
logging.info(message)
if i+1 != total_requests:
# wait at all except the last LT api call
time.sleep(4)
responses['results'].append({'content': request,
'response': response})
else:
chunks = corrector.get_chunks(content)
corrector.correct(chunks, responses)
with open(self.outpath, 'w') as out:
json.dump(responses, out, indent = 2)
return responses
def correcttest(self):
if not login.current_user.is_authenticated:
return redirect(url_for('.login_view'))
found = tests.find()
if request.method == 'POST':
title = request.form.get('title')
tfound = tests.find_one({"TITLE": title})
# if request does not contain the file part
if 'file' not in request.files:
flash('No file was sent', category='danger')
return redirect(request.url)
file = request.files['file']
# if user does not select file, browser will
# submit an empty part without filename
if file.filename == '':
flash('No file was selected', category='danger')
return redirect(request.url)
# if file was selected but of the wrong type
if file and not self.allowed_file(file.filename):
flash('Please select a .pdf file', category='danger')
return redirect(request.url)
# if file was selected & is correct type
if file and self.allowed_file(file.filename):
as_jpeg = PDF2jpg.convert(file)
# fetches the answer key corresponding to the test
key = self.getAnswerKey(tfound)
print(key)
# corrects the test image using the answer key
# returns (location, score, correct, AMOUNT)
loc, corr, am, sc, flag = corrector.correct(as_jpeg, key)
curr_time = time.localtime()
ctime = time.strftime('%a, %d %b %Y %H:%M:%S GMT', curr_time)
corrected = {
"TEST": title,
"SCORE": sc,
"CORRECT": corr,
"AMOUNT": am,
"FLAG": flag,
"CREATED": ctime
}
# insert into the db
result = results.insert_one(corrected)
# obtain the MongoDB ObjectID in string form
id = str(result.inserted_id) + '.png'
# move and give a unique name to the test image for storage
# destination = path to file
destination = shutil.move(loc, 'results/' + id)
# update the document with test file location
results.update({'_id': result.inserted_id},
{"$set": {
"HREF": id
}},
upsert=False)
print('NEW_FILE_SAVED={}'.format(destination))
flash("File was corrected. Visit 'Test Results' to see scores",
category='success')
return render_template('sb-admin/pages/uploadtest.html',
tests=found,
admin_view=self)
self.header = "Correct Test"
return render_template('sb-admin/pages/uploadtest.html',
tests=found,
admin_view=self)
def test_correct(self):
response = {'title':'test'}
correct(self.test_chunks, response)
self.assertNotEqual(len(response['results']), 0)