def gera_nova_lista(request):
f = open('list_words.txt', 'r')
string = ''
lang = Language.objects.get(id=1)
for i in f.readlines():
string = i.split()
sent = Sentence(
language=lang, original_text=string[1],
translate_text=string[3], level=1)
sent.save()
def gera_nova_lista(request):
f = open('list_words.txt', 'r')
string = ''
lang = Language.objects.get(id=1)
for i in f.readlines():
string = i.split()
sent = Sentence(language=lang,
original_text=string[1],
translate_text=string[3],
level=1)
sent.save()
def run(self):
"""Start listening to the server"""
logger.info(u'Started listening')
while not self._stop:
xml = self._readxml()
# Exit on invalid XML
if xml is None:
break
# Raw xml only
if not self.modelize:
logger.info(u'Raw xml: %s' % xml)
self.results.put(xml)
continue
# Model objects + raw xml as fallback
if xml.tag == 'RECOGOUT':
sentence = Sentence.from_shypo(xml.find('SHYPO'), self.encoding)
logger.info(u'Modelized recognition: %r' % sentence)
self.results.put(sentence)
else:
logger.info(u'Unmodelized xml: %s' % xml)
self.results.put(xml)
logger.info(u'Stopped listening')
def run(self):
"""Start listening to the server"""
logger.info(u"Started listening")
while not self._stop:
xml = self._readxml()
# Exit on invalid XML
if xml is None:
break
# Raw xml only
if not self.modelize:
logger.info(u"Raw xml: %s" % xml)
self.results.put(xml)
continue
# Model objects + raw xml as fallback
if xml.tag == "RECOGOUT":
sentence = Sentence.from_shypo(xml.find("SHYPO"), self.encoding)
logger.info(u"Modelized recognition: %r" % sentence)
self.results.put(sentence)
else:
logger.info(u"Unmodelized xml: %s" % xml)
self.results.put(xml)
logger.info(u"Stopped listening")
def register():
if request.method == "GET":
if current_user.lname_r == None or current_user.fname_r == None:
return redirect(url_for(".preset"))
clearCount = Keystroke.getCreatCount(current_user.id)
if clearCount >= 5:
return redirect(url_for(".index"))
sentence = Sentence.getByID(clearCount + 1)
if sentence.id == 4:
sentence.text_k = sentence.text_k.replace("[LAST]",
current_user.lname_k)
sentence.text_r = sentence.text_r.replace("[LAST]",
current_user.lname_r)
sentence.text_k = sentence.text_k.replace("[FIRST]",
current_user.fname_k)
sentence.text_r = sentence.text_r.replace("[FIRST]",
current_user.fname_r)
return render_template("register.html", sentence=sentence)
else:
clearCount = Keystroke.getCreatCount(current_user.id)
strokesList = request.json["strokesList"]
for index, strokes in enumerate(strokesList):
for stroke in strokes:
Keystroke.save(current_user.id, clearCount + 1, index,
stroke["key"], stroke["press"],
stroke["release"])
return "OK!"
def importSentences(data):
counter = 0
for sent in data:
for iso, langstring in sent.items():
Sentence(value=utils.force_unicode(langstring),
group=str(counter),
locale=iso).save()
counter += 1
def load_sentences(session, directory):
folder = os.path.join(directory, 'sentences')
for filename in os.listdir(folder):
if filename.endswith('.csv'):
with open(os.path.join(folder, filename), newline='') as csv_file:
reader = csv.DictReader(csv_file, delimiter=',', quotechar='"')
for sentence_csv in reader:
sentence = Sentence.create(session, **sentence_csv)
session.add(sentence)
session.commit()
def front_page():
"""on the front page, we assume that if the sentences cache exists, then the default sentence is at root.
if not, we should create it and store it in the cache."""
from models import Sentence
sentences = cache.get("sentence_list")
if sentences:
return render_template('index.html', sentence=sentences[0])
else:
current_sentence = Sentence(
"Once upon a time, there was a big bad wolf.")
cache.set("sentence_list", [current_sentence], timeout=None)
return render_template('index.html', sentence=current_sentence)
def add_sentence():
if request.is_json:
try:
content = request.get_json()
to_add = Sentence(content['text'], content['article_id'],
content['date'], content['time'])
db.session.add(to_add)
db.session.commit()
return flask.Response(status=201)
except:
return flask.Response(status=400)
else:
return flask.Response(status=400)
def add_sentence():
"""Internal functionality to add sentences to the current session cache.
Gets the text of the new sentence and position from the submitted form,
adds it to the current sentence using setattr and creates a new object that we can navigate
to later."""
from application.models import Sentence
if request.method == 'POST':
sentences = cache.get("sentence_list")
current_sentence = get_sentence_object(
request.form["current-sentence"], sentences)
added_sentence = request.form['sentence']
position = request.form['position']
setattr(current_sentence, position, request.form['sentence'])
sentences.append(Sentence(added_sentence))
cache.set("sentence_list", sentences, timeout=None)
else:
current_sentence = cache.get("sentence_list")[0]
return render_template('index.html', sentence=current_sentence)
def new():
text = Text()
session.add(text)
payload = json.loads(request.data)
for pl in payload:
sentence = Sentence(pl['sentence'], pl['language'], pl['order'], pl['translated'], text)
words = pl['words']
for w in words:
word = session.query(Word).filter_by(base_form=w['base_form'], language=w['language']).first()
if not word:
word = Word(w['base_form'], w['language'])
sentence.words.append(word)
seen_word = session.query(SeenWord).filter_by(seen_form=w['seen_form'], language=w['language']).first()
if not seen_word:
seen_word = SeenWord(w['seen_form'], w['language'], word)
session.add(seen_word)
session.add(sentence)
session.commit()
output = {'Message': 'Sentences saved in database'}
return json.dumps(output)
def process_worker(textId):
if not textId:
raise Exception(
"There is nothing to extract, please add text id to the message")
# Getting the result from the DB
result = Text.query.filter_by(id=textId).first()
if not result:
raise Exception("There is no such text stored in the database")
# Split text on sentences
sentences = split(result.text)
# Store sentences in the database
for sentenceFromTheList in sentences:
sent = Sentence(parent_id=result.id, sentence=sentenceFromTheList)
db.session.add(sent)
db.session.commit()
def get_exam(self):
exam = self._get_old_in_time_exam()
if exam:
return self.exam_to_dict(exam)
ids_query = select(el.explain.id for el in Exam)
while True:
try:
query = Explain.select(lambda x: x.id not in ids_query)
explain = self._get_query_random(query)
sentence = self._get_query_random(
Sentence.select(lambda x: x.explain == explain))
exam_query = Exam.select(lambda x: x.sentence == sentence)
if exam_query.count():
exam = exam_query.first()
exam.appear_cout += 1
else:
exam = Exam(word=explain.word,
explain=explain,
sentence=sentence)
return self.exam_to_dict(exam)
except Exception:
pass
def save(self):
s = Sentence()
s.sentence = self.cleaned_data['sentence']
s.ref_word_id = self.cleaned_data['ref_word_id']
s.save()
return s
def insSentenceControl():
if (request.method == 'POST'):
sentence_data = request.form['sentenceData'] # 프론트에서 받은 문장Data
standard = "" # 표준발음
sentence_dict = {"sentenceId": 0, "sentenceData": "", "standard": ""}
# 문장이 중복되면 "duplicate sentence" 반환
for sen in db_session.query(Sentence).filter(Sentence.sentenceData == sentence_data):
return jsonify(
message="duplicate sentence"
)
# 문장이 빈 문장, space, tab이면 "empty sentence" 반환
if not sentence_data or sentence_data.isspace():
return jsonify(
message="empty sentence"
)
# 부산대 표준발음 변환기
url = parse.urlparse \
("http://pronunciation.cs.pusan.ac.kr/pronunc2.asp?text1=안녕하세요&submit1=확인하기")
# url parse
qs = dict(parse.parse_qsl(url.query))
# sentence_data를 공백 기준으로(어절로) split 후 표준발음으로 변환. 변환되면 다시 합쳐서
standard_lst = [] # 어절로 나눈 표준발음 list
# sentence_data를 어절로 나누어서 sentence_data_split에 저장
sentence_data_split = sentence_data.split(" ")
for sen_data in sentence_data_split:
qs['text1'] = sen_data
url = url._replace(query=parse.urlencode(qs, encoding='euc-kr'))
new_url = parse.urlunparse(url)
# 표준발음 변환기에서 표준발음 가져오기
html = requests.get(new_url).content
bs = BeautifulSoup(html, 'html.parser')
# 표준 발음으로 변환할 수 없는 경우 "sentence cannot be converted" 반환
if bs.body == None:
return jsonify(
message="sentence cannot be converted"
)
# 표준발음으로 변환할 수 있는 경우
else:
search = bs.find_all('td', attrs={'class': 'td2'})
search_standard = search[2].text[:-1]
# 발음이 여러개일 때 앞에 것만 가져오기
search_standard = search_standard.split('/')
if sen_data == "이":
search_standard[0] = "이"
standard_lst.append(search_standard[0]) # 표준발음
# 표준발음 어절을 문장으로 합침
standard = " ".join(standard_lst)
# print(standard)
# 문장 내용, 표준 발음, userCheck를 sentence table에 insert
ins_sen = Sentence(sentence_data, standard, True)
db_session.add(ins_sen)
db_session.commit()
# 문장 분석 후 word 테이블에 insert
han = Hannanum()
sentence_analyze = han.pos(sentence_data)
# print(sentence_analyze)
# word 테이블에 wordData insert
for word_analyze in sentence_analyze:
if word_analyze[1] == 'N' or word_analyze[1] == 'P':
ins_word = Word(ins_sen.sentenceId, word_analyze[0], word_analyze[1])
db_session.add(ins_word)
db_session.commit()
# print(word_analyze[0])
for sen in db_session.query(Sentence).order_by(Sentence.sentenceId). \
filter(Sentence.sentenceData == sentence_data):
sentence_dict["sentenceId"] = sen.sentenceId
sentence_dict["sentenceData"] = sen.sentenceData
sentence_dict["standard"] = sen.standard
return jsonify(
message="insSentenceControl success",
newSentence=sentence_dict
)
