def index(translation_id):
from_lang = 'en'
to_lang = 'fr'
form = TranslateForm()
last_translations = current_user.get_latest_translations(
10) if not current_user.is_anonymous else []
# TODO: Error handling for incorrect input
# Detect language
if request.method == 'POST':
text = form.text.data
from_lang = form.lang_from.data
to_lang = form.lang_to.data
response = AWS_Translate.translate_text(Text=text,
SourceLanguageCode=from_lang,
TargetLanguageCode=to_lang)
translated_text = response.get('TranslatedText')
# Add translation to DB
translation = Translation(
lang_from=from_lang,
lang_detected=from_lang,
lang_to=to_lang,
text_from=text,
text_to=translated_text,
)
if not current_user.is_anonymous:
translation.user_id = current_user.id
db.session.add(translation)
db.session.commit()
return render_template('home/index.html',
title='Home',
form=form,
languages=Languages,
from_lang=from_lang,
to_lang=to_lang,
text=text,
translated_text=translated_text,
last_translations=last_translations)
translation = Translation.query.filter_by(id=translation_id).first()
form.lang_from.data = ((translation and translation.lang_from)
or from_lang)
form.lang_to.data = ((translation and translation.lang_to) or to_lang)
return render_template('home/index.html',
title='Home',
languages=Languages,
form=form,
from_lang=from_lang,
to_lang=to_lang,
last_translations=last_translations,
translation=translation)
def addTranslation():
print('above')
form = TransForm()
print('below')
form['csrf_token'].data = request.cookies['csrf_token']
if form.validate_on_submit():
trans = Translation(translation=form.data['translation'],
startIndex=form.data['startIndex'],
stopIndex=form.data['stopIndex'],
userId=form.data['userId'],
songId=form.data['songId'])
db.session.add(trans)
db.session.commit()
return trans.to_dict()
return {'errors': validation_errors_to_error_messages(form.errors)}, 401
def post(self, request, **kwargs):
if all(k in request.POST for k in ('lang1_words', 'lang2_words')):
word_objects = []
unallowed_characters = False
languages = get_languages(self.kwargs)
for i, lang in enumerate(languages, start=1):
for w in request.POST.getlist('lang%d_words' % i):
if re.match(r'^[\w, \'!?\-".]+$', w):
# TODO: exclude underscore
word_object = Word.objects.get_or_create(language=lang, word=w)[0]
elementary_words_list = w.strip(',!?".').split(' ')
# if first letter is entire word (not elementary) is capital,
# lower it (in elementary)
if elementary_words_list[0][0].isupper():
elementary_words_list[0] = elementary_words_list[0].lower()
elementary_words_set = set(elementary_words_list)
elementary_words_set.discard('-')
# i don't strip "-" because of words like "well-known"
# and discard them only if there were between spaces
if len(elementary_words_set) > 1:
for ew in elementary_words_set:
word_object.elementary_words.add(Word.objects.get_or_create(language=lang, word=ew)[0])
word_objects += [word_object]
else:
unallowed_characters = True
messages.error(request, 'Unallowed characters in %s words' % lang.name)
break
if not unallowed_characters:
same_translations = Translation.objects.all()
for w in word_objects:
same_translations = same_translations.filter(words=w)
if not same_translations:
t = Translation()
t.save()
t.languages.add(*languages)
t.words.add(*word_objects)
messages.success(request, "Translation created.")
else:
messages.warning(request, "Exactly the same translation already exists. Not added.")
else:
messages.error(request, "You need to enter words in both languages.")
return self.get(request)
def clone_translation(doc_id, user_id):
print("cloning translation (doc %s) from user %s" % (doc_id, user_id))
tr_to_be_cloned = Translation.query.filter(Translation.user_id == user_id,
Translation.doc_id == doc_id).first()
if not tr_to_be_cloned:
return make_404()
teacher = current_app.get_current_user()
teacher_tr = Translation.query.filter(Translation.user_id == teacher.id,
Translation.doc_id == doc_id).first()
if teacher_tr is None:
teacher_tr = Translation(doc_id=doc_id, user_id=teacher.id, content=tr_to_be_cloned.content)
else:
# replace the teacher's tr content
teacher_tr.content = tr_to_be_cloned.content
# remove the old teacher's notes
for note in teacher_tr.notes:
db.session.delete(note)
# teacher_tr.notes = []
# clone notes
for thn_to_be_cloned in tr_to_be_cloned.translation_has_note:
note = Note(type_id=thn_to_be_cloned.note.type_id, user_id=teacher.id,
content=thn_to_be_cloned.note.content)
db.session.add(note)
db.session.flush()
teacher_tr.translation_has_note.append(
TranslationHasNote(ptr_start=thn_to_be_cloned.ptr_start,
ptr_end=thn_to_be_cloned.ptr_end,
note_id=note.id,
translation_id=teacher_tr.id),
)
db.session.add(teacher_tr)
try:
db.session.commit()
except Exception as e:
db.session.rollback()
print(str(e))
return make_400(str(e))
return make_200()
def insert_quran_translation(verses, verse_translations, file, key, lang,
author, bio):
logger.info("Adding Quran translation file %s", file)
id = lang + "." + key
qt = Translation()
qt.lang = lang
qt.name = author
qt.id = id
verse_translations.append(qt)
index = 0
with open(file, 'r', encoding='utf8') as qfile:
for line in qfile.readlines():
text = line.strip()
if text and not text.startswith('#'):
verses[index].translations[id] = [text]
index = index + 1
def create():
identity = get_jwt_identity()
user = User.query.filter_by(username=identity['username']).first()
g.current_user = user
data = request.get_json() or {}
dictionary = Dictionary.query.get(data['dictionary_id'])
if dictionary in g.current_user.dictionaries:
dictionary_id = dictionary.id
else:
abort(403)
if 'name' not in data:
return bad_request('word cannot be blank')
if Word.query.filter_by(name=data['name']).first():
word = Word.query.filter_by(name=data['name']).first()
else:
word = Word(data["name"], created_by=g.current_user.username)
db.session.add(word)
db.session.commit()
user_word = UserWord(word.id, dictionary_id, g.current_user.id)
db.session.add(user_word)
try:
db.session.commit()
except IntegrityError:
return bad_request('word already is in your dictionary')
if 'description' in data:
user_word.description = data['description']
db.session.add(user_word)
db.session.commit()
if 'translations' in data:
for t in data['translations']:
trns = Translation(word.id, dictionary.id, t)
db.session.add(trns)
db.session.commit()
response = jsonify(word.as_json())
response.status_code = 201
response.headers['Location'] = url_for('words.show', id=word.id)
return response
def build_alhassanain_baabs(file) -> List[Chapter]:
baabs: List[Chapter] = []
logger.info("Adding Al-Kafi file %s", file)
translation = Translation()
translation.name = "HubeAli.com"
translation.lang = Language.EN.value
translation.id = HUBEALI_TRANSLATION_ID
with open(file, 'r', encoding='utf8') as qfile:
inner_html = qfile.read()
sections = inner_html.split("<br clear=all>")
for section in sections:
section_soup = BeautifulSoup(section, 'html.parser')
headings = section_soup.select(".Heading1Center")
if not headings:
continue
# process "the book of" chapter
baab_titles = extract_headings(headings)
en_title = baab_titles[Language.EN.value]
baab = None
for existing_baab in baabs:
if existing_baab.titles[Language.EN.value] == en_title:
baab = existing_baab
if not baab:
baab = Chapter()
baab.part_type = PartType.Book
baab.titles = baab_titles
baab.chapters = []
baabs.append(baab)
# process chapters
chapters = section_soup.select(".Heading2Center")
chapters_len = len(chapters)
for subchapter_index in range(math.ceil(chapters_len / 2)):
subchapter_heading_index = subchapter_index * 2
remaining_chapters = chapters[subchapter_heading_index:]
if len(remaining_chapters) > 1:
remaining_chapters = remaining_chapters[:2]
chapter_titles = extract_headings(remaining_chapters)
chapter = Chapter()
chapter.part_type = PartType.Chapter
chapter.titles = chapter_titles
chapter.verse_translations = [translation]
chapter.verses = []
baab.chapters.append(chapter)
last_element = remaining_chapters[-1]
last_element = last_element.next_sibling
verse: Verse = None
while (last_element is not None and
(isinstance(last_element, NavigableString) or
(is_tag(last_element)
and 'Heading2Center' not in last_element['class']))):
is_a_tag = is_tag(last_element)
if is_a_tag and 'libAr' in last_element['class']:
# push the last verse if its not the start of chapter
if verse != None:
chapter.verses.append(verse)
verse = Verse()
verse.part_type = PartType.Hadith
verse.translations = {}
verse.translations[HUBEALI_TRANSLATION_ID] = []
verse.text = [last_element.get_text(strip=True)]
if is_a_tag and 'libNormal' in last_element['class']:
verse.translations[HUBEALI_TRANSLATION_ID].append(
last_element.get_text(strip=True))
last_element = last_element.next_sibling
if verse != None:
chapter.verses.append(verse)
return baabs
return baabs
HUBEALI_TRANSLATION_ID = "en.hubeali"
VOLUME_HEADING_PATTERN = re.compile("^AL-KAFI VOLUME")
TABLE_OF_CONTENTS_PATTERN = re.compile("^TABLE OF CONTENTS")
WHITESPACE_PATTERN = re.compile(r"^\s*$")
V8_HADITH_TITLE_PATTERN = re.compile(r"^H \d+")
V8_HADITH_BEGINNING_PATTERN = re.compile(r"^-? ?(1\d+)-?")
END_OF_HADITH_PATTERN = re.compile(r"<sup>\[\d+\]</sup>\s*$")
END_OF_HADITH_CLEANUP_PATTERN = re.compile(
r'<a id="[^"]+"/?>(</a>)?<sup>\[\d+\]</sup>\s*$')
hubbeali_translation = Translation(name="HubeAli.com",
lang=Language.EN.value,
id=HUBEALI_TRANSLATION_ID)
def we_dont_care(heading):
if heading is None:
return True
htext = heading.get_text(strip=True).upper()
if VOLUME_HEADING_PATTERN.match(htext):
return True
return False
def table_of_contents(heading):
def update_collection(current_user):
if not check_session():
return jsonify({'message': 'You are not logged in!', 'isLoggedIn': False})
data = request.get_json(silent=True, force=True)
if data == None:
data = {}
request_action = request.args.get('action')
request_collection_id = request.args.get('collectionId')
request_translation_id = request.args.get('translationId')
primaryLanguage = data.get('primaryLanguage')
secondaryLanguage = data.get('secondaryLanguage')
primaryPhrase = data.get('primaryPhrase')
secondaryPhrase = data.get('secondaryPhrase')
if request_collection_id == None:
return jsonify({'message': '"collectionId" argument has to be instance of Int', 'isUpdated': False})
target_collection = None
target_collection = Collection.query.filter(db.and_(Collection.id==int(request_collection_id), Collection.userId==current_user.id)).first()
if not target_collection:
return jsonify({'message': 'Not found collection named', 'isUpdated': False})
if request_action == 'add':
translation = Translation.query.filter(db.and_(Translation.primaryPhrase == primaryPhrase, Translation.secondaryPhrase==secondaryPhrase)).first()
if not translation:
translation = Translation(
primaryPhrase=primaryPhrase,
secondaryPhrase=secondaryPhrase,
primaryLanguage=primaryLanguage,
secondaryLanguage=secondaryLanguage
)
for translation_in_col in target_collection.translations:
if translation.primaryPhrase == translation_in_col.primaryPhrase and translation.secondaryPhrase == translation_in_col.secondaryPhrase:
return jsonify({'message': 'You already have the same translation in this collection','isUpdated': False})
db.session.add(translation)
db.session.commit()
target_collection.translations.append(translation)
new_translation_status = TranslationStatus(collectionId=target_collection.id,translationId=translation.id)
db.session.add(new_translation_status)
db.session.commit()
return jsonify({'isUpdated': True, 'newTranslation': {'id': translation.id, 'primaryPhrase': translation.primaryPhrase, 'secondaryPhrase': translation.secondaryPhrase, 'primaryLanguage': translation.primaryLanguage, 'secondaryLanguage': translation.secondaryLanguage,'createdAt': translation.createdAt, 'collectionId': new_translation_status.collectionId, 'isLearned': new_translation_status.isLearned, 'message': 'Translation has been added!' }})
elif request_action == 'delete':
if request.args.get('translationId') == None:
return jsonify({'message': '"translationId" argument has to be instance of Int', 'isUpdated': False})
target_translation = None
for translation_in_coll in target_collection.translations:
if translation_in_coll.id == int(request_translation_id):
target_translation = translation_in_coll
break
if not target_translation:
return jsonify({'message': 'Translation is not found', 'isUpdated': False})
translation_status = TranslationStatus.query.filter(db.and_(TranslationStatus.collectionId==target_collection.id, TranslationStatus.translationId==target_translation.id)).first()
if translation_status:
db.session.delete(translation_status)
translationId = target_translation.id
target_collection.translations.remove(target_translation)
db.session.commit()
return jsonify({'isUpdated': True, 'id': translationId})
elif request_action == 'check':
translation_status = TranslationStatus.query.filter(db.and_(TranslationStatus.collectionId==target_collection.id, TranslationStatus.translationId==request_translation_id)).first()
if not translation_status:
return jsonify({'message': 'Translation does not have any status', 'isUpdated': False})
translation_status.isLearned = not translation_status.isLearned
translation_status.updatedAt = datetime.datetime.now()
db.session.commit()
return jsonify({'translationIsLearned': translation_status.isLearned, 'isUpdated': True, 'id': translation_status.translationId})
else:
return jsonify({'message': 'Invalid request "action" argument', 'isUpdated': False})
from pprint import pprint
from bs4 import BeautifulSoup, NavigableString, Tag
from fastapi.encoders import jsonable_encoder
from app.lib_bs4 import get_contents, is_rtl_tag
from app.lib_db import insert_chapter, load_chapter, write_file
from app.lib_model import SEQUENCE_ERRORS, set_index
from app.models import Chapter, Language, PartType, Translation, Verse
logging.basicConfig(level=logging.INFO)
logger = logging.getLogger(__name__)
SARWAR_TRANSLATION_ID = "en.sarwar"
sarwar_translation = Translation(
name="Shaykh Muhammad Sarwar (from Thaqalayn.net)",
lang=Language.EN.value,
id=SARWAR_TRANSLATION_ID)
# Created cumulative sum of verse numbers in volume 8 to adjust chapter index like this:
# from itertools import accumulate
# x = [len(x.verses) for x in book.chapters]
# print(list(accumulate(x)))
V8_HADITH_CUMSUM = [
1, 3, 4, 6, 7, 15, 16, 20, 21, 22, 23, 26, 28, 29, 30, 32, 40, 51, 54, 56,
62, 66, 68, 70, 91, 93, 94, 95, 98, 102, 104, 107, 125, 142, 143, 153, 192,
193, 254, 273, 299, 313, 392, 457, 477, 532, 536, 550, 551, 584, 586, 597
]
def we_dont_care(html: str) -> bool:
return '<body>' in html or '</body>' in html
v_number_in_chapter = row['vocabulary.number_in_chapter']
if v_id and v_hanzi and v_chapter_id and v_number_in_chapter:
v_count += 1
v = Vocabulary(id=v_id,
hanzi=v_hanzi,
chapter_id=v_chapter_id,
number_in_chapter=v_number_in_chapter)
db.session.add(v)
t_id = row['translation.id']
t_vocabulary_id = row['translation.vocabulary_id']
t_translation_en = row['translation.translation_en']
t_gram_term_id = row['translation.gram_term_id']
if t_id and t_vocabulary_id and t_translation_en and t_gram_term_id:
t_count += 1
t = Translation(id=t_id,
vocabulary_id=t_vocabulary_id,
translation_en=t_translation_en,
gram_term_id=t_gram_term_id)
db.session.add(t)
a_translation_id = row['asoc_translation_example.translation_id']
a_example_id = row['asoc_translation_example.example_id']
te_example = row['translationexample.example']
if a_example_id and te_example:
te_count += 1
te = TranslationExample(id=a_example_id, example=te_example)
db.session.add(te)
if a_translation_id and a_example_id:
s = asoc_translation_example.insert().values(
translation_id=a_translation_id, example_id=a_example_id)
db.session.execute(s)
with open('csv/tbl_chapter.csv') as csv_file:
def api_post_documents_translations(api_version, doc_id, user_id):
"""
at least one of "content" or "notes" is required
{
"data":
{
"content" : "My first translation"
"notes": [
{
"content": "note1 content",
"ptr_start": 5,
"ptr_end": 7
}
]
}
}
:param user_id:
:param api_version:
:param doc_id:
:return:
"""
#forbid = forbid_if_other_user(current_app, user_id)
#if forbid:
# return forbid
# teachers can still post notes in validated translation
current_user = current_app.get_current_user()
if not current_user.is_teacher and get_doc(doc_id).is_translation_validated:
return make_403()
forbid = is_closed(doc_id)
if forbid:
return forbid
data = request.get_json()
if "data" in data:
data = data["data"]
try:
tr = None
# case 1) "content" in data
if "content" in data:
error = check_no_XMLParserError(data["content"])
if error:
raise Exception('Translation content is malformed: %s', str(error))
tr = Translation(doc_id=doc_id, content=data["content"], user_id=user_id)
db.session.add(tr)
db.session.flush()
# case 2) there's only "notes" in data
if "notes" in data:
tr = Translation.query.filter(Translation.doc_id == doc_id,
Translation.user_id == user_id).first()
if tr is None:
return make_404()
if "notes" in data:
print("======= notes =======")
for note in data["notes"]:
# 1) simply reuse notes which come with an id
note_id = note.get('id', None)
if note_id is not None:
reused_note = Note.query.filter(Note.id == note_id).first()
if reused_note is None:
return make_400(details="Wrong note id %s" % note_id)
db.session.add(reused_note)
db.session.flush()
thn = TranslationHasNote.query.filter(TranslationHasNote.note_id == reused_note.id,
TranslationHasNote.translation_id == tr.id).first()
# 1.a) the note is already present in the translation, so update its ptrs
if thn is not None:
raise Exception("Translation note already exists. Consider using PUT method")
else:
# 1.b) the note is not present on the translation side, so create it
thn = TranslationHasNote(translation_id=tr.id,
note_id=reused_note.id,
ptr_start=note["ptr_start"],
ptr_end=note["ptr_end"])
db.session.add(thn)
print("reuse:", thn.translation_id, thn.note_id)
else:
# 2) make new note
error = check_no_XMLParserError(note["content"])
if error:
raise Exception('Note content is malformed: %s', str(error))
new_note = Note(type_id=note.get("type_id", 0), user_id=user_id, content=note["content"])
db.session.add(new_note)
db.session.flush()
thn = TranslationHasNote(translation_id=tr.id,
note_id=new_note.id,
ptr_start=note["ptr_start"],
ptr_end=note["ptr_end"])
db.session.add(thn)
print("make:", thn.translation_id, thn.note_id)
db.session.flush()
print("thn:", [thn.note.id for thn in tr.translation_has_note])
print("====================")
db.session.add(tr)
db.session.commit()
except Exception as e:
db.session.rollback()
print("Error", str(e))
return make_400(str(e))
return make_200(data=tr.serialize_for_user(user_id))
else:
return make_400("no data")
if verse != None:
chapter.verses.append(verse)
return baabs
HUBEALI_TRANSLATION_ID = "en.hubeali"
VOLUME_HEADING_PATTERN = re.compile("^AL-KAFI VOLUME")
TABLE_OF_CONTENTS_PATTERN = re.compile("^TABLE OF CONTENTS")
WHITESPACE_PATTERN = re.compile(r"^\s*$")
V8_HADITH_TITLE_PATTERN = re.compile(r"^H \d+")
V8_HADITH_BEGINNING_PATTERN = re.compile(r"^-? ?(1\d+)-?")
END_OF_HADITH_PATTERN = re.compile(r"<sup>\[\d+\]</sup>\s*$")
END_OF_HADITH_CLEANUP_PATTERN = re.compile(r'<a id="[^"]+"/?>(</a>)?<sup>\[\d+\]</sup>\s*$')
hubbeali_translation = Translation()
hubbeali_translation.name = "HubeAli.com"
hubbeali_translation.lang = Language.EN.value
hubbeali_translation.id = HUBEALI_TRANSLATION_ID
def we_dont_care(heading):
if heading is None:
return True
htext = heading.get_text(strip=True).upper()
if VOLUME_HEADING_PATTERN.match(htext):
return True
return False
def table_of_contents(heading):
users.append(User(email=email, password=password, publicId=str(uuid.uuid4())))
db.session.add(users[len(users)-1])
if i <= 200:
name = 'My collection'+str(i)
collections.append(Collection(name=name, user=users[randint(0,len(users)-1)]))
db.session.add(collections[len(collections)-1])
primaryPhrase = 'Uwielbiam programować w Pythonie i JavaScript!'
secondaryPhrase = 'I love programming in Python and JavaScript!'
primaryLanguage = 'pl'
secondaryLanguage = 'en'
translations.append(Translation(primaryPhrase=primaryPhrase,secondaryPhrase=secondaryPhrase,primaryLanguage=primaryLanguage,secondaryLanguage=secondaryLanguage))
db.session.add(translations[len(translations)-1])
db.session.commit()
half_len = (len(collections) // 2)
if half_len > 20:
half_len = (len(collections) // 3)
elif half_len > 80:
half_len = (len(collections) // 5)
elif half_len > 140:
half_len = (len(collections) // 8)
elif half_len > 220:
half_len = (len(collections) // 10)
elif half_len > 300:
half_len = (len(collections) // 16)