Beispiel #1
0
def index(translation_id):
    from_lang = 'en'
    to_lang = 'fr'

    form = TranslateForm()

    last_translations = current_user.get_latest_translations(
        10) if not current_user.is_anonymous else []

    # TODO: Error handling for incorrect input
    # Detect language
    if request.method == 'POST':
        text = form.text.data
        from_lang = form.lang_from.data
        to_lang = form.lang_to.data

        response = AWS_Translate.translate_text(Text=text,
                                                SourceLanguageCode=from_lang,
                                                TargetLanguageCode=to_lang)
        translated_text = response.get('TranslatedText')

        # Add translation to DB
        translation = Translation(
            lang_from=from_lang,
            lang_detected=from_lang,
            lang_to=to_lang,
            text_from=text,
            text_to=translated_text,
        )
        if not current_user.is_anonymous:
            translation.user_id = current_user.id
        db.session.add(translation)
        db.session.commit()
        return render_template('home/index.html',
                               title='Home',
                               form=form,
                               languages=Languages,
                               from_lang=from_lang,
                               to_lang=to_lang,
                               text=text,
                               translated_text=translated_text,
                               last_translations=last_translations)

    translation = Translation.query.filter_by(id=translation_id).first()

    form.lang_from.data = ((translation and translation.lang_from)
                           or from_lang)
    form.lang_to.data = ((translation and translation.lang_to) or to_lang)

    return render_template('home/index.html',
                           title='Home',
                           languages=Languages,
                           form=form,
                           from_lang=from_lang,
                           to_lang=to_lang,
                           last_translations=last_translations,
                           translation=translation)
def insert_quran_translation(verses, verse_translations, file, key, lang,
                             author, bio):
    logger.info("Adding Quran translation file %s", file)

    id = lang + "." + key
    qt = Translation(lang=lang, name=author, id=id)
    verse_translations.append(qt)

    index = 0
    with open(file, 'r', encoding='utf8') as qfile:
        for line in qfile.readlines():
            text = line.strip()
            if text and not text.startswith('#'):
                verses[index].translations[id] = [text]
                index = index + 1
Beispiel #3
0
def addTranslation():
    print('above')
    form = TransForm()
    print('below')
    form['csrf_token'].data = request.cookies['csrf_token']
    if form.validate_on_submit():
        trans = Translation(translation=form.data['translation'],
                            startIndex=form.data['startIndex'],
                            stopIndex=form.data['stopIndex'],
                            userId=form.data['userId'],
                            songId=form.data['songId'])
        db.session.add(trans)
        db.session.commit()
        return trans.to_dict()
    return {'errors': validation_errors_to_error_messages(form.errors)}, 401
Beispiel #4
0
    def post(self, request, **kwargs):
        if all(k in request.POST for k in ('lang1_words', 'lang2_words')):
            word_objects = []
            unallowed_characters = False
            languages = get_languages(self.kwargs)
            for i, lang in enumerate(languages, start=1):
                for w in request.POST.getlist('lang%d_words' % i):
                    if re.match(r'^[\w, \'!?\-".]+$', w):
                        # TODO: exclude underscore
                        word_object = Word.objects.get_or_create(language=lang, word=w)[0]
                        elementary_words_list = w.strip(',!?".').split(' ')
                        # if first letter is entire word (not elementary) is capital,
                        # lower it (in elementary)
                        if elementary_words_list[0][0].isupper():
                            elementary_words_list[0] = elementary_words_list[0].lower()
                        elementary_words_set = set(elementary_words_list)
                        elementary_words_set.discard('-')
                        # i don't strip "-" because of words like "well-known"
                        # and discard them only if there were between spaces
                        if len(elementary_words_set) > 1:
                            for ew in elementary_words_set:
                                word_object.elementary_words.add(Word.objects.get_or_create(language=lang, word=ew)[0])
                        word_objects += [word_object]
                    else:
                        unallowed_characters = True
                        messages.error(request, 'Unallowed characters in %s words' % lang.name)
                        break

            if not unallowed_characters:
                same_translations = Translation.objects.all()
                for w in word_objects:
                    same_translations = same_translations.filter(words=w)

                if not same_translations:
                    t = Translation()
                    t.save()
                    t.languages.add(*languages)
                    t.words.add(*word_objects)
                    messages.success(request, "Translation created.")
                else:
                    messages.warning(request, "Exactly the same translation already exists. Not added.")

        else:
            messages.error(request, "You need to enter words in both languages.")
        return self.get(request)
Beispiel #5
0
def clone_translation(doc_id, user_id):
    print("cloning translation (doc %s) from user %s" % (doc_id, user_id))

    tr_to_be_cloned = Translation.query.filter(Translation.user_id == user_id,
                                               Translation.doc_id == doc_id).first()

    if not tr_to_be_cloned:
        return make_404()

    teacher = current_app.get_current_user()
    teacher_tr = Translation.query.filter(Translation.user_id == teacher.id,
                                          Translation.doc_id == doc_id).first()
    if teacher_tr is None:
        teacher_tr = Translation(doc_id=doc_id, user_id=teacher.id, content=tr_to_be_cloned.content)
    else:
        # replace the teacher's tr content
        teacher_tr.content = tr_to_be_cloned.content
        # remove the old teacher's notes
        for note in teacher_tr.notes:
            db.session.delete(note)
        # teacher_tr.notes = []

    # clone notes
    for thn_to_be_cloned in tr_to_be_cloned.translation_has_note:
        note = Note(type_id=thn_to_be_cloned.note.type_id, user_id=teacher.id,
                    content=thn_to_be_cloned.note.content)
        db.session.add(note)
        db.session.flush()
        teacher_tr.translation_has_note.append(
            TranslationHasNote(ptr_start=thn_to_be_cloned.ptr_start,
                               ptr_end=thn_to_be_cloned.ptr_end,
                               note_id=note.id,
                               translation_id=teacher_tr.id),
        )

    db.session.add(teacher_tr)

    try:
        db.session.commit()
    except Exception as e:
        db.session.rollback()
        print(str(e))
        return make_400(str(e))

    return make_200()
Beispiel #6
0
def create():
    identity = get_jwt_identity()
    user = User.query.filter_by(username=identity['username']).first()
    g.current_user = user
    data = request.get_json() or {}
    dictionary = Dictionary.query.get(data['dictionary_id'])
    if dictionary in g.current_user.dictionaries:
        dictionary_id = dictionary.id
    else:
        abort(403)
    if 'name' not in data:
        return bad_request('word cannot be blank')
    if Word.query.filter_by(name=data['name']).first():
        word = Word.query.filter_by(name=data['name']).first()
    else:
        word = Word(data["name"], created_by=g.current_user.username)
        db.session.add(word)
        db.session.commit()
    user_word = UserWord(word.id, dictionary_id, g.current_user.id)
    db.session.add(user_word)
    try:
        db.session.commit()
    except IntegrityError:
        return bad_request('word already is in your dictionary')
    if 'description' in data:
        user_word.description = data['description']
        db.session.add(user_word)
        db.session.commit()
    if 'translations' in data:
        for t in data['translations']:
            trns = Translation(word.id, dictionary.id, t)
            db.session.add(trns)
            db.session.commit()
    response = jsonify(word.as_json())
    response.status_code = 201
    response.headers['Location'] = url_for('words.show', id=word.id)
    return response
def build_alhassanain_baabs(file) -> List[Chapter]:
    baabs: List[Chapter] = []
    logger.info("Adding Al-Kafi file %s", file)

    translation = Translation()
    translation.name = "HubeAli.com"
    translation.lang = Language.EN.value
    translation.id = HUBEALI_TRANSLATION_ID

    with open(file, 'r', encoding='utf8') as qfile:
        inner_html = qfile.read()
        sections = inner_html.split("<br clear=all>")
        for section in sections:
            section_soup = BeautifulSoup(section, 'html.parser')

            headings = section_soup.select(".Heading1Center")
            if not headings:
                continue

            # process "the book of" chapter
            baab_titles = extract_headings(headings)

            en_title = baab_titles[Language.EN.value]

            baab = None
            for existing_baab in baabs:
                if existing_baab.titles[Language.EN.value] == en_title:
                    baab = existing_baab

            if not baab:
                baab = Chapter()
                baab.part_type = PartType.Book
                baab.titles = baab_titles
                baab.chapters = []

                baabs.append(baab)

            # process chapters
            chapters = section_soup.select(".Heading2Center")
            chapters_len = len(chapters)
            for subchapter_index in range(math.ceil(chapters_len / 2)):
                subchapter_heading_index = subchapter_index * 2

                remaining_chapters = chapters[subchapter_heading_index:]
                if len(remaining_chapters) > 1:
                    remaining_chapters = remaining_chapters[:2]
                chapter_titles = extract_headings(remaining_chapters)

                chapter = Chapter()
                chapter.part_type = PartType.Chapter
                chapter.titles = chapter_titles
                chapter.verse_translations = [translation]
                chapter.verses = []

                baab.chapters.append(chapter)

                last_element = remaining_chapters[-1]
                last_element = last_element.next_sibling

                verse: Verse = None
                while (last_element is not None and
                       (isinstance(last_element, NavigableString) or
                        (is_tag(last_element)
                         and 'Heading2Center' not in last_element['class']))):
                    is_a_tag = is_tag(last_element)
                    if is_a_tag and 'libAr' in last_element['class']:

                        # push the last verse if its not the start of chapter
                        if verse != None:
                            chapter.verses.append(verse)

                        verse = Verse()
                        verse.part_type = PartType.Hadith
                        verse.translations = {}
                        verse.translations[HUBEALI_TRANSLATION_ID] = []

                        verse.text = [last_element.get_text(strip=True)]

                    if is_a_tag and 'libNormal' in last_element['class']:
                        verse.translations[HUBEALI_TRANSLATION_ID].append(
                            last_element.get_text(strip=True))

                    last_element = last_element.next_sibling

                if verse != None:
                    chapter.verses.append(verse)

    return baabs
    return baabs


HUBEALI_TRANSLATION_ID = "en.hubeali"
VOLUME_HEADING_PATTERN = re.compile("^AL-KAFI VOLUME")
TABLE_OF_CONTENTS_PATTERN = re.compile("^TABLE OF CONTENTS")
WHITESPACE_PATTERN = re.compile(r"^\s*$")
V8_HADITH_TITLE_PATTERN = re.compile(r"^H \d+")
V8_HADITH_BEGINNING_PATTERN = re.compile(r"^-? ?(1\d+)-?")
END_OF_HADITH_PATTERN = re.compile(r"<sup>\[\d+\]</sup>\s*$")
END_OF_HADITH_CLEANUP_PATTERN = re.compile(
    r'<a id="[^"]+"/?>(</a>)?<sup>\[\d+\]</sup>\s*$')

hubbeali_translation = Translation(name="HubeAli.com",
                                   lang=Language.EN.value,
                                   id=HUBEALI_TRANSLATION_ID)


def we_dont_care(heading):
    if heading is None:
        return True

    htext = heading.get_text(strip=True).upper()
    if VOLUME_HEADING_PATTERN.match(htext):
        return True

    return False


def table_of_contents(heading):
Beispiel #9
0
def update_collection(current_user):
  if not check_session():

    return jsonify({'message': 'You are not logged in!', 'isLoggedIn': False})

  data = request.get_json(silent=True, force=True)

  if data == None:
    data = {}

  request_action = request.args.get('action')
  request_collection_id = request.args.get('collectionId')
  request_translation_id = request.args.get('translationId')

  primaryLanguage = data.get('primaryLanguage')
  secondaryLanguage = data.get('secondaryLanguage')
  primaryPhrase = data.get('primaryPhrase')
  secondaryPhrase = data.get('secondaryPhrase')

  if request_collection_id == None:

    return jsonify({'message': '"collectionId" argument has to be instance of Int', 'isUpdated': False})

  target_collection =  None
  target_collection = Collection.query.filter(db.and_(Collection.id==int(request_collection_id), Collection.userId==current_user.id)).first()

  if not target_collection:

    return jsonify({'message': 'Not found collection named', 'isUpdated': False})


  if request_action == 'add':
    translation = Translation.query.filter(db.and_(Translation.primaryPhrase == primaryPhrase, Translation.secondaryPhrase==secondaryPhrase)).first()

    if not translation:
      translation = Translation(
        primaryPhrase=primaryPhrase,
        secondaryPhrase=secondaryPhrase,
        primaryLanguage=primaryLanguage,
        secondaryLanguage=secondaryLanguage
        )

    for translation_in_col in target_collection.translations:
      if translation.primaryPhrase == translation_in_col.primaryPhrase and  translation.secondaryPhrase == translation_in_col.secondaryPhrase:

        return jsonify({'message': 'You already have the same translation in this collection','isUpdated': False})
    
    db.session.add(translation)
    db.session.commit()
    target_collection.translations.append(translation)
    new_translation_status = TranslationStatus(collectionId=target_collection.id,translationId=translation.id)
    db.session.add(new_translation_status)
    db.session.commit()

    return jsonify({'isUpdated': True, 'newTranslation': {'id': translation.id, 'primaryPhrase': translation.primaryPhrase, 'secondaryPhrase': translation.secondaryPhrase, 'primaryLanguage': translation.primaryLanguage, 'secondaryLanguage': translation.secondaryLanguage,'createdAt': translation.createdAt, 'collectionId': new_translation_status.collectionId, 'isLearned': new_translation_status.isLearned, 'message': 'Translation has been added!' }})

  elif request_action == 'delete':
    if request.args.get('translationId') == None:

      return jsonify({'message': '"translationId" argument has to be instance of Int', 'isUpdated': False})
    
    target_translation = None

    for translation_in_coll in target_collection.translations:
      if translation_in_coll.id == int(request_translation_id):
        target_translation = translation_in_coll
        break

    if not target_translation:

      return jsonify({'message': 'Translation is not found', 'isUpdated': False})

    translation_status = TranslationStatus.query.filter(db.and_(TranslationStatus.collectionId==target_collection.id, TranslationStatus.translationId==target_translation.id)).first()

    if translation_status:
      db.session.delete(translation_status) 

    translationId = target_translation.id

    target_collection.translations.remove(target_translation)
    db.session.commit()

    return jsonify({'isUpdated': True, 'id': translationId})

  elif request_action == 'check':
 
    translation_status = TranslationStatus.query.filter(db.and_(TranslationStatus.collectionId==target_collection.id, TranslationStatus.translationId==request_translation_id)).first()

    if not translation_status:

      return jsonify({'message': 'Translation does not have any status', 'isUpdated': False})

    translation_status.isLearned = not translation_status.isLearned
    translation_status.updatedAt = datetime.datetime.now()
    db.session.commit()

    return jsonify({'translationIsLearned': translation_status.isLearned, 'isUpdated': True, 'id': translation_status.translationId})

  else:

    return jsonify({'message': 'Invalid request "action" argument', 'isUpdated': False})
from pprint import pprint

from bs4 import BeautifulSoup, NavigableString, Tag
from fastapi.encoders import jsonable_encoder

from app.lib_bs4 import get_contents, is_rtl_tag
from app.lib_db import insert_chapter, load_chapter, write_file
from app.lib_model import SEQUENCE_ERRORS, set_index
from app.models import Chapter, Language, PartType, Translation, Verse

logging.basicConfig(level=logging.INFO)
logger = logging.getLogger(__name__)

SARWAR_TRANSLATION_ID = "en.sarwar"
sarwar_translation = Translation(
    name="Shaykh Muhammad Sarwar (from Thaqalayn.net)",
    lang=Language.EN.value,
    id=SARWAR_TRANSLATION_ID)

# Created cumulative sum of verse numbers in volume 8 to adjust chapter index like this:
# from itertools import accumulate
# x = [len(x.verses) for x in book.chapters]
# print(list(accumulate(x)))
V8_HADITH_CUMSUM = [
    1, 3, 4, 6, 7, 15, 16, 20, 21, 22, 23, 26, 28, 29, 30, 32, 40, 51, 54, 56,
    62, 66, 68, 70, 91, 93, 94, 95, 98, 102, 104, 107, 125, 142, 143, 153, 192,
    193, 254, 273, 299, 313, 392, 457, 477, 532, 536, 550, 551, 584, 586, 597
]


def we_dont_care(html: str) -> bool:
    return '<body>' in html or '</body>' in html
Beispiel #11
0
        v_number_in_chapter = row['vocabulary.number_in_chapter']
        if v_id and v_hanzi and v_chapter_id and v_number_in_chapter:
            v_count += 1
            v = Vocabulary(id=v_id,
                           hanzi=v_hanzi,
                           chapter_id=v_chapter_id,
                           number_in_chapter=v_number_in_chapter)
            db.session.add(v)
        t_id = row['translation.id']
        t_vocabulary_id = row['translation.vocabulary_id']
        t_translation_en = row['translation.translation_en']
        t_gram_term_id = row['translation.gram_term_id']
        if t_id and t_vocabulary_id and t_translation_en and t_gram_term_id:
            t_count += 1
            t = Translation(id=t_id,
                            vocabulary_id=t_vocabulary_id,
                            translation_en=t_translation_en,
                            gram_term_id=t_gram_term_id)
            db.session.add(t)
        a_translation_id = row['asoc_translation_example.translation_id']
        a_example_id = row['asoc_translation_example.example_id']
        te_example = row['translationexample.example']
        if a_example_id and te_example:
            te_count += 1
            te = TranslationExample(id=a_example_id, example=te_example)
            db.session.add(te)
        if a_translation_id and a_example_id:
            s = asoc_translation_example.insert().values(
                translation_id=a_translation_id, example_id=a_example_id)
            db.session.execute(s)

with open('csv/tbl_chapter.csv') as csv_file:
Beispiel #12
0
def api_post_documents_translations(api_version, doc_id, user_id):
    """
    at least one of "content" or "notes" is required
    {
        "data":
            {
                "content" :  "My first translation"
                "notes": [
                        {
                           "content": "note1 content",
                           "ptr_start": 5,
                           "ptr_end": 7
                       }
                ]
            }
    }
    :param user_id:
    :param api_version:
    :param doc_id:
    :return:
    """

    #forbid = forbid_if_other_user(current_app, user_id)
    #if forbid:
    #    return forbid

    # teachers can still post notes in validated translation
    current_user = current_app.get_current_user()
    if not current_user.is_teacher and get_doc(doc_id).is_translation_validated:
        return make_403()

    forbid = is_closed(doc_id)
    if forbid:
        return forbid

    data = request.get_json()
    if "data" in data:
        data = data["data"]
        try:
            tr = None
            # case 1) "content" in data
            if "content" in data:
                error = check_no_XMLParserError(data["content"])
                if error:
                    raise Exception('Translation content is malformed: %s', str(error))
                tr = Translation(doc_id=doc_id, content=data["content"], user_id=user_id)
                db.session.add(tr)
                db.session.flush()
            # case 2) there's only "notes" in data
            if "notes" in data:
                tr = Translation.query.filter(Translation.doc_id == doc_id,
                                              Translation.user_id == user_id).first()
            if tr is None:
                return make_404()

            if "notes" in data:
                print("======= notes =======")
                for note in data["notes"]:
                    # 1) simply reuse notes which come with an id
                    note_id = note.get('id', None)
                    if note_id is not None:
                        reused_note = Note.query.filter(Note.id == note_id).first()
                        if reused_note is None:
                            return make_400(details="Wrong note id %s" % note_id)
                        db.session.add(reused_note)
                        db.session.flush()
                        thn = TranslationHasNote.query.filter(TranslationHasNote.note_id == reused_note.id,
                                                              TranslationHasNote.translation_id == tr.id).first()
                        # 1.a) the note is already present in the translation, so update its ptrs
                        if thn is not None:
                            raise Exception("Translation note already exists. Consider using PUT method")
                        else:
                            # 1.b) the note is not present on the translation side, so create it
                            thn = TranslationHasNote(translation_id=tr.id,
                                                     note_id=reused_note.id,
                                                     ptr_start=note["ptr_start"],
                                                     ptr_end=note["ptr_end"])
                        db.session.add(thn)
                        print("reuse:", thn.translation_id, thn.note_id)
                    else:
                        # 2) make new note
                        error = check_no_XMLParserError(note["content"])
                        if error:
                            raise Exception('Note content is malformed: %s', str(error))
                        new_note = Note(type_id=note.get("type_id", 0), user_id=user_id, content=note["content"])
                        db.session.add(new_note)
                        db.session.flush()
                        thn = TranslationHasNote(translation_id=tr.id,
                                                 note_id=new_note.id,
                                                 ptr_start=note["ptr_start"],
                                                 ptr_end=note["ptr_end"])
                        db.session.add(thn)
                        print("make:", thn.translation_id, thn.note_id)
                db.session.flush()
                print("thn:", [thn.note.id for thn in tr.translation_has_note])
                print("====================")
            db.session.add(tr)
            db.session.commit()
        except Exception as e:
            db.session.rollback()
            print("Error", str(e))
            return make_400(str(e))

        return make_200(data=tr.serialize_for_user(user_id))
    else:
        return make_400("no data")
Beispiel #13
0
				if verse != None:
					chapter.verses.append(verse)

	
	return baabs

HUBEALI_TRANSLATION_ID = "en.hubeali"
VOLUME_HEADING_PATTERN = re.compile("^AL-KAFI VOLUME")
TABLE_OF_CONTENTS_PATTERN = re.compile("^TABLE OF CONTENTS")
WHITESPACE_PATTERN = re.compile(r"^\s*$")
V8_HADITH_TITLE_PATTERN = re.compile(r"^H \d+")
V8_HADITH_BEGINNING_PATTERN = re.compile(r"^-? ?(1\d+)-?")
END_OF_HADITH_PATTERN = re.compile(r"<sup>\[\d+\]</sup>\s*$")
END_OF_HADITH_CLEANUP_PATTERN = re.compile(r'<a id="[^"]+"/?>(</a>)?<sup>\[\d+\]</sup>\s*$')

hubbeali_translation = Translation()
hubbeali_translation.name = "HubeAli.com"
hubbeali_translation.lang = Language.EN.value
hubbeali_translation.id = HUBEALI_TRANSLATION_ID

def we_dont_care(heading):
	if heading is None:
		return True
	
	htext = heading.get_text(strip=True).upper()
	if VOLUME_HEADING_PATTERN.match(htext):
		return True
	
	return False

def table_of_contents(heading):
Beispiel #14
0
    users.append(User(email=email, password=password, publicId=str(uuid.uuid4())))
    db.session.add(users[len(users)-1])

  if i <= 200:
    name = 'My collection'+str(i)

    collections.append(Collection(name=name, user=users[randint(0,len(users)-1)]))
    db.session.add(collections[len(collections)-1])

  primaryPhrase = 'Uwielbiam programować w Pythonie i JavaScript!'
  secondaryPhrase = 'I love programming in Python and JavaScript!'
  primaryLanguage = 'pl'
  secondaryLanguage = 'en'

  translations.append(Translation(primaryPhrase=primaryPhrase,secondaryPhrase=secondaryPhrase,primaryLanguage=primaryLanguage,secondaryLanguage=secondaryLanguage))
  db.session.add(translations[len(translations)-1])
  db.session.commit()

  half_len = (len(collections) // 2)

  if half_len > 20:
    half_len = (len(collections) // 3)
  elif half_len > 80:
    half_len = (len(collections) // 5)
  elif half_len > 140:
    half_len = (len(collections) // 8)
  elif half_len > 220:
    half_len = (len(collections) // 10)
  elif half_len > 300:
    half_len = (len(collections) // 16)