def test_user_language_preference_fr_from_en_it_from_fr_it_entry(
admin_user, langstring_body, test_adminuser_webrequest,
user_language_preference_fr_mtfrom_en,
user_language_preference_it_mtfrom_fr, en_from_it_langstring_entry,
langstring_entry_values, it_langstring_entry, test_session):
"""
Body: fr, it-x-mtfrom-fr, en-x-mtfrom-fr
User Language Preference: {en-to-fr: explicit, fr-to-it}
Expect: fr-x-mtfrom-en
"""
from assembl.models.langstrings import LangStringEntry
entry = LangStringEntry(
locale_confirmed=False,
langstring=langstring_body,
locale='fr',
mt_trans_of=it_langstring_entry,
value=langstring_entry_values.get('body').get('fr'))
test_session.expire(langstring_body, ["entries"])
test_session.add(entry)
test_session.commit()
lang_prefs = admin_user.language_preference
best = langstring_body.best_lang(user_prefs=lang_prefs, allow_errors=True)
assert best.locale_code == 'it'
def fr_from_und_langstring_entry(request, test_session,
langstring_body, und_langstring_entry,
langstring_entry_values):
"""LangStringEntry fixture with und locale + FR from und locale"""
from assembl.models.langstrings import LangStringEntry
entry = LangStringEntry(
locale_confirmed=False,
langstring=langstring_body,
locale='fr',
mt_trans_of=und_langstring_entry,
value=langstring_entry_values.get('body').get('fr')
)
test_session.expire(langstring_body, ["entries"])
def fin():
test_session.delete(entry)
test_session.flush()
test_session.add(entry)
test_session.flush()
request.addfinalizer(fin)
return entry
def it_langstring_entry(request, test_session, it_locale, langstring_body,
langstring_entry_values):
"""LangStringEntry fixture with Italian locale"""
from assembl.models.langstrings import LangStringEntry
entry = LangStringEntry(
locale_confirmed=False,
langstring=langstring_body,
locale=it_locale,
value=langstring_entry_values.get('body').get('it'))
test_session.expire(langstring_body, ["entries"])
def fin():
test_session.delete(entry)
test_session.flush()
test_session.add(entry)
test_session.flush()
request.addfinalizer(fin)
return entry
def und_langstring_entry(request, test_session,
langstring_body, langstring_entry_values):
"""LangStringEntry fixture with undefined locale"""
from assembl.models.langstrings import LangStringEntry, LocaleLabel
entry = LangStringEntry(
locale_confirmed=False,
langstring=langstring_body,
locale=LocaleLabel.UNDEFINED,
value=langstring_entry_values.get('body').get('en')
)
test_session.expire(langstring_body, ["entries"])
def fin():
test_session.delete(entry)
test_session.flush()
test_session.add(entry)
test_session.flush()
request.addfinalizer(fin)
return entry
def fr_from_en_langstring_entry(request, test_session, fr_from_en_locale,
langstring_body, en_langstring_entry,
langstring_entry_values):
"""LangStringEntry fixture with EN locale + FR from EN locale"""
from assembl.models.langstrings import LangStringEntry
entry = LangStringEntry(
locale_confirmed=False,
langstring=langstring_body,
locale=fr_from_en_locale,
value=langstring_entry_values.get('body').get('fr'))
test_session.expire(langstring_body, ["entries"])
def fin():
print "Destroying fr_from_en_langstring_entry"
test_session.delete(entry)
test_session.flush()
test_session.add(entry)
test_session.flush()
request.addfinalizer(fin)
return entry
def translate_lse(
self, source_lse, target, retranslate=False, is_html=False,
constrain_locale_threshold=SECURE_IDENTIFICATION_LIMIT):
if not source_lse.value:
# don't translate empty strings
return source_lse
source_locale = source_lse.locale_code
if source_locale == Locale.NON_LINGUISTIC:
return source_lse
# TODO: Handle MULTILINGUAL
if (source_locale == Locale.UNDEFINED and
self.strlen_nourl(source_lse.value) < 5):
source_lse.identify_locale(Locale.NON_LINGUISTIC, None, True)
return source_lse
if (source_locale == Locale.UNDEFINED
and self.distinct_identify_step):
self.confirm_locale(
source_lse,
constrain_locale_threshold=constrain_locale_threshold)
# TODO: bail if identification failed
source_locale = source_lse.locale_code
# TODO: Handle script differences
if (Locale.compatible(source_locale, target.code)):
return source_lse
target_lse = None
is_new_lse = False
if (source_locale != Locale.UNDEFINED
or not self.distinct_identify_step
or self.has_fatal_error(source_lse)):
# We try to avoid ???-mt-from-und locales in the DB.
# This is only stored if both identification and translation
# failed to identify a language.
mt_target_name = self.get_mt_name(source_locale, target.code)
target_lse = source_lse.langstring.entries_as_dict.get(
Locale.get_id_of(mt_target_name), None)
if target_lse and not retranslate:
if self.has_fatal_error(target_lse):
return target_lse
if target_lse is None:
target_lse = LangStringEntry(
langstring_id=source_lse.langstring_id,
locale_id = Locale.UNDEFINED_LOCALEID,
value='')
is_new_lse = True
if self.canTranslate(source_locale, target.code):
try:
trans, lang = self.translate(
source_lse.value,
target.code,
is_html,
source=source_locale if source_locale != Locale.UNDEFINED
else None,
db=source_lse.db)
lang = self.asPosixLocale(lang)
# What if detected language is not a discussion language?
if source_locale == Locale.UNDEFINED:
if constrain_locale_threshold and (
self.strlen_nourl(source_lse.value) <
constrain_locale_threshold):
if (not lang) or not Locale.any_compatible(
lang, self.discussion.discussion_locales):
self.set_error(
source_lse,
LangStringStatus.IDENTIFIED_TO_UNKNOWN,
"Identified to "+lang)
return source_lse
source_lse.identify_locale(lang, dict(
service=self.__class__.__name__))
# This should never actually happen, because
# it would mean that the language id. was forgotten.
# Still, to be sure that all cases are covered.
mt_target_name = self.get_mt_name(lang, target.code)
other_target_lse = source_lse.langstring.entries_as_dict.get(
Locale.get_id_of(mt_target_name), None)
if other_target_lse:
target_lse = other_target_lse
is_new_lse = False
source_locale = source_lse.locale_code
if Locale.compatible(source_locale, target.code):
return source_lse
target_lse.value = trans
target_lse.error_count = 0
target_lse.error_code = None
target_lse.locale_identification_data_json = dict(
service=self.__class__.__name__)
if trans.strip() == source_lse.value.strip():
# TODO: Check modulo spaces in the middle
target_lse.error_count = 1
target_lse.error_code = \
LangStringStatus.IDENTICAL_TRANSLATION.value
except Exception as e:
print_exc()
self.set_error(target_lse, *self.decode_exception(e))
target_lse.value = None
else:
# Note: when retranslating, we may lose a valid translation.
if source_locale == Locale.UNDEFINED:
if not self.distinct_identify_step:
# At least do this much.
self.confirm_locale(source_lse)
source_locale = source_lse.locale_code
self.set_error(
target_lse, LangStringStatus.CANNOT_TRANSLATE,
"cannot translate")
target_lse.value = None
if (not target_lse.locale or
(source_locale != Locale.UNDEFINED
and Locale.extract_base_locale(
target_lse.locale_code) == Locale.UNDEFINED)):
mt_target_name = self.get_mt_name(
source_lse.locale_code, target.code)
target_lse.locale = Locale.get_or_create(
mt_target_name, source_lse.db)
if is_new_lse:
source_lse.db.add(target_lse)
return target_lse
def translate_lse(
self, source_lse, target, retranslate=False,
constrain_to_discussion_locales=SECURE_IDENTIFICATION_LIMIT):
if not source_lse.value:
# don't translate empty strings
return source_lse
source_locale = source_lse.locale_code
if source_locale == Locale.NON_LINGUISTIC:
return source_lse
# TODO: Handle MULTILINGUAL
if (source_locale == Locale.UNDEFINED and
self.strlen_nourl(source_lse.value) < 5):
source_lse.identify_locale(Locale.NON_LINGUISTIC, None, True)
return source_lse
if (source_locale == Locale.UNDEFINED
and self.distinct_identify_step):
self.confirm_locale(source_lse, constrain_to_discussion_locales)
# TODO: bail if identification failed
source_locale = source_lse.locale_code
# TODO: Handle script differences
if (Locale.compatible(source_locale, target.code)):
return source_lse
target_lse = None
is_new_lse = False
if (source_locale != Locale.UNDEFINED
or not self.distinct_identify_step
or self.has_fatal_error(source_lse)):
# We try to avoid ???-mt-from-und locales in the DB.
# This is only stored if both identification and translation
# failed to identify a language.
mt_target_name = self.get_mt_name(source_locale, target.code)
target_lse = source_lse.langstring.entries_as_dict.get(
Locale.get_id_of(mt_target_name), None)
if target_lse and not retranslate:
if self.has_fatal_error(target_lse):
return target_lse
if target_lse is None:
target_lse = LangStringEntry(
langstring_id=source_lse.langstring_id,
locale_id = Locale.UNDEFINED_LOCALEID,
value='')
is_new_lse = True
if self.canTranslate(source_locale, target.code):
try:
trans, lang = self.translate(
source_lse.value,
target.code,
source_locale if source_locale != Locale.UNDEFINED
else None,
source_lse.db)
lang = self.asPosixLocale(lang)
# What if detected language is not a discussion language?
if source_locale == Locale.UNDEFINED:
if constrain_to_discussion_locales and (
self.strlen_nourl(source_lse.value) <
constrain_to_discussion_locales):
if (not lang) or not Locale.any_compatible(
lang, self.discussion.discussion_locales):
self.set_error(
source_lse,
LangStringStatus.IDENTIFIED_TO_UNKNOWN,
"Identified to "+lang)
return source_lse
source_lse.identify_locale(lang, dict(
service=self.__class__.__name__))
# This should never actually happen, because
# it would mean that the language id. was forgotten.
# Still, to be sure that all cases are covered.
mt_target_name = self.get_mt_name(lang, target.code)
other_target_lse = source_lse.langstring.entries_as_dict.get(
Locale.get_id_of(mt_target_name), None)
if other_target_lse:
target_lse = other_target_lse
is_new_lse = False
source_locale = source_lse.locale_code
if Locale.compatible(source_locale, target.code):
return source_lse
target_lse.value = trans
target_lse.error_count = 0
target_lse.error_code = None
target_lse.locale_identification_data_json = dict(
service=self.__class__.__name__)
if trans.strip() == source_lse.value.strip():
# TODO: Check modulo spaces in the middle
target_lse.error_count = 1
target_lse.error_code = \
LangStringStatus.IDENTICAL_TRANSLATION.value
except Exception as e:
print_exc()
self.set_error(target_lse, *self.decode_exception(e))
target_lse.value = None
else:
# Note: when retranslating, we may lose a valid translation.
if source_locale == Locale.UNDEFINED:
if not self.distinct_identify_step:
# At least do this much.
self.confirm_locale(source_lse)
source_locale = source_lse.locale_code
self.set_error(
target_lse, LangStringStatus.CANNOT_TRANSLATE,
"cannot translate")
target_lse.value = None
if (not target_lse.locale or
(source_locale != Locale.UNDEFINED
and Locale.extract_base_locale(
target_lse.locale_code) == Locale.UNDEFINED)):
mt_target_name = self.get_mt_name(
source_lse.locale_code, target.code)
target_lse.locale = Locale.get_or_create(
mt_target_name, source_lse.db)
if is_new_lse:
source_lse.db.add(target_lse)
return target_lse
def translate_lse(self, source_lse, target, retranslate=False):
if source_lse.langstring_id == LangString.EMPTY_ID:
# don't translate the empty string
return source_lse
source_locale = source_lse.locale_code
if (source_locale == Locale.UNDEFINED
and self.distinct_identify_step):
self.confirm_locale(source_lse)
# TODO: bail if identification failed
source_locale = source_lse.locale_code
target_lse = None
is_new_lse = False
if (source_locale != Locale.UNDEFINED
or not self.distinct_identify_step
or self.has_fatal_error(source_lse)):
# We try to avoid ???-mt-from-und locales in the DB.
# This is only stored if both identification and translation
# failed to identify a language.
mt_target_name = self.get_mt_name(source_locale, target.code)
target_lse = source_lse.langstring.entries_as_dict.get(
mt_target_name, None)
if target_lse and not retranslate:
if (not target_lse.error_count
or self.has_fatal_error(target_lse)):
return target_lse
if target_lse is None:
target_lse = LangStringEntry(
langstring_id=source_lse.langstring_id)
is_new_lse = True
if self.canTranslate(source_locale, target.code):
try:
trans, lang = self.translate(
source_lse.value,
target.code,
source_locale if source_locale != Locale.UNDEFINED
else None,
source_lse.db)
lang = self.asPosixLocale(lang)
# What if detected language is not a discussion language?
if source_locale == Locale.UNDEFINED:
source_lse.identify_locale(lang, dict(
service=self.__class__.__name__))
source_locale = source_lse.locale_code
if Locale.len_common_parts(source_locale, target.code):
return source_lse
target_lse.value = trans
target_lse.error_count = 0
target_lse.error_code = None
target_lse.locale_identification_data_json = dict(
service=self.__class__.__name__)
except Exception as e:
print_exc()
self.set_error(target_lse, *self.decode_exception(e))
target_lse.value = ''
else:
# Note: when retranslating, we may lose a valid translation.
if source_locale == Locale.UNDEFINED:
if not self.distinct_identify_step:
# At least do this much.
self.confirm_locale(source_lse)
source_locale = source_lse.locale_code
self.set_error(
target_lse, LangStringStatus.CANNOT_TRANSLATE,
"cannot translate")
target_lse.value = ''
if (not target_lse.locale or
(source_locale != Locale.UNDEFINED
and Locale.extract_base_locale(
target_lse.locale_code) == Locale.UNDEFINED)):
mt_target_name = self.get_mt_name(
source_lse.locale_code, target.code)
target_lse.locale = Locale.get_or_create(
mt_target_name, source_lse.db)
if is_new_lse:
source_lse.db.add(target_lse)
return target_lse
