def identify(self,
text,
constrain_to_discussion_locales=SECURE_IDENTIFICATION_LIMIT):
"Try to identify locale of text. Boost if one of the expected locales."
if not text:
return Locale.UNDEFINED, {Locale.UNDEFINED: 1}
len_nourl = self.strlen_nourl(text)
if len_nourl < 5:
return Locale.NON_LINGUISTIC
expected_locales = set((Locale.extract_root_locale(l)
for l in self.discussion.discussion_locales))
language_data = detect_langs(text)
if constrain_to_discussion_locales and (
len_nourl < constrain_to_discussion_locales):
data = [(x.prob, x.lang) for x in language_data
if Locale.any_compatible(
Locale.extract_root_locale(x.lang), expected_locales)]
else:
# boost with discussion locales.
data = [(x.prob * (5 if Locale.Locale.extract_root_locale(x.lang)
in expected_locales else 1), x.lang)
for x in language_data]
data.sort(reverse=True)
top = data[0][1] if (data and (data[0][0] > 0.5)) else Locale.UNDEFINED
return top, {lang: prob for (prob, lang) in data}
def identify(
self, text,
constrain_to_discussion_locales=SECURE_IDENTIFICATION_LIMIT):
"Try to identify locale of text. Boost if one of the expected locales."
if not text:
return Locale.UNDEFINED, {Locale.UNDEFINED: 1}
len_nourl = self.strlen_nourl(text)
if len_nourl < 5:
return Locale.NON_LINGUISTIC
expected_locales = set((
Locale.extract_root_locale(l)
for l in self.discussion.discussion_locales))
language_data = detect_langs(text)
if constrain_to_discussion_locales and (
len_nourl < constrain_to_discussion_locales):
data = [(x.prob, x.lang)
for x in language_data
if Locale.any_compatible(
Locale.extract_root_locale(x.lang),
expected_locales)]
else:
# boost with discussion locales.
data = [
(x.prob * (
5 if Locale.Locale.extract_root_locale(x.lang)
in expected_locales else 1
), x.lang) for x in language_data]
data.sort(reverse=True)
top = data[0][1] if (data and (data[0][0] > 0.5)
) else Locale.UNDEFINED
return top, {lang: prob for (prob, lang) in data}
def translate(self, text, target, is_html=False, source=None, db=None):
if not text:
return text, Locale.NON_LINGUISTIC
if not source or source == Locale.UNDEFINED:
lang, data = self.identify(text)
source = Locale.get_or_create(source, db)
return text, lang
def process_locale(
locale_code, user, session, source_of_evidence):
locale_code = to_posix_string(locale_code)
# Updated: Now Locale is a model. Converting posix_string into its
# equivalent model. Creates it if it does not exist
locale = Locale.get_or_create(locale_code, session)
if source_of_evidence in LanguagePreferenceOrder.unique_prefs:
lang_pref_signatures = defaultdict(list)
for lp in user.language_preference:
lang_pref_signatures[lp.source_of_evidence].append(lp)
while len(lang_pref_signatures[source_of_evidence]) > 1:
# legacy multiple values
lang_pref_signatures[source_of_evidence][-1].delete()
if len(lang_pref_signatures[source_of_evidence]) == 1:
lang_pref_signatures[source_of_evidence][0].locale = locale
session.flush()
return
# else creation below
else:
lang_pref_signatures = {
(lp.locale_id, lp.source_of_evidence)
for lp in user.language_preference
}
if (locale.id, source_of_evidence) in lang_pref_signatures:
return
lang = UserLanguagePreference(
user=user, source_of_evidence=source_of_evidence, locale=locale)
session.add(lang)
session.flush()
def process_locale(locale_code, user, session, source_of_evidence):
locale_code = to_posix_string(locale_code)
# Updated: Now Locale is a model. Converting posix_string into its
# equivalent model. Creates it if it does not exist
locale = Locale.get_or_create(locale_code, session)
if source_of_evidence in LanguagePreferenceOrder.unique_prefs:
lang_pref_signatures = defaultdict(list)
for lp in user.language_preference:
lang_pref_signatures[lp.source_of_evidence].append(lp)
while len(lang_pref_signatures[source_of_evidence]) > 1:
# legacy multiple values
lp = lang_pref_signatures[source_of_evidence].pop()
lp.delete()
if len(lang_pref_signatures[source_of_evidence]) == 1:
lang_pref_signatures[source_of_evidence][0].locale = locale
session.flush()
return
# else creation below
else:
lang_pref_signatures = {(lp.locale_id, lp.source_of_evidence)
for lp in user.language_preference}
if (locale.id, source_of_evidence) in lang_pref_signatures:
return
lang = UserLanguagePreference(user=user,
source_of_evidence=source_of_evidence.value,
locale=locale)
session.add(lang)
session.flush()
def translate(self, text, target, source=None, db=None):
if not text:
return text, Locale.NON_LINGUISTIC
if not source or source == Locale.UNDEFINED:
lang, data = self.identify(text)
source = Locale.get_or_create(source, db)
return text, lang
def fr_from_en_locale(request, test_session, locale_cache, en_locale,
fr_locale):
"""French (fr) locale fixture, machine translated from English (en)"""
from assembl.models.langstrings import Locale
locale = Locale.create_mt_locale(en_locale, fr_locale, db=test_session)
# DO NOT DELETE core locales.
return locale
def undefined_locale(request, test_session, locale_cache):
"""undefined (und) locale fixture"""
from assembl.models.langstrings import Locale
locale = Locale.get_or_create(Locale.UNDEFINED, test_session)
# DO NOT DELETE core locales.
return locale
def non_linguistic_locale(request, test_session, locale_cache):
"""non-linguistic locale fixture"""
from assembl.models.langstrings import Locale
locale = Locale.get_or_create(Locale.NON_LINGUISTIC, test_session)
# DO NOT DELETE core locales.
return locale
def frontend_test_view(request):
context = get_default_context(request)
discussion = context["discussion"]
target_locale = Locale.get_or_create('en', discussion.db)
locale_labels = json.dumps(
DummyGoogleTranslationService.target_locale_labels_cls(target_locale))
context['translation_locale_names_json'] = locale_labels
context['translation_service_data_json'] = '{}'
context['preferences_json'] = json.dumps(dict(discussion.preferences))
return context
def frontend_test_view(request):
context = get_default_context(request)
discussion = context["discussion"]
target_locale = Locale.get_or_create('en', discussion.db)
locale_labels = json.dumps(
DummyGoogleTranslationService.target_locale_labels_cls(target_locale))
context['translation_locale_names_json'] = locale_labels
context['translation_service_data_json'] = '{}'
context['preferences_json'] = json.dumps(dict(discussion.preferences))
return context
def it_locale(request, test_session, locale_cache):
"""Italian (it) locale fixture"""
from assembl.models.langstrings import Locale
locale = Locale.get_or_create("it", test_session)
def fin():
test_session.delete(locale)
request.addfinalizer(fin)
return locale
def it_from_en_locale(request, test_session, locale_cache, en_locale, it_locale):
"""Italian (it) locale fixture, machine translated from English (en)"""
from assembl.models.langstrings import Locale
locale = Locale.create_mt_locale(en_locale, it_locale, db=test_session)
def fin():
test_session.delete(locale)
request.addfinalizer(fin)
return locale
def it_from_en_locale(request, test_session, locale_cache, en_locale, it_locale):
"""Italian (it) locale fixture, machine translated from English (en)"""
from assembl.models.langstrings import Locale
locale = Locale.create_mt_locale(en_locale, it_locale, db=test_session)
def fin():
test_session.delete(locale)
request.addfinalizer(fin)
return locale
def fr_from_it_locale(request, test_session, locale_cache, fr_locale, it_locale):
"""French (fr) locale fixture, machine translated from Italian (it)"""
from assembl.models.langstrings import Locale
locale = Locale.create_mt_locale(it_locale, fr_locale, db=test_session)
def fin():
test_session.delete(locale)
request.addfinalizer(fin)
return locale
def fr_from_it_locale(request, test_session, locale_cache, fr_locale, it_locale):
"""French (fr) locale fixture, machine translated from Italian (it)"""
from assembl.models.langstrings import Locale
locale = Locale.create_mt_locale(it_locale, fr_locale, db=test_session)
def fin():
test_session.delete(locale)
request.addfinalizer(fin)
return locale
def en_ca_locale(request, test_session, locale_cache):
"""Canadian English (en_CA) locale fixture"""
from assembl.models.langstrings import Locale
locale = Locale.get_or_create("en_CA", test_session)
def fin():
test_session.delete(locale)
request.addfinalizer(fin)
return locale
def undefined_locale(request, test_session, locale_cache):
"""undefined (und) locale fixture"""
from assembl.models.langstrings import Locale
locale = Locale.get_or_create(Locale.UNDEFINED, test_session)
def fin():
test_session.delete(locale)
request.addfinalizer(fin)
return locale
def de_from_tr_locale(request, test_session, locale_cache, de_locale, tr_locale):
"""German (de) locale fixture, machine translated from Turkish (tr)"""
from assembl.models.langstrings import Locale
locale = Locale.create_mt_locale(tr_locale, de_locale,
db=test_session)
def fin():
test_session.delete(locale)
request.addfinalizer(fin)
return locale
def undefined_locale(request, test_session, locale_cache):
"""undefined (und) locale fixture"""
from assembl.models.langstrings import Locale
locale = Locale.get_or_create(Locale.UNDEFINED, test_session)
def fin():
test_session.delete(locale)
request.addfinalizer(fin)
return locale
def de_from_tr_locale(request, test_session, locale_cache, de_locale,
tr_locale):
"""German (de) locale fixture, machine translated from Turkish (tr)"""
from assembl.models.langstrings import Locale
locale = Locale.create_mt_locale(tr_locale, de_locale, db=test_session)
def fin():
test_session.delete(locale)
request.addfinalizer(fin)
return locale
def non_linguistic_locale(request, test_session, locale_cache):
"""non-linguistic locale fixture"""
from assembl.models.langstrings import Locale
locale = Locale.get_or_create(Locale.NON_LINGUISTIC, test_session)
def fin():
test_session.delete(locale)
request.addfinalizer(fin)
return locale
def non_linguistic_locale(request, test_session, locale_cache):
"""non-linguistic locale fixture"""
from assembl.models.langstrings import Locale
locale = Locale.get_or_create(Locale.NON_LINGUISTIC, test_session)
def fin():
test_session.delete(locale)
request.addfinalizer(fin)
return locale
def en_from_tr_locale(request, test_session, tr_locale, en_locale):
"""English (en) locale fixture, machine translated from Turkish (tr)"""
from assembl.models.langstrings import Locale
locale = Locale.create_mt_locale(tr_locale, en_locale, db=test_session)
def fin():
test_session.delete(locale)
test_session.flush()
Locale.reset_cache()
request.addfinalizer(fin)
return locale
def fr_locale(request, test_session, locale_cache):
"""French (fr) locale fixture"""
from assembl.models.langstrings import Locale
delete_locale = Locale.locale_collection.get('fr', None) is None
locale = Locale.get_or_create("fr", test_session)
def fin():
test_session.delete(locale)
if delete_locale:
request.addfinalizer(fin)
return locale
def fr_locale(request, test_session, locale_cache):
"""French (fr) locale fixture"""
from assembl.models.langstrings import Locale
delete_locale = Locale.locale_collection.get('fr', None) is None
locale = Locale.get_or_create("fr", test_session)
def fin():
test_session.delete(locale)
if delete_locale:
request.addfinalizer(fin)
return locale
def fr_locale(request, test_session):
"""French (fr) locale fixture"""
from assembl.models.langstrings import Locale
locale = Locale.get_or_create("fr", test_session)
def fin():
test_session.delete(locale)
test_session.flush()
Locale.reset_cache()
request.addfinalizer(fin)
return locale
def it_from_fr_locale(request, test_session, fr_locale, it_locale):
"""Italian (it) locale fixture, machine translated from French (fr)"""
from assembl.models.langstrings import Locale
locale = Locale.create_mt_locale(fr_locale, it_locale, db=test_session)
def fin():
test_session.delete(locale)
test_session.flush()
Locale.reset_cache()
request.addfinalizer(fin)
return locale
def identify(self, text, constrain_to_discussion_locales=True):
"Try to identify locale of text. Boost if one of the expected locales."
if not text:
return Locale.UNDEFINED, {Locale.UNDEFINED: 1}
expected_locales = set((
Locale.extract_root_locale(l)
for l in self.discussion.discussion_locales))
language_data = detect_langs(text)
if constrain_to_discussion_locales:
data = [(x.prob, x.lang)
for x in language_data
if Locale.extract_root_locale(x.lang) in expected_locales]
else:
# boost with discussion locales.
data = [
(x.prob * (
5 if Locale.Locale.extract_root_locale(x.lang)
in expected_locales else 1
), x.lang) for x in language_data]
data.sort(reverse=True)
top = data[0][1] if (data and (data[0][0] > 0.5)
) else Locale.UNDEFINED
return top, {lang: prob for (prob, lang) in data}
def en_ca_locale(request, test_session, locale_cache):
"""Canadian English (en_CA) locale fixture"""
from assembl.models.langstrings import Locale
# Delete the locale at fixture tearDown only if it was created
# in this fixture.
# This is to keep locale names, that is created in Locale.populate_db
# from the db_default_data fixture.
delete_locale = Locale.locale_collection.get('en_CA', None) is None
locale = Locale.get_or_create("en_CA", test_session)
def fin():
test_session.delete(locale)
if delete_locale:
request.addfinalizer(fin)
return locale
def confirm_locale(self, langstring_entry, expected_locales=None):
try:
lang, data = self.identify(
langstring_entry.value, expected_locales)
data["service"] = self.__class__.__name__
changed = langstring_entry.identify_locale(lang, data)
if changed:
langstring_entry.db.expire(langstring_entry, ["locale"])
langstring_entry.db.expire(
langstring_entry.langstring, ["entries"])
if lang == Locale.UNDEFINED:
pass # say you can't identify
except Exception as e:
print_exc()
expected_locales = [
Locale.extract_root_locale(l)
for l in self.discussion.discussion_locales]
self.set_error(langstring_entry, *self.decode_exception(e, True))
def en_ca_locale(request, test_session, locale_cache):
"""Canadian English (en_CA) locale fixture"""
from assembl.models.langstrings import Locale
# Delete the locale at fixture tearDown only if it was created
# in this fixture.
# This is to keep locale names, that is created in Locale.populate_db
# from the db_default_data fixture.
delete_locale = Locale.locale_collection.get('en_CA', None) is None
locale = Locale.get_or_create("en_CA", test_session)
def fin():
test_session.delete(locale)
if delete_locale:
request.addfinalizer(fin)
return locale
def confirm_locale(
self,
langstring_entry,
constrain_to_discussion_locales=SECURE_IDENTIFICATION_LIMIT):
try:
lang, data = self.identify(langstring_entry.value,
constrain_to_discussion_locales)
data["service"] = self.__class__.__name__
changed = langstring_entry.identify_locale(lang, data)
if changed:
langstring_entry.db.expire(langstring_entry, ["locale"])
langstring_entry.db.expire(langstring_entry.langstring,
["entries"])
if lang == Locale.UNDEFINED:
pass # say you can't identify
except Exception as e:
print_exc()
expected_locales = [
Locale.extract_root_locale(l)
for l in self.discussion.discussion_locales
]
self.set_error(langstring_entry, *self.decode_exception(e, True))
def home_view(request):
user_id = authenticated_userid(request) or Everyone
context = get_default_context(request)
discussion = context["discussion"]
request.session["discussion"] = discussion.slug
canRead = user_has_permission(discussion.id, user_id, P_READ)
if not canRead and user_id == Everyone:
# User isn't logged-in and discussion isn't public:
# redirect to login page
# need to pass the route to go to *after* login as well
# With regards to a next_view, if explicitly stated, then
# that is the next view. If not stated, the referer takes
# precedence. In case of failure, login redirects to the
# discussion which is its context.
next_view = request.params.get('next_view', None)
if not next_view and discussion:
# If referred here from a post url, want to be able to
# send the user back. Usually, Assembl will send the user
# here to login on private discussions.
referrer = request.url
next_view = path_qs(referrer)
if next_view:
login_url = request.route_url("contextual_login",
discussion_slug=discussion.slug,
_query={"next_view": next_view})
else:
login_url = request.route_url(
'contextual_login', discussion_slug=discussion.slug)
return HTTPSeeOther(login_url)
elif not canRead:
# User is logged-in but doesn't have access to the discussion
# Would use render_to_response, except for the 401
from pyramid_jinja2 import IJinja2Environment
jinja_env = request.registry.queryUtility(
IJinja2Environment, name='.jinja2')
template = jinja_env.get_template('cannot_read_discussion.jinja2')
body = template.render(get_default_context(request))
return Response(body, 401)
# if the route asks for a post, get post content (because this is needed for meta tags)
route_name = request.matched_route.name
if route_name == "purl_posts":
post_id = FrontendUrls.getRequestedPostId(request)
if not post_id:
return HTTPSeeOther(request.route_url(
'home', discussion_slug=discussion.slug))
post = Post.get_instance(post_id)
if not post or post.discussion_id != discussion.id:
return HTTPSeeOther(request.route_url(
'home', discussion_slug=discussion.slug))
context['post'] = post
elif route_name == "purl_idea":
idea_id = FrontendUrls.getRequestedIdeaId(request)
if not idea_id:
return HTTPSeeOther(request.route_url(
'home', discussion_slug=discussion.slug))
idea = Idea.get_instance(idea_id)
if not idea or idea.discussion_id != discussion.id:
return HTTPSeeOther(request.route_url(
'home', discussion_slug=discussion.slug))
context['idea'] = idea
canAddExtract = user_has_permission(discussion.id, user_id, P_ADD_EXTRACT)
context['canAddExtract'] = canAddExtract
context['canDisplayTabs'] = True
preferences = discussion.preferences
if user_id != Everyone:
from assembl.models import UserPreferenceCollection
user = User.get(user_id)
preferences = UserPreferenceCollection(user_id, discussion)
# TODO: user may not exist. Case of session with BD change.
user.is_visiting_discussion(discussion.id)
session = Discussion.default_db
if '_LOCALE_' in request.cookies:
locale = request.cookies['_LOCALE_']
process_locale(locale, user, session,
LanguagePreferenceOrder.Cookie)
elif '_LOCALE_' in request.params:
locale = request.params['_LOCALE_']
process_locale(locale, user, session,
LanguagePreferenceOrder.Parameter)
else:
locale = locale_negotiator(request)
process_locale(locale, user, session,
LanguagePreferenceOrder.OS_Default)
else:
locale = request.localizer.locale_name
target_locale = Locale.get_or_create(
strip_country(locale), discussion.db)
translation_service_data = {}
try:
service = discussion.translation_service()
if service:
translation_service_data = service.serviceData()
except:
pass
context['translation_service_data_json'] = json.dumps(
translation_service_data)
locale_labels = json.dumps(
DummyGoogleTranslationService.target_locale_labels_cls(target_locale))
context['translation_locale_names_json'] = locale_labels
context['preferences_json'] = json.dumps(dict(preferences))
response = render_to_response('../../templates/index.jinja2', context,
request=request)
# Prevent caching the home, especially for proper login/logout
response.cache_control.max_age = 0
response.cache_control.prevent_auto = True
return response
def home_view(request):
"""The main view on a discussion"""
user_id = authenticated_userid(request) or Everyone
context = get_default_context(request)
discussion = context["discussion"]
canRead = user_has_permission(discussion.id, user_id, P_READ)
if not canRead and user_id == Everyone:
# User isn't logged-in and discussion isn't public:
# redirect to login page
# need to pass the route to go to *after* login as well
# With regards to a next_view, if explicitly stated, then
# that is the next view. If not stated, the referer takes
# precedence. In case of failure, login redirects to the
# discussion which is its context.
next_view = request.params.get('next', None)
if not next_view and discussion:
# If referred here from a post url, want to be able to
# send the user back. Usually, Assembl will send the user
# here to login on private discussions.
referrer = request.url
next_view = path_qs(referrer)
if discussion.preferences['authorization_server_backend']:
login_url = request.route_url(
"contextual_social_auth",
discussion_slug=discussion.slug,
backend=discussion.preferences['authorization_server_backend'],
_query={"next": next_view})
elif next_view:
login_url = request.route_url("contextual_login",
discussion_slug=discussion.slug,
_query={"next": next_view})
else:
login_url = request.route_url('contextual_login',
discussion_slug=discussion.slug)
return HTTPTemporaryRedirect(login_url)
elif not canRead:
# User is logged-in but doesn't have access to the discussion
# Would use render_to_response, except for the 401
from pyramid_jinja2 import IJinja2Environment
jinja_env = request.registry.queryUtility(IJinja2Environment,
name='.jinja2')
template = jinja_env.get_template('cannot_read_discussion.jinja2')
body = template.render(get_default_context(request))
return Response(body, 401)
# if the route asks for a post, get post content (because this is needed for meta tags)
route_name = request.matched_route.name
if route_name == "purl_posts":
post_id = FrontendUrls.getRequestedPostId(request)
if not post_id:
return HTTPSeeOther(
request.route_url('home', discussion_slug=discussion.slug))
post = Post.get_instance(post_id)
if not post or post.discussion_id != discussion.id:
return HTTPSeeOther(
request.route_url('home', discussion_slug=discussion.slug))
context['post'] = post
elif route_name == "purl_idea":
idea_id = FrontendUrls.getRequestedIdeaId(request)
if not idea_id:
return HTTPSeeOther(
request.route_url('home', discussion_slug=discussion.slug))
idea = Idea.get_instance(idea_id)
if not idea or idea.discussion_id != discussion.id:
return HTTPSeeOther(
request.route_url('home', discussion_slug=discussion.slug))
context['idea'] = idea
canAddExtract = user_has_permission(discussion.id, user_id, P_ADD_EXTRACT)
context['canAddExtract'] = canAddExtract
context['canDisplayTabs'] = True
preferences = discussion.preferences
if user_id != Everyone:
from assembl.models import UserPreferenceCollection
user = User.get(user_id)
preferences = UserPreferenceCollection(user_id, discussion)
# TODO: user may not exist. Case of session with BD change.
user.is_visiting_discussion(discussion.id)
session = Discussion.default_db
if '_LOCALE_' in request.cookies:
locale = request.cookies['_LOCALE_']
process_locale(locale, user, session,
LanguagePreferenceOrder.Cookie)
elif '_LOCALE_' in request.params:
locale = request.params['_LOCALE_']
process_locale(locale, user, session,
LanguagePreferenceOrder.Parameter)
else:
locale = locale_negotiator(request)
process_locale(locale, user, session,
LanguagePreferenceOrder.OS_Default)
else:
locale = request.localizer.locale_name
target_locale = Locale.get_or_create(strip_country(locale), discussion.db)
translation_service_data = {}
try:
service = discussion.translation_service()
if service:
translation_service_data = service.serviceData()
except:
pass
context['translation_service_data_json'] = json.dumps(
translation_service_data)
locale_labels = json.dumps(
DummyGoogleTranslationService.target_locale_labels_cls(target_locale))
context['translation_locale_names_json'] = locale_labels
context['preferences_json'] = json.dumps(dict(preferences))
response = render_to_response('../../templates/index.jinja2',
context,
request=request)
# Prevent caching the home, especially for proper login/logout
response.cache_control.max_age = 0
response.cache_control.prevent_auto = True
return response
def translate_lse(
self, source_lse, target, retranslate=False, is_html=False,
constrain_locale_threshold=SECURE_IDENTIFICATION_LIMIT):
if not source_lse.value:
# don't translate empty strings
return source_lse
source_locale = source_lse.locale_code
if source_locale == Locale.NON_LINGUISTIC:
return source_lse
# TODO: Handle MULTILINGUAL
if (source_locale == Locale.UNDEFINED and
self.strlen_nourl(source_lse.value) < 5):
source_lse.identify_locale(Locale.NON_LINGUISTIC, None, True)
return source_lse
if (source_locale == Locale.UNDEFINED
and self.distinct_identify_step):
self.confirm_locale(
source_lse,
constrain_locale_threshold=constrain_locale_threshold)
# TODO: bail if identification failed
source_locale = source_lse.locale_code
# TODO: Handle script differences
if (Locale.compatible(source_locale, target.code)):
return source_lse
target_lse = None
is_new_lse = False
if (source_locale != Locale.UNDEFINED
or not self.distinct_identify_step
or self.has_fatal_error(source_lse)):
# We try to avoid ???-mt-from-und locales in the DB.
# This is only stored if both identification and translation
# failed to identify a language.
mt_target_name = self.get_mt_name(source_locale, target.code)
target_lse = source_lse.langstring.entries_as_dict.get(
Locale.get_id_of(mt_target_name), None)
if target_lse and not retranslate:
if self.has_fatal_error(target_lse):
return target_lse
if target_lse is None:
target_lse = LangStringEntry(
langstring_id=source_lse.langstring_id,
locale_id = Locale.UNDEFINED_LOCALEID,
value='')
is_new_lse = True
if self.canTranslate(source_locale, target.code):
try:
trans, lang = self.translate(
source_lse.value,
target.code,
is_html,
source=source_locale if source_locale != Locale.UNDEFINED
else None,
db=source_lse.db)
lang = self.asPosixLocale(lang)
# What if detected language is not a discussion language?
if source_locale == Locale.UNDEFINED:
if constrain_locale_threshold and (
self.strlen_nourl(source_lse.value) <
constrain_locale_threshold):
if (not lang) or not Locale.any_compatible(
lang, self.discussion.discussion_locales):
self.set_error(
source_lse,
LangStringStatus.IDENTIFIED_TO_UNKNOWN,
"Identified to "+lang)
return source_lse
source_lse.identify_locale(lang, dict(
service=self.__class__.__name__))
# This should never actually happen, because
# it would mean that the language id. was forgotten.
# Still, to be sure that all cases are covered.
mt_target_name = self.get_mt_name(lang, target.code)
other_target_lse = source_lse.langstring.entries_as_dict.get(
Locale.get_id_of(mt_target_name), None)
if other_target_lse:
target_lse = other_target_lse
is_new_lse = False
source_locale = source_lse.locale_code
if Locale.compatible(source_locale, target.code):
return source_lse
target_lse.value = trans
target_lse.error_count = 0
target_lse.error_code = None
target_lse.locale_identification_data_json = dict(
service=self.__class__.__name__)
if trans.strip() == source_lse.value.strip():
# TODO: Check modulo spaces in the middle
target_lse.error_count = 1
target_lse.error_code = \
LangStringStatus.IDENTICAL_TRANSLATION.value
except Exception as e:
print_exc()
self.set_error(target_lse, *self.decode_exception(e))
target_lse.value = None
else:
# Note: when retranslating, we may lose a valid translation.
if source_locale == Locale.UNDEFINED:
if not self.distinct_identify_step:
# At least do this much.
self.confirm_locale(source_lse)
source_locale = source_lse.locale_code
self.set_error(
target_lse, LangStringStatus.CANNOT_TRANSLATE,
"cannot translate")
target_lse.value = None
if (not target_lse.locale or
(source_locale != Locale.UNDEFINED
and Locale.extract_base_locale(
target_lse.locale_code) == Locale.UNDEFINED)):
mt_target_name = self.get_mt_name(
source_lse.locale_code, target.code)
target_lse.locale = Locale.get_or_create(
mt_target_name, source_lse.db)
if is_new_lse:
source_lse.db.add(target_lse)
return target_lse
def process_locale(locale_code, user_id, current_prefs, session, order):
# Current is the current locale for the given order (ex. 'en' for
# cookie order)
current = [x for x in current_prefs if x.preferred_order == order]
user = session.query(User).filter_by(id=user_id).first()
posix_string = to_posix_string(locale_code)
posix_string = ensure_locale_has_country(posix_string)
# Updated: Now Locale is a model. Converting posix_string into its
# equivalent model. Creates it if it does not exist
locale = Locale.get_or_create(posix_string, session)
# Fresh slate for user, create a lang_pref
if not current_prefs:
lang = UserLanguagePreference(user=user, source_of_evidence=order,
preferred_order=order, locale=locale)
session.add(lang)
session.flush()
elif locale in {x.locale for x in current_prefs}:
if not current:
# current priority does not exist, but locale does
updated_pref = session.query(UserLanguagePreference).\
filter_by(user_id=user_id, locale=locale).\
first()
updated_pref.preferred_order = order
session.flush()
elif current[0].locale != locale:
# Current priority exists, but is not the desired locale
# Update current to desired local, remove current.
pref_to_remove = session.query(UserLanguagePreference).\
filter_by(user_id=user_id, preferred_order=order).first()
session.delete(pref_to_remove)
session.flush()
updated_pref = \
session.query(UserLanguagePreference).\
filter_by(user_id=user_id, locale=locale).\
first()
updated_pref.preferred_order = order
# updated_pref.source_of_evidence = order
session.flush()
else:
print "Current %s locale exists." % order
# non-empty list of current preferences, and current locale does not exist
elif current_prefs and not current:
lang = UserLanguagePreference(locale=locale,
preferred_order=order,
source_of_evidence=order,
user=user)
session.add(lang)
session.flush()
# Finally, locale not previously set, and there exists a previous
# priority locale
else:
pref = session.query(UserLanguagePreference).\
filter_by(user_id=user_id, preferred_order=order).first()
pref.locale = locale
session.add(pref)
session.flush()
def fin():
Locale.reset_cache()
test_session.flush()
def get_mt_name(self, source_name, target_name):
return Locale.create_mt_code(source_name, target_name)
def translate_lse(
self, source_lse, target, retranslate=False,
constrain_to_discussion_locales=SECURE_IDENTIFICATION_LIMIT):
if not source_lse.value:
# don't translate empty strings
return source_lse
source_locale = source_lse.locale_code
if source_locale == Locale.NON_LINGUISTIC:
return source_lse
# TODO: Handle MULTILINGUAL
if (source_locale == Locale.UNDEFINED and
self.strlen_nourl(source_lse.value) < 5):
source_lse.identify_locale(Locale.NON_LINGUISTIC, None, True)
return source_lse
if (source_locale == Locale.UNDEFINED
and self.distinct_identify_step):
self.confirm_locale(source_lse, constrain_to_discussion_locales)
# TODO: bail if identification failed
source_locale = source_lse.locale_code
# TODO: Handle script differences
if (Locale.compatible(source_locale, target.code)):
return source_lse
target_lse = None
is_new_lse = False
if (source_locale != Locale.UNDEFINED
or not self.distinct_identify_step
or self.has_fatal_error(source_lse)):
# We try to avoid ???-mt-from-und locales in the DB.
# This is only stored if both identification and translation
# failed to identify a language.
mt_target_name = self.get_mt_name(source_locale, target.code)
target_lse = source_lse.langstring.entries_as_dict.get(
Locale.get_id_of(mt_target_name), None)
if target_lse and not retranslate:
if self.has_fatal_error(target_lse):
return target_lse
if target_lse is None:
target_lse = LangStringEntry(
langstring_id=source_lse.langstring_id,
locale_id = Locale.UNDEFINED_LOCALEID,
value='')
is_new_lse = True
if self.canTranslate(source_locale, target.code):
try:
trans, lang = self.translate(
source_lse.value,
target.code,
source_locale if source_locale != Locale.UNDEFINED
else None,
source_lse.db)
lang = self.asPosixLocale(lang)
# What if detected language is not a discussion language?
if source_locale == Locale.UNDEFINED:
if constrain_to_discussion_locales and (
self.strlen_nourl(source_lse.value) <
constrain_to_discussion_locales):
if (not lang) or not Locale.any_compatible(
lang, self.discussion.discussion_locales):
self.set_error(
source_lse,
LangStringStatus.IDENTIFIED_TO_UNKNOWN,
"Identified to "+lang)
return source_lse
source_lse.identify_locale(lang, dict(
service=self.__class__.__name__))
# This should never actually happen, because
# it would mean that the language id. was forgotten.
# Still, to be sure that all cases are covered.
mt_target_name = self.get_mt_name(lang, target.code)
other_target_lse = source_lse.langstring.entries_as_dict.get(
Locale.get_id_of(mt_target_name), None)
if other_target_lse:
target_lse = other_target_lse
is_new_lse = False
source_locale = source_lse.locale_code
if Locale.compatible(source_locale, target.code):
return source_lse
target_lse.value = trans
target_lse.error_count = 0
target_lse.error_code = None
target_lse.locale_identification_data_json = dict(
service=self.__class__.__name__)
if trans.strip() == source_lse.value.strip():
# TODO: Check modulo spaces in the middle
target_lse.error_count = 1
target_lse.error_code = \
LangStringStatus.IDENTICAL_TRANSLATION.value
except Exception as e:
print_exc()
self.set_error(target_lse, *self.decode_exception(e))
target_lse.value = None
else:
# Note: when retranslating, we may lose a valid translation.
if source_locale == Locale.UNDEFINED:
if not self.distinct_identify_step:
# At least do this much.
self.confirm_locale(source_lse)
source_locale = source_lse.locale_code
self.set_error(
target_lse, LangStringStatus.CANNOT_TRANSLATE,
"cannot translate")
target_lse.value = None
if (not target_lse.locale or
(source_locale != Locale.UNDEFINED
and Locale.extract_base_locale(
target_lse.locale_code) == Locale.UNDEFINED)):
mt_target_name = self.get_mt_name(
source_lse.locale_code, target.code)
target_lse.locale = Locale.get_or_create(
mt_target_name, source_lse.db)
if is_new_lse:
source_lse.db.add(target_lse)
return target_lse
def get_mt_name(self, source_name, target_name):
return Locale.create_mt_code(source_name, target_name)
def translate_lse(self, source_lse, target, retranslate=False):
if source_lse.langstring_id == LangString.EMPTY_ID:
# don't translate the empty string
return source_lse
source_locale = source_lse.locale_code
if (source_locale == Locale.UNDEFINED
and self.distinct_identify_step):
self.confirm_locale(source_lse)
# TODO: bail if identification failed
source_locale = source_lse.locale_code
target_lse = None
is_new_lse = False
if (source_locale != Locale.UNDEFINED
or not self.distinct_identify_step
or self.has_fatal_error(source_lse)):
# We try to avoid ???-mt-from-und locales in the DB.
# This is only stored if both identification and translation
# failed to identify a language.
mt_target_name = self.get_mt_name(source_locale, target.code)
target_lse = source_lse.langstring.entries_as_dict.get(
mt_target_name, None)
if target_lse and not retranslate:
if (not target_lse.error_count
or self.has_fatal_error(target_lse)):
return target_lse
if target_lse is None:
target_lse = LangStringEntry(
langstring_id=source_lse.langstring_id)
is_new_lse = True
if self.canTranslate(source_locale, target.code):
try:
trans, lang = self.translate(
source_lse.value,
target.code,
source_locale if source_locale != Locale.UNDEFINED
else None,
source_lse.db)
lang = self.asPosixLocale(lang)
# What if detected language is not a discussion language?
if source_locale == Locale.UNDEFINED:
source_lse.identify_locale(lang, dict(
service=self.__class__.__name__))
source_locale = source_lse.locale_code
if Locale.len_common_parts(source_locale, target.code):
return source_lse
target_lse.value = trans
target_lse.error_count = 0
target_lse.error_code = None
target_lse.locale_identification_data_json = dict(
service=self.__class__.__name__)
except Exception as e:
print_exc()
self.set_error(target_lse, *self.decode_exception(e))
target_lse.value = ''
else:
# Note: when retranslating, we may lose a valid translation.
if source_locale == Locale.UNDEFINED:
if not self.distinct_identify_step:
# At least do this much.
self.confirm_locale(source_lse)
source_locale = source_lse.locale_code
self.set_error(
target_lse, LangStringStatus.CANNOT_TRANSLATE,
"cannot translate")
target_lse.value = ''
if (not target_lse.locale or
(source_locale != Locale.UNDEFINED
and Locale.extract_base_locale(
target_lse.locale_code) == Locale.UNDEFINED)):
mt_target_name = self.get_mt_name(
source_lse.locale_code, target.code)
target_lse.locale = Locale.get_or_create(
mt_target_name, source_lse.db)
if is_new_lse:
source_lse.db.add(target_lse)
return target_lse
def fin():
Locale.reset_cache()
test_session.flush()
def fin():
test_session.delete(locale)
test_session.flush()
Locale.reset_cache()