def upgrade():
### commands auto generated by Alembic - please adjust! ###
op.add_column('ActiveTranslationMessages', sa.Column('from_developer', sa.Boolean(), nullable=True))
op.create_index(u'ix_ActiveTranslationMessages_from_developer', 'ActiveTranslationMessages', ['from_developer'], unique=False)
### end Alembic commands ###
update_stmt = active_translation.update().where(active_translation.c.value == None).values(value = "")
with app.app_context():
db.session.execute(update_stmt)
active_translations = {
# bundle_id : {
# key : value,
# }
}
# 15256
with app.app_context():
for key, value, bundle_id in db.session.execute(sql.select([active_translation.c.key, active_translation.c.value, active_translation.c.bundle_id])):
if bundle_id not in active_translations:
active_translations[bundle_id] = {}
active_translations[bundle_id][key] = value
default_bundle_ids = {
# translation_url_id : bundle_id # pointing to English
}
for bundle_id, translation_url_id, target in db.session.execute(sql.select([bundle.c.id, bundle.c.translation_url_id, bundle.c.target], bundle.c.language == 'en_ALL')):
default_bundle_ids[translation_url_id] = bundle_id
active_translation_query = sql.select([active_translation.c.id, active_translation.c.taken_from_default, bundle.c.from_developer, bundle.c.translation_url_id, bundle.c.id, active_translation.c.key, active_translation.c.value], active_translation.c.bundle_id == bundle.c.id, use_labels = True)
for row in db.session.execute(active_translation_query):
active_msg_id = row[active_translation.c.id]
from_default = row[active_translation.c.taken_from_default]
bundle_from_developer = row[bundle.c.from_developer]
bundle_id = row[bundle.c.id]
translation_url_id = row[bundle.c.translation_url_id]
current_value = row[active_translation.c.value]
current_key = row[active_translation.c.key]
default_bundle_id = default_bundle_ids.get(translation_url_id)
default_value = active_translations.get(default_bundle_id, {}).get(current_key)
if bundle_id == default_bundle_id:
from_developer = True
else:
if bundle_from_developer and not from_default:
if default_value is not None and current_value != default_value and current_value:
from_developer = True
else:
from_developer = False
from_default = True
else:
from_developer = False
update_stmt = active_translation.update().where(active_translation.c.id == active_msg_id).values(from_developer = from_developer, taken_from_default = from_default)
db.session.execute(update_stmt)
db.session.commit()
def upgrade():
with app.app_context():
duplicated_suggestions = list(db.session.query(TranslationExternalSuggestion.c.engine, TranslationExternalSuggestion.c.language, TranslationExternalSuggestion.c.human_key_hash).group_by(TranslationExternalSuggestion.c.engine, TranslationExternalSuggestion.c.language, TranslationExternalSuggestion.c.human_key_hash).having(func.count(TranslationExternalSuggestion.c.id) > 1).all())
engines = [ engine for engine, language, human_key_hash in duplicated_suggestions ]
languages = [ language for engine, language, human_key_hash in duplicated_suggestions ]
human_key_hashes = [ human_key_hash for engine, language, human_key_hash in duplicated_suggestions ]
all_results = defaultdict(list)
for suggestion in db.session.query(TranslationExternalSuggestion).filter(TranslationExternalSuggestion.c.engine.in_(engines), TranslationExternalSuggestion.c.language.in_(languages), TranslationExternalSuggestion.c.human_key_hash.in_(human_key_hashes)).all():
all_results[suggestion.engine, suggestion.language, suggestion.human_key_hash].append(suggestion)
all_ids = []
for engine, language, human_key_hash in duplicated_suggestions:
for suggestion in all_results[engine, language, human_key_hash][1:]:
all_ids.append(suggestion.id)
delete_stmt = TranslationExternalSuggestion.delete(TranslationExternalSuggestion.c.id.in_(all_ids))
connection = op.get_bind()
connection.execute(delete_stmt)
# ### commands auto generated by Alembic - please adjust! ###
op.create_unique_constraint(None, 'TranslationExternalSuggestions', ['engine', 'human_key_hash', 'language'])
def upgrade():
# ### commands auto generated by Alembic - please adjust! ###
with app.app_context():
duplicated_active_messages = list(
db.session.query(
ActiveTranslationMessage.c.key,
ActiveTranslationMessage.c.bundle_id).group_by(
ActiveTranslationMessage.c.bundle_id,
ActiveTranslationMessage.c.key).having(
func.count(ActiveTranslationMessage.c.id) > 1).all())
keys = [k for k, b in duplicated_active_messages]
bundle_ids = [b for k, b in duplicated_active_messages]
all_results = defaultdict(list)
for atm in db.session.query(ActiveTranslationMessage).filter(
ActiveTranslationMessage.c.key.in_(keys),
ActiveTranslationMessage.c.bundle_id.in_(bundle_ids)).all():
all_results[atm.key, atm.bundle_id].append(atm)
all_ids = []
for key, bundle_id in duplicated_active_messages:
for atm in all_results[key, bundle_id][1:]:
all_ids.append(atm.id)
delete_stmt = ActiveTranslationMessage.delete(
ActiveTranslationMessage.c.id.in_(all_ids))
connection = op.get_bind()
connection.execute(delete_stmt)
op.create_unique_constraint(None, 'ActiveTranslationMessages',
['bundle_id', 'key'])
op.create_unique_constraint(None, 'RepositoryApp2languages',
['repository_app_id', 'language_id'])
def upgrade():
with app.app_context():
duplicated_bundles = list(db.session.query(TranslationBundle.c.translation_url_id, TranslationBundle.c.language, TranslationBundle.c.target).group_by(TranslationBundle.c.translation_url_id, TranslationBundle.c.language, TranslationBundle.c.target).having(func.count(TranslationBundle.c.id) > 1).all())
translation_url_ids = [ tr_id for tr_id, language, target in duplicated_bundles ]
languages = [ language for tr_id, language, target in duplicated_bundles ]
targets = [ target for tr_id, language, target in duplicated_bundles ]
all_results = defaultdict(list)
for bundle in db.session.query(TranslationBundle).filter(TranslationBundle.c.translation_url_id.in_(translation_url_ids), TranslationBundle.c.language.in_(languages), TranslationBundle.c.target.in_(targets)).all():
all_results[bundle.translation_url_id, bundle.language, bundle.target].append(bundle)
all_bundle_ids = []
for key in duplicated_bundles:
for bundle in all_results[key][1:]:
all_bundle_ids.append(bundle.id)
delete_msg_stmt = ActiveTranslationMessage.delete(ActiveTranslationMessage.c.bundle_id.in_(all_bundle_ids))
delete_hist_stmt = TranslationMessageHistory.delete(TranslationMessageHistory.c.bundle_id.in_(all_bundle_ids))
delete_bundle_stmt = TranslationBundle.delete(TranslationBundle.c.id.in_(all_bundle_ids))
connection = op.get_bind()
connection.execute(delete_msg_stmt)
connection.execute(delete_hist_stmt)
connection.execute(delete_bundle_stmt)
# ### commands auto generated by Alembic - please adjust! ###
op.create_unique_constraint(None, 'TranslationBundles', ['translation_url_id', 'language', 'target'])
def upgrade():
### commands auto generated by Alembic - please adjust! ###
op.add_column('TranslationFastCaches', sa.Column('app_metadata', sa.UnicodeText(), nullable=True))
### end Alembic commands ###
with app.app_context():
delete_stmt = translation_fast_caches.delete()
db.session.execute(delete_stmt)
db.session.commit()
def upgrade():
### commands auto generated by Alembic - please adjust! ###
op.add_column('TranslationSubscriptions', sa.Column('last_check', sa.DateTime(), nullable=True))
op.create_index(u'ix_TranslationSubscriptions_last_check', 'TranslationSubscriptions', ['last_check'], unique=False)
### end Alembic commands ###
with app.app_context():
update_stmt = translation_subscriptions.update().where(translation_subscriptions.c.last_check == None).values(last_check = datetime.datetime.utcnow() - datetime.timedelta(hours = 72))
db.session.execute(update_stmt)
db.session.commit()
def upgrade():
### commands auto generated by Alembic - please adjust! ###
op.add_column('TranslationFastCaches',
sa.Column('app_metadata', sa.UnicodeText(), nullable=True))
### end Alembic commands ###
with app.app_context():
delete_stmt = translation_fast_caches.delete()
db.session.execute(delete_stmt)
db.session.commit()
def upgrade():
### commands auto generated by Alembic - please adjust! ###
op.add_column('TranslationUrls', sa.Column('automatic', sa.Boolean(), nullable=True))
op.create_index(u'ix_TranslationUrls_automatic', 'TranslationUrls', ['automatic'], unique=False)
### end Alembic commands ###
with app.app_context():
update_stmt = translation_urls.update().values(automatic = True)
db.session.execute(update_stmt)
db.session.commit()
def upgrade():
### commands auto generated by Alembic - please adjust! ###
op.add_column('TranslationUrls',
sa.Column('automatic', sa.Boolean(), nullable=True))
op.create_index(u'ix_TranslationUrls_automatic',
'TranslationUrls', ['automatic'],
unique=False)
### end Alembic commands ###
with app.app_context():
update_stmt = translation_urls.update().values(automatic=True)
db.session.execute(update_stmt)
db.session.commit()
def sync(self, only_recent):
"""
Fully synchronizes the local database leading translations with
the MongoDB.
"""
if not flask_app.config["ACTIVATE_TRANSLATOR_MONGODB_PUSHES"]:
return
logger.info("[SYNC]: Starting Sync task")
start_time = datetime.utcnow()
if only_recent:
oldest = datetime.utcnow() - timedelta(minutes=30)
else:
oldest = datetime(1970, 1, 1)
with flask_app.app_context():
translation_bundles = [ {
'translation_url' : bundle.translation_url.url,
'language' : bundle.language,
'target' : bundle.target
} for bundle in db.session.query(TranslationBundle).filter(ActiveTranslationMessage.datetime >= oldest, ActiveTranslationMessage.bundle_id == TranslationBundle.id).group_by(TranslationBundle.id).options(joinedload("translation_url")).all() ]
if translation_bundles:
all_translation_url_ids = []
all_app_ids = []
for translation_bundle in translation_bundles:
responses = push(self = None, translation_url = translation_bundle['translation_url'], lang = translation_bundle['language'], target = translation_bundle['target'])
if responses is None:
logger.warn("Pushing translation for %s of %s returned None" % (translation_bundle['translation_url'], translation_bundle['language']))
continue
if responses == 'timeout':
logger.warn("Pushing translation for %s of %s returned a timeout error" % (translation_bundle['translation_url'], translation_bundle['language']))
break
for response in responses:
translation_url_id, app_ids = response
all_translation_url_ids.append(translation_url_id)
all_app_ids.extend(app_ids)
if not only_recent:
for mongo_bundles in all_mongo_bundles:
mongo_bundles.remove({"_id": {"$nin": all_app_ids}, "time": {"$lt": start_time}})
for mongo_translation_urls in all_mongo_translation_urls:
mongo_translation_urls.remove({"_id": {"$nin": all_translation_url_ids}, "time": {"$lt": start_time}})
logger.info("[SYNC]: Sync finished.")
def push(self, translation_url, lang, target):
if not flask_app.config["ACTIVATE_TRANSLATOR_MONGODB_PUSHES"]:
return
try:
logger.info("[PUSH] Pushing to %s@%s" % (lang, translation_url))
print("[PUSH] Pushing to %s@%s" % (lang, translation_url))
with flask_app.app_context():
translation_bundle = db.session.query(TranslationBundle).filter(TranslationBundle.translation_url_id == TranslationUrl.id, TranslationUrl.url == translation_url, TranslationBundle.language == lang, TranslationBundle.target == target).options(joinedload("translation_url")).first()
if translation_bundle is None:
return
payload = {}
max_date = datetime(1970, 1, 1)
for message in translation_bundle.active_messages:
payload[message.key] = message.value
if message.datetime > max_date:
max_date = message.datetime
data = json.dumps(payload)
lang_pack = lang + '_' + target
bundle_id = lang_pack + '::' + translation_url
bundle = { '_id' : bundle_id, 'url' : translation_url, 'bundle' : lang_pack, 'data' : data, 'time' : max_date }
try:
mongo_translation_urls.update({'_id' : bundle_id, 'time' : { '$lt' : max_date }}, bundle, upsert = True)
logger.info("[PUSH]: Updated translation URL bundle %s" % bundle_id)
print("[PUSH]: Updated translation URL bundle %s" % bundle_id)
except DuplicateKeyError:
print("[PUSH]: Ignoring push for translation URL bundle %s (newer date exists already)" % bundle_id)
app_bundle_ids = []
for application in translation_bundle.translation_url.apps:
app_bundle_id = lang_pack + '::' + application.url
app_bundle_ids.append(app_bundle_id)
bundle = { '_id' : app_bundle_id, 'spec' : application.url, 'bundle' : lang_pack, 'data' : data, 'time' : max_date }
try:
mongo_bundles.update({'_id' : app_bundle_id, 'time' : { '$lt' : max_date }}, bundle, upsert = True)
logger.info("[PUSH]: Updated application bundle %s" % app_bundle_id)
print("[PUSH]: Updated application bundle %s" % app_bundle_id)
except DuplicateKeyError:
print("[PUSH]: Ignoring push for application bundle %s (newer date exists already)" % app_bundle_id)
return bundle_id, app_bundle_ids
except Exception as exc:
logger.warn("[PUSH]: Exception occurred. Retrying soon.", exc_info = True)
print("[PUSH]: Exception occurred. Retrying soon.")
if self is not None:
raise self.retry(exc=exc, default_retry_delay=60, max_retries=None)
def upgrade():
with app.app_context():
duplicated_suggestions = list(
db.session.query(
TranslationExternalSuggestion.c.engine,
TranslationExternalSuggestion.c.language,
TranslationExternalSuggestion.c.human_key_hash).group_by(
TranslationExternalSuggestion.c.engine,
TranslationExternalSuggestion.c.language,
TranslationExternalSuggestion.c.human_key_hash).
having(func.count(TranslationExternalSuggestion.c.id) > 1).all())
engines = [
engine
for engine, language, human_key_hash in duplicated_suggestions
]
languages = [
language
for engine, language, human_key_hash in duplicated_suggestions
]
human_key_hashes = [
human_key_hash
for engine, language, human_key_hash in duplicated_suggestions
]
all_results = defaultdict(list)
for suggestion in db.session.query(
TranslationExternalSuggestion).filter(
TranslationExternalSuggestion.c.engine.in_(engines),
TranslationExternalSuggestion.c.language.in_(languages),
TranslationExternalSuggestion.c.human_key_hash.in_(
human_key_hashes)).all():
all_results[suggestion.engine, suggestion.language,
suggestion.human_key_hash].append(suggestion)
all_ids = []
for engine, language, human_key_hash in duplicated_suggestions:
for suggestion in all_results[engine, language,
human_key_hash][1:]:
all_ids.append(suggestion.id)
delete_stmt = TranslationExternalSuggestion.delete(
TranslationExternalSuggestion.c.id.in_(all_ids))
connection = op.get_bind()
connection.execute(delete_stmt)
# ### commands auto generated by Alembic - please adjust! ###
op.create_unique_constraint(None, 'TranslationExternalSuggestions',
['engine', 'human_key_hash', 'language'])
def upgrade():
### commands auto generated by Alembic - please adjust! ###
op.add_column('TranslationSubscriptions',
sa.Column('last_check', sa.DateTime(), nullable=True))
op.create_index(u'ix_TranslationSubscriptions_last_check',
'TranslationSubscriptions', ['last_check'],
unique=False)
### end Alembic commands ###
with app.app_context():
update_stmt = translation_subscriptions.update().where(
translation_subscriptions.c.last_check == None).values(
last_check=datetime.datetime.utcnow() -
datetime.timedelta(hours=72))
db.session.execute(update_stmt)
db.session.commit()
def sync(self, only_recent):
"""
Fully synchronizes the local database leading translations with
the MongoDB.
"""
if not flask_app.config["ACTIVATE_TRANSLATOR_MONGODB_PUSHES"]:
return
logger.info("[SYNC]: Starting Sync task")
start_time = datetime.utcnow()
if only_recent:
oldest = datetime.utcnow() - timedelta(minutes=30)
else:
oldest = datetime(1970, 1, 1)
with flask_app.app_context():
translation_bundles = [ {
'translation_url' : bundle.translation_url.url,
'language' : bundle.language,
'target' : bundle.target
} for bundle in db.session.query(TranslationBundle).filter(ActiveTranslationMessage.datetime >= oldest, ActiveTranslationMessage.bundle_id == TranslationBundle.id).group_by(TranslationBundle.id).options(joinedload("translation_url")).all() ]
if translation_bundles:
all_translation_url_ids = []
all_app_ids = []
for translation_bundle in translation_bundles:
responses = push(self = None, translation_url = translation_bundle['translation_url'], lang = translation_bundle['language'], target = translation_bundle['target'])
if responses is None:
logger.warn("Pushing translation for %s of %s returned None" % (translation_bundle['translation_url'], translation_bundle['language']))
continue
if responses == 'timeout':
logger.warn("Pushing translation for %s of %s returned a timeout error" % (translation_bundle['translation_url'], translation_bundle['language']))
break
for response in responses:
translation_url_id, app_ids = response
all_translation_url_ids.append(translation_url_id)
all_app_ids.extend(app_ids)
if not only_recent:
mongo_bundles.remove({"_id": {"$nin": all_app_ids}, "time": {"$lt": start_time}})
mongo_translation_urls.remove({"_id": {"$nin": all_translation_url_ids}, "time": {"$lt": start_time}})
logger.info("[SYNC]: Sync finished.")
def upgrade():
with app.app_context():
duplicated_bundles = list(
db.session.query(
TranslationBundle.c.translation_url_id,
TranslationBundle.c.language,
TranslationBundle.c.target).group_by(
TranslationBundle.c.translation_url_id,
TranslationBundle.c.language,
TranslationBundle.c.target).having(
func.count(TranslationBundle.c.id) > 1).all())
translation_url_ids = [
tr_id for tr_id, language, target in duplicated_bundles
]
languages = [
language for tr_id, language, target in duplicated_bundles
]
targets = [target for tr_id, language, target in duplicated_bundles]
all_results = defaultdict(list)
for bundle in db.session.query(TranslationBundle).filter(
TranslationBundle.c.translation_url_id.in_(
translation_url_ids),
TranslationBundle.c.language.in_(languages),
TranslationBundle.c.target.in_(targets)).all():
all_results[bundle.translation_url_id, bundle.language,
bundle.target].append(bundle)
all_bundle_ids = []
for key in duplicated_bundles:
for bundle in all_results[key][1:]:
all_bundle_ids.append(bundle.id)
delete_msg_stmt = ActiveTranslationMessage.delete(
ActiveTranslationMessage.c.bundle_id.in_(all_bundle_ids))
delete_hist_stmt = TranslationMessageHistory.delete(
TranslationMessageHistory.c.bundle_id.in_(all_bundle_ids))
delete_bundle_stmt = TranslationBundle.delete(
TranslationBundle.c.id.in_(all_bundle_ids))
connection = op.get_bind()
connection.execute(delete_msg_stmt)
connection.execute(delete_hist_stmt)
connection.execute(delete_bundle_stmt)
# ### commands auto generated by Alembic - please adjust! ###
op.create_unique_constraint(None, 'TranslationBundles',
['translation_url_id', 'language', 'target'])
def upgrade():
### commands auto generated by Alembic - please adjust! ###
op.add_column('TranslationMessageHistory', sa.Column('category', sa.Unicode(length=255), nullable=True))
op.add_column('TranslationMessageHistory', sa.Column('fmt', sa.Unicode(length=255), nullable=True))
op.add_column('TranslationMessageHistory', sa.Column('from_developer', sa.Boolean(), nullable=True))
op.add_column('TranslationMessageHistory', sa.Column('namespace', sa.Unicode(length=255), nullable=True))
op.add_column('TranslationMessageHistory', sa.Column('position', sa.Integer(), nullable=True))
op.add_column('TranslationMessageHistory', sa.Column('same_tool', sa.Boolean(), nullable=True))
op.add_column('TranslationMessageHistory', sa.Column('tool_id', sa.Unicode(length=255), nullable=True))
op.create_index(u'ix_TranslationMessageHistory_category', 'TranslationMessageHistory', ['category'], unique=False)
op.create_index(u'ix_TranslationMessageHistory_fmt', 'TranslationMessageHistory', ['fmt'], unique=False)
op.create_index(u'ix_TranslationMessageHistory_from_developer', 'TranslationMessageHistory', ['from_developer'], unique=False)
op.create_index(u'ix_TranslationMessageHistory_namespace', 'TranslationMessageHistory', ['namespace'], unique=False)
op.create_index(u'ix_TranslationMessageHistory_position', 'TranslationMessageHistory', ['position'], unique=False)
op.create_index(u'ix_TranslationMessageHistory_same_tool', 'TranslationMessageHistory', ['same_tool'], unique=False)
op.create_index(u'ix_TranslationMessageHistory_tool_id', 'TranslationMessageHistory', ['tool_id'], unique=False)
### end Alembic commands ###
with app.app_context():
# Create a list of fields such as active_translation.c.history_id, ...
FIELD_NAMES = 'category', 'fmt', 'from_developer', 'namespace', 'position', 'same_tool', 'tool_id'
fields = []
for field in ('history_id',) + FIELD_NAMES:
fields.append(getattr(active_translation.c, field))
for active_message in db.session.execute(sql.select(fields)):
parent_id = active_message[0]
# field_values = { 'category' : (category), 'fmt' : (fmt), ... }
field_values = dict(zip(FIELD_NAMES, active_message[1:]))
while parent_id is not None:
update_stmt = translation_history.update().where(translation_history.c.id == parent_id).values(**field_values)
db.session.execute(update_stmt)
parent_ids = db.session.execute(sql.select([ translation_history.c.parent_translation_id ], translation_history.c.id == parent_id))
parent_ids = list(parent_ids)
if len(parent_ids) == 0 or len(parent_ids[0]) == 0:
break
parent_id = parent_ids[0][0]
db.session.commit()
def upgrade():
### commands auto generated by Alembic - please adjust! ###
op.add_column('TranslationMessageHistory',
sa.Column('category', sa.Unicode(length=255), nullable=True))
op.add_column('TranslationMessageHistory',
sa.Column('fmt', sa.Unicode(length=255), nullable=True))
op.add_column('TranslationMessageHistory',
sa.Column('from_developer', sa.Boolean(), nullable=True))
op.add_column(
'TranslationMessageHistory',
sa.Column('namespace', sa.Unicode(length=255), nullable=True))
op.add_column('TranslationMessageHistory',
sa.Column('position', sa.Integer(), nullable=True))
op.add_column('TranslationMessageHistory',
sa.Column('same_tool', sa.Boolean(), nullable=True))
op.add_column('TranslationMessageHistory',
sa.Column('tool_id', sa.Unicode(length=255), nullable=True))
op.create_index(u'ix_TranslationMessageHistory_category',
'TranslationMessageHistory', ['category'],
unique=False)
op.create_index(u'ix_TranslationMessageHistory_fmt',
'TranslationMessageHistory', ['fmt'],
unique=False)
op.create_index(u'ix_TranslationMessageHistory_from_developer',
'TranslationMessageHistory', ['from_developer'],
unique=False)
op.create_index(u'ix_TranslationMessageHistory_namespace',
'TranslationMessageHistory', ['namespace'],
unique=False)
op.create_index(u'ix_TranslationMessageHistory_position',
'TranslationMessageHistory', ['position'],
unique=False)
op.create_index(u'ix_TranslationMessageHistory_same_tool',
'TranslationMessageHistory', ['same_tool'],
unique=False)
op.create_index(u'ix_TranslationMessageHistory_tool_id',
'TranslationMessageHistory', ['tool_id'],
unique=False)
### end Alembic commands ###
with app.app_context():
# Create a list of fields such as active_translation.c.history_id, ...
FIELD_NAMES = 'category', 'fmt', 'from_developer', 'namespace', 'position', 'same_tool', 'tool_id'
fields = []
for field in ('history_id', ) + FIELD_NAMES:
fields.append(getattr(active_translation.c, field))
for active_message in db.session.execute(sql.select(fields)):
parent_id = active_message[0]
# field_values = { 'category' : (category), 'fmt' : (fmt), ... }
field_values = dict(zip(FIELD_NAMES, active_message[1:]))
while parent_id is not None:
update_stmt = translation_history.update().where(
translation_history.c.id == parent_id).values(
**field_values)
db.session.execute(update_stmt)
parent_ids = db.session.execute(
sql.select([translation_history.c.parent_translation_id],
translation_history.c.id == parent_id))
parent_ids = list(parent_ids)
if len(parent_ids) == 0 or len(parent_ids[0]) == 0:
break
parent_id = parent_ids[0][0]
db.session.commit()
def push(self, translation_url, lang, target, recursive = False):
if not flask_app.config["ACTIVATE_TRANSLATOR_MONGODB_PUSHES"]:
return
if lang.startswith('en_'):
# Don't send any English text
return
previous = []
if not recursive:
if lang == 'zh_CN':
for record in push(self, translation_url, 'zh_ALL', target, recursive=True):
previous.append(record)
elif lang == 'zh_ALL':
for record in push(self, translation_url, 'zh_CN', target, recursive=True):
previous.append(record)
try:
logger.info("[PUSH] Pushing to %s@%s" % (lang, translation_url))
print("[PUSH] Pushing to %s@%s" % (lang, translation_url))
with flask_app.app_context():
translation_bundle = db.session.query(TranslationBundle).filter(TranslationBundle.translation_url_id == TranslationUrl.id, TranslationUrl.url == translation_url, TranslationBundle.language == lang, TranslationBundle.target == target).options(joinedload("translation_url")).first()
if translation_bundle is None:
if lang == 'zh_CN':
translation_bundle = db.session.query(TranslationBundle).filter(TranslationBundle.translation_url_id == TranslationUrl.id, TranslationUrl.url == translation_url, TranslationBundle.language == 'zh_ALL', TranslationBundle.target == target).options(joinedload("translation_url")).first()
elif lang == 'zh_ALL':
translation_bundle = db.session.query(TranslationBundle).filter(TranslationBundle.translation_url_id == TranslationUrl.id, TranslationUrl.url == translation_url, TranslationBundle.language == 'zh_CN', TranslationBundle.target == target).options(joinedload("translation_url")).first()
if translation_bundle is None:
return
payload = {}
max_date = datetime(1970, 1, 1)
for message in translation_bundle.active_messages:
payload[message.key] = message.value
if message.datetime > max_date:
max_date = message.datetime
data = json.dumps(payload)
lang_pack = lang + '_' + target
bundle_id = lang_pack + '::' + translation_url
bundle = { '_id' : bundle_id, 'url' : translation_url, 'bundle' : lang_pack, 'data' : data, 'time' : max_date }
for mongo_translation_urls in all_mongo_translation_urls:
try:
mongo_translation_urls.update({'_id' : bundle_id, 'time' : { '$lt' : max_date }}, bundle, upsert = True)
logger.info("[PUSH]: Updated translation URL bundle %s" % bundle_id)
print("[PUSH]: Updated translation URL bundle %s" % bundle_id)
except DuplicateKeyError:
print("[PUSH]: Ignoring push for translation URL bundle %s (newer date exists already)" % bundle_id)
app_bundle_ids = []
for application in translation_bundle.translation_url.apps:
app_bundle_id = lang_pack + '::' + application.url
app_bundle_ids.append(app_bundle_id)
bundle = { '_id' : app_bundle_id, 'spec' : application.url, 'bundle' : lang_pack, 'data' : data, 'time' : max_date }
for mongo_bundles in all_mongo_bundles:
try:
mongo_bundles.update({'_id' : app_bundle_id, 'time' : { '$lt' : max_date }}, bundle, upsert = True)
logger.info("[PUSH]: Updated application bundle %s" % app_bundle_id)
print("[PUSH]: Updated application bundle %s" % app_bundle_id)
except DuplicateKeyError:
print("[PUSH]: Ignoring push for application bundle %s (newer date exists already)" % app_bundle_id)
previous.append([bundle_id, app_bundle_ids])
return previous
except ServerSelectionTimeoutError as exc:
logger.warn("[PUSH]: Exception occurred due to server disconnect. NOT RETRYING.", exc_info = True)
return 'timeout'
except Exception as exc:
logger.warn("[PUSH]: Exception occurred. Retrying soon.", exc_info = True)
print("[PUSH]: Exception occurred. Retrying soon.")
if self is not None:
raise self.retry(exc=exc, default_retry_delay=60, max_retries=None)
# 'Buenas' : 1,
# }
}
remaining_texts = []
for translator in TRANSLATORS:
current_translations, current_remaining_texts = translator.existing_translations(texts, language, origin_language)
for remaining_text in current_remaining_texts:
if remaining_text not in translations:
remaining_texts.append(remaining_text)
for translation_key, translation_values in current_translations.iteritems():
if translation_key in remaining_texts:
remaining_texts.remove(translation_key)
if translation_key not in translations:
translations[translation_key] = {}
for value, weight in translation_values.iteritems():
if value not in translations[translation_key]:
translations[translation_key] = { value : 0 }
translations[translation_key][value] += weight
return translations, remaining_texts
if __name__ == '__main__':
from appcomposer import app
with app.app_context():
print existing_translations(["Hello", "Bye", "Good morning", "This was never stored"], 'es')
print translate_texts(["Hello", "Bye", "Good morning"], 'es')
def upgrade():
### commands auto generated by Alembic - please adjust! ###
op.add_column('ActiveTranslationMessages',
sa.Column('from_developer', sa.Boolean(), nullable=True))
op.create_index(u'ix_ActiveTranslationMessages_from_developer',
'ActiveTranslationMessages', ['from_developer'],
unique=False)
### end Alembic commands ###
update_stmt = active_translation.update().where(
active_translation.c.value == None).values(value="")
with app.app_context():
db.session.execute(update_stmt)
active_translations = {
# bundle_id : {
# key : value,
# }
}
# 15256
with app.app_context():
for key, value, bundle_id in db.session.execute(
sql.select([
active_translation.c.key, active_translation.c.value,
active_translation.c.bundle_id
])):
if bundle_id not in active_translations:
active_translations[bundle_id] = {}
active_translations[bundle_id][key] = value
default_bundle_ids = {
# translation_url_id : bundle_id # pointing to English
}
for bundle_id, translation_url_id, target in db.session.execute(
sql.select([
bundle.c.id, bundle.c.translation_url_id, bundle.c.target
], bundle.c.language == 'en_ALL')):
default_bundle_ids[translation_url_id] = bundle_id
active_translation_query = sql.select(
[
active_translation.c.id,
active_translation.c.taken_from_default,
bundle.c.from_developer, bundle.c.translation_url_id,
bundle.c.id, active_translation.c.key,
active_translation.c.value
],
active_translation.c.bundle_id == bundle.c.id,
use_labels=True)
for row in db.session.execute(active_translation_query):
active_msg_id = row[active_translation.c.id]
from_default = row[active_translation.c.taken_from_default]
bundle_from_developer = row[bundle.c.from_developer]
bundle_id = row[bundle.c.id]
translation_url_id = row[bundle.c.translation_url_id]
current_value = row[active_translation.c.value]
current_key = row[active_translation.c.key]
default_bundle_id = default_bundle_ids.get(translation_url_id)
default_value = active_translations.get(default_bundle_id,
{}).get(current_key)
if bundle_id == default_bundle_id:
from_developer = True
else:
if bundle_from_developer and not from_default:
if default_value is not None and current_value != default_value and current_value:
from_developer = True
else:
from_developer = False
from_default = True
else:
from_developer = False
update_stmt = active_translation.update().where(
active_translation.c.id == active_msg_id).values(
from_developer=from_developer,
taken_from_default=from_default)
db.session.execute(update_stmt)
db.session.commit()
def push(self, translation_url, lang, target):
if not flask_app.config["ACTIVATE_TRANSLATOR_MONGODB_PUSHES"]:
return
try:
logger.info("[PUSH] Pushing to %s@%s" % (lang, translation_url))
print("[PUSH] Pushing to %s@%s" % (lang, translation_url))
with flask_app.app_context():
translation_bundle = db.session.query(TranslationBundle).filter(
TranslationBundle.translation_url_id == TranslationUrl.id,
TranslationUrl.url == translation_url,
TranslationBundle.language == lang,
TranslationBundle.target == target).options(
joinedload("translation_url")).first()
if translation_bundle is None:
return
payload = {}
max_date = datetime(1970, 1, 1)
for message in translation_bundle.active_messages:
payload[message.key] = message.value
if message.datetime > max_date:
max_date = message.datetime
data = json.dumps(payload)
lang_pack = lang + '_' + target
bundle_id = lang_pack + '::' + translation_url
bundle = {
'_id': bundle_id,
'url': translation_url,
'bundle': lang_pack,
'data': data,
'time': max_date
}
try:
mongo_translation_urls.update(
{
'_id': bundle_id,
'time': {
'$lt': max_date
}
},
bundle,
upsert=True)
logger.info("[PUSH]: Updated translation URL bundle %s" %
bundle_id)
print("[PUSH]: Updated translation URL bundle %s" % bundle_id)
except DuplicateKeyError:
print(
"[PUSH]: Ignoring push for translation URL bundle %s (newer date exists already)"
% bundle_id)
app_bundle_ids = []
for application in translation_bundle.translation_url.apps:
app_bundle_id = lang_pack + '::' + application.url
app_bundle_ids.append(app_bundle_id)
bundle = {
'_id': app_bundle_id,
'spec': application.url,
'bundle': lang_pack,
'data': data,
'time': max_date
}
try:
mongo_bundles.update(
{
'_id': app_bundle_id,
'time': {
'$lt': max_date
}
},
bundle,
upsert=True)
logger.info("[PUSH]: Updated application bundle %s" %
app_bundle_id)
print("[PUSH]: Updated application bundle %s" %
app_bundle_id)
except DuplicateKeyError:
print(
"[PUSH]: Ignoring push for application bundle %s (newer date exists already)"
% app_bundle_id)
return bundle_id, app_bundle_ids
except Exception as exc:
logger.warn("[PUSH]: Exception occurred. Retrying soon.",
exc_info=True)
print("[PUSH]: Exception occurred. Retrying soon.")
if self is not None:
raise self.retry(exc=exc, default_retry_delay=60, max_retries=None)
remaining_texts = []
for translator in TRANSLATORS:
current_translations, current_remaining_texts = translator.existing_translations(
texts, language, origin_language)
for remaining_text in current_remaining_texts:
if remaining_text not in translations:
remaining_texts.append(remaining_text)
for translation_key, translation_values in current_translations.iteritems(
):
if translation_key in remaining_texts:
remaining_texts.remove(translation_key)
if translation_key not in translations:
translations[translation_key] = {}
for value, weight in translation_values.iteritems():
if value not in translations[translation_key]:
translations[translation_key] = {value: 0}
translations[translation_key][value] += weight
return translations, remaining_texts
if __name__ == '__main__':
from appcomposer import app
with app.app_context():
print existing_translations(
["Hello", "Bye", "Good morning", "This was never stored"], 'es')
print translate_texts(["Hello", "Bye", "Good morning"], 'es')
