def upgrade(): ### commands auto generated by Alembic - please adjust! ### op.add_column('ActiveTranslationMessages', sa.Column('from_developer', sa.Boolean(), nullable=True)) op.create_index(u'ix_ActiveTranslationMessages_from_developer', 'ActiveTranslationMessages', ['from_developer'], unique=False) ### end Alembic commands ### update_stmt = active_translation.update().where(active_translation.c.value == None).values(value = "") with app.app_context(): db.session.execute(update_stmt) active_translations = { # bundle_id : { # key : value, # } } # 15256 with app.app_context(): for key, value, bundle_id in db.session.execute(sql.select([active_translation.c.key, active_translation.c.value, active_translation.c.bundle_id])): if bundle_id not in active_translations: active_translations[bundle_id] = {} active_translations[bundle_id][key] = value default_bundle_ids = { # translation_url_id : bundle_id # pointing to English } for bundle_id, translation_url_id, target in db.session.execute(sql.select([bundle.c.id, bundle.c.translation_url_id, bundle.c.target], bundle.c.language == 'en_ALL')): default_bundle_ids[translation_url_id] = bundle_id active_translation_query = sql.select([active_translation.c.id, active_translation.c.taken_from_default, bundle.c.from_developer, bundle.c.translation_url_id, bundle.c.id, active_translation.c.key, active_translation.c.value], active_translation.c.bundle_id == bundle.c.id, use_labels = True) for row in db.session.execute(active_translation_query): active_msg_id = row[active_translation.c.id] from_default = row[active_translation.c.taken_from_default] bundle_from_developer = row[bundle.c.from_developer] bundle_id = row[bundle.c.id] translation_url_id = row[bundle.c.translation_url_id] current_value = row[active_translation.c.value] current_key = row[active_translation.c.key] default_bundle_id = default_bundle_ids.get(translation_url_id) default_value = active_translations.get(default_bundle_id, {}).get(current_key) if bundle_id == default_bundle_id: from_developer = True else: if bundle_from_developer and not from_default: if default_value is not None and current_value != default_value and current_value: from_developer = True else: from_developer = False from_default = True else: from_developer = False update_stmt = active_translation.update().where(active_translation.c.id == active_msg_id).values(from_developer = from_developer, taken_from_default = from_default) db.session.execute(update_stmt) db.session.commit()
def upgrade(): with app.app_context(): duplicated_suggestions = list(db.session.query(TranslationExternalSuggestion.c.engine, TranslationExternalSuggestion.c.language, TranslationExternalSuggestion.c.human_key_hash).group_by(TranslationExternalSuggestion.c.engine, TranslationExternalSuggestion.c.language, TranslationExternalSuggestion.c.human_key_hash).having(func.count(TranslationExternalSuggestion.c.id) > 1).all()) engines = [ engine for engine, language, human_key_hash in duplicated_suggestions ] languages = [ language for engine, language, human_key_hash in duplicated_suggestions ] human_key_hashes = [ human_key_hash for engine, language, human_key_hash in duplicated_suggestions ] all_results = defaultdict(list) for suggestion in db.session.query(TranslationExternalSuggestion).filter(TranslationExternalSuggestion.c.engine.in_(engines), TranslationExternalSuggestion.c.language.in_(languages), TranslationExternalSuggestion.c.human_key_hash.in_(human_key_hashes)).all(): all_results[suggestion.engine, suggestion.language, suggestion.human_key_hash].append(suggestion) all_ids = [] for engine, language, human_key_hash in duplicated_suggestions: for suggestion in all_results[engine, language, human_key_hash][1:]: all_ids.append(suggestion.id) delete_stmt = TranslationExternalSuggestion.delete(TranslationExternalSuggestion.c.id.in_(all_ids)) connection = op.get_bind() connection.execute(delete_stmt) # ### commands auto generated by Alembic - please adjust! ### op.create_unique_constraint(None, 'TranslationExternalSuggestions', ['engine', 'human_key_hash', 'language'])
def upgrade(): # ### commands auto generated by Alembic - please adjust! ### with app.app_context(): duplicated_active_messages = list( db.session.query( ActiveTranslationMessage.c.key, ActiveTranslationMessage.c.bundle_id).group_by( ActiveTranslationMessage.c.bundle_id, ActiveTranslationMessage.c.key).having( func.count(ActiveTranslationMessage.c.id) > 1).all()) keys = [k for k, b in duplicated_active_messages] bundle_ids = [b for k, b in duplicated_active_messages] all_results = defaultdict(list) for atm in db.session.query(ActiveTranslationMessage).filter( ActiveTranslationMessage.c.key.in_(keys), ActiveTranslationMessage.c.bundle_id.in_(bundle_ids)).all(): all_results[atm.key, atm.bundle_id].append(atm) all_ids = [] for key, bundle_id in duplicated_active_messages: for atm in all_results[key, bundle_id][1:]: all_ids.append(atm.id) delete_stmt = ActiveTranslationMessage.delete( ActiveTranslationMessage.c.id.in_(all_ids)) connection = op.get_bind() connection.execute(delete_stmt) op.create_unique_constraint(None, 'ActiveTranslationMessages', ['bundle_id', 'key']) op.create_unique_constraint(None, 'RepositoryApp2languages', ['repository_app_id', 'language_id'])
def upgrade(): with app.app_context(): duplicated_bundles = list(db.session.query(TranslationBundle.c.translation_url_id, TranslationBundle.c.language, TranslationBundle.c.target).group_by(TranslationBundle.c.translation_url_id, TranslationBundle.c.language, TranslationBundle.c.target).having(func.count(TranslationBundle.c.id) > 1).all()) translation_url_ids = [ tr_id for tr_id, language, target in duplicated_bundles ] languages = [ language for tr_id, language, target in duplicated_bundles ] targets = [ target for tr_id, language, target in duplicated_bundles ] all_results = defaultdict(list) for bundle in db.session.query(TranslationBundle).filter(TranslationBundle.c.translation_url_id.in_(translation_url_ids), TranslationBundle.c.language.in_(languages), TranslationBundle.c.target.in_(targets)).all(): all_results[bundle.translation_url_id, bundle.language, bundle.target].append(bundle) all_bundle_ids = [] for key in duplicated_bundles: for bundle in all_results[key][1:]: all_bundle_ids.append(bundle.id) delete_msg_stmt = ActiveTranslationMessage.delete(ActiveTranslationMessage.c.bundle_id.in_(all_bundle_ids)) delete_hist_stmt = TranslationMessageHistory.delete(TranslationMessageHistory.c.bundle_id.in_(all_bundle_ids)) delete_bundle_stmt = TranslationBundle.delete(TranslationBundle.c.id.in_(all_bundle_ids)) connection = op.get_bind() connection.execute(delete_msg_stmt) connection.execute(delete_hist_stmt) connection.execute(delete_bundle_stmt) # ### commands auto generated by Alembic - please adjust! ### op.create_unique_constraint(None, 'TranslationBundles', ['translation_url_id', 'language', 'target'])
def upgrade(): ### commands auto generated by Alembic - please adjust! ### op.add_column('TranslationFastCaches', sa.Column('app_metadata', sa.UnicodeText(), nullable=True)) ### end Alembic commands ### with app.app_context(): delete_stmt = translation_fast_caches.delete() db.session.execute(delete_stmt) db.session.commit()
def upgrade(): ### commands auto generated by Alembic - please adjust! ### op.add_column('TranslationSubscriptions', sa.Column('last_check', sa.DateTime(), nullable=True)) op.create_index(u'ix_TranslationSubscriptions_last_check', 'TranslationSubscriptions', ['last_check'], unique=False) ### end Alembic commands ### with app.app_context(): update_stmt = translation_subscriptions.update().where(translation_subscriptions.c.last_check == None).values(last_check = datetime.datetime.utcnow() - datetime.timedelta(hours = 72)) db.session.execute(update_stmt) db.session.commit()
def upgrade(): ### commands auto generated by Alembic - please adjust! ### op.add_column('TranslationUrls', sa.Column('automatic', sa.Boolean(), nullable=True)) op.create_index(u'ix_TranslationUrls_automatic', 'TranslationUrls', ['automatic'], unique=False) ### end Alembic commands ### with app.app_context(): update_stmt = translation_urls.update().values(automatic = True) db.session.execute(update_stmt) db.session.commit()
def upgrade(): ### commands auto generated by Alembic - please adjust! ### op.add_column('TranslationUrls', sa.Column('automatic', sa.Boolean(), nullable=True)) op.create_index(u'ix_TranslationUrls_automatic', 'TranslationUrls', ['automatic'], unique=False) ### end Alembic commands ### with app.app_context(): update_stmt = translation_urls.update().values(automatic=True) db.session.execute(update_stmt) db.session.commit()
def sync(self, only_recent): """ Fully synchronizes the local database leading translations with the MongoDB. """ if not flask_app.config["ACTIVATE_TRANSLATOR_MONGODB_PUSHES"]: return logger.info("[SYNC]: Starting Sync task") start_time = datetime.utcnow() if only_recent: oldest = datetime.utcnow() - timedelta(minutes=30) else: oldest = datetime(1970, 1, 1) with flask_app.app_context(): translation_bundles = [ { 'translation_url' : bundle.translation_url.url, 'language' : bundle.language, 'target' : bundle.target } for bundle in db.session.query(TranslationBundle).filter(ActiveTranslationMessage.datetime >= oldest, ActiveTranslationMessage.bundle_id == TranslationBundle.id).group_by(TranslationBundle.id).options(joinedload("translation_url")).all() ] if translation_bundles: all_translation_url_ids = [] all_app_ids = [] for translation_bundle in translation_bundles: responses = push(self = None, translation_url = translation_bundle['translation_url'], lang = translation_bundle['language'], target = translation_bundle['target']) if responses is None: logger.warn("Pushing translation for %s of %s returned None" % (translation_bundle['translation_url'], translation_bundle['language'])) continue if responses == 'timeout': logger.warn("Pushing translation for %s of %s returned a timeout error" % (translation_bundle['translation_url'], translation_bundle['language'])) break for response in responses: translation_url_id, app_ids = response all_translation_url_ids.append(translation_url_id) all_app_ids.extend(app_ids) if not only_recent: for mongo_bundles in all_mongo_bundles: mongo_bundles.remove({"_id": {"$nin": all_app_ids}, "time": {"$lt": start_time}}) for mongo_translation_urls in all_mongo_translation_urls: mongo_translation_urls.remove({"_id": {"$nin": all_translation_url_ids}, "time": {"$lt": start_time}}) logger.info("[SYNC]: Sync finished.")
def push(self, translation_url, lang, target): if not flask_app.config["ACTIVATE_TRANSLATOR_MONGODB_PUSHES"]: return try: logger.info("[PUSH] Pushing to %s@%s" % (lang, translation_url)) print("[PUSH] Pushing to %s@%s" % (lang, translation_url)) with flask_app.app_context(): translation_bundle = db.session.query(TranslationBundle).filter(TranslationBundle.translation_url_id == TranslationUrl.id, TranslationUrl.url == translation_url, TranslationBundle.language == lang, TranslationBundle.target == target).options(joinedload("translation_url")).first() if translation_bundle is None: return payload = {} max_date = datetime(1970, 1, 1) for message in translation_bundle.active_messages: payload[message.key] = message.value if message.datetime > max_date: max_date = message.datetime data = json.dumps(payload) lang_pack = lang + '_' + target bundle_id = lang_pack + '::' + translation_url bundle = { '_id' : bundle_id, 'url' : translation_url, 'bundle' : lang_pack, 'data' : data, 'time' : max_date } try: mongo_translation_urls.update({'_id' : bundle_id, 'time' : { '$lt' : max_date }}, bundle, upsert = True) logger.info("[PUSH]: Updated translation URL bundle %s" % bundle_id) print("[PUSH]: Updated translation URL bundle %s" % bundle_id) except DuplicateKeyError: print("[PUSH]: Ignoring push for translation URL bundle %s (newer date exists already)" % bundle_id) app_bundle_ids = [] for application in translation_bundle.translation_url.apps: app_bundle_id = lang_pack + '::' + application.url app_bundle_ids.append(app_bundle_id) bundle = { '_id' : app_bundle_id, 'spec' : application.url, 'bundle' : lang_pack, 'data' : data, 'time' : max_date } try: mongo_bundles.update({'_id' : app_bundle_id, 'time' : { '$lt' : max_date }}, bundle, upsert = True) logger.info("[PUSH]: Updated application bundle %s" % app_bundle_id) print("[PUSH]: Updated application bundle %s" % app_bundle_id) except DuplicateKeyError: print("[PUSH]: Ignoring push for application bundle %s (newer date exists already)" % app_bundle_id) return bundle_id, app_bundle_ids except Exception as exc: logger.warn("[PUSH]: Exception occurred. Retrying soon.", exc_info = True) print("[PUSH]: Exception occurred. Retrying soon.") if self is not None: raise self.retry(exc=exc, default_retry_delay=60, max_retries=None)
def upgrade(): with app.app_context(): duplicated_suggestions = list( db.session.query( TranslationExternalSuggestion.c.engine, TranslationExternalSuggestion.c.language, TranslationExternalSuggestion.c.human_key_hash).group_by( TranslationExternalSuggestion.c.engine, TranslationExternalSuggestion.c.language, TranslationExternalSuggestion.c.human_key_hash). having(func.count(TranslationExternalSuggestion.c.id) > 1).all()) engines = [ engine for engine, language, human_key_hash in duplicated_suggestions ] languages = [ language for engine, language, human_key_hash in duplicated_suggestions ] human_key_hashes = [ human_key_hash for engine, language, human_key_hash in duplicated_suggestions ] all_results = defaultdict(list) for suggestion in db.session.query( TranslationExternalSuggestion).filter( TranslationExternalSuggestion.c.engine.in_(engines), TranslationExternalSuggestion.c.language.in_(languages), TranslationExternalSuggestion.c.human_key_hash.in_( human_key_hashes)).all(): all_results[suggestion.engine, suggestion.language, suggestion.human_key_hash].append(suggestion) all_ids = [] for engine, language, human_key_hash in duplicated_suggestions: for suggestion in all_results[engine, language, human_key_hash][1:]: all_ids.append(suggestion.id) delete_stmt = TranslationExternalSuggestion.delete( TranslationExternalSuggestion.c.id.in_(all_ids)) connection = op.get_bind() connection.execute(delete_stmt) # ### commands auto generated by Alembic - please adjust! ### op.create_unique_constraint(None, 'TranslationExternalSuggestions', ['engine', 'human_key_hash', 'language'])
def upgrade(): ### commands auto generated by Alembic - please adjust! ### op.add_column('TranslationSubscriptions', sa.Column('last_check', sa.DateTime(), nullable=True)) op.create_index(u'ix_TranslationSubscriptions_last_check', 'TranslationSubscriptions', ['last_check'], unique=False) ### end Alembic commands ### with app.app_context(): update_stmt = translation_subscriptions.update().where( translation_subscriptions.c.last_check == None).values( last_check=datetime.datetime.utcnow() - datetime.timedelta(hours=72)) db.session.execute(update_stmt) db.session.commit()
def sync(self, only_recent): """ Fully synchronizes the local database leading translations with the MongoDB. """ if not flask_app.config["ACTIVATE_TRANSLATOR_MONGODB_PUSHES"]: return logger.info("[SYNC]: Starting Sync task") start_time = datetime.utcnow() if only_recent: oldest = datetime.utcnow() - timedelta(minutes=30) else: oldest = datetime(1970, 1, 1) with flask_app.app_context(): translation_bundles = [ { 'translation_url' : bundle.translation_url.url, 'language' : bundle.language, 'target' : bundle.target } for bundle in db.session.query(TranslationBundle).filter(ActiveTranslationMessage.datetime >= oldest, ActiveTranslationMessage.bundle_id == TranslationBundle.id).group_by(TranslationBundle.id).options(joinedload("translation_url")).all() ] if translation_bundles: all_translation_url_ids = [] all_app_ids = [] for translation_bundle in translation_bundles: responses = push(self = None, translation_url = translation_bundle['translation_url'], lang = translation_bundle['language'], target = translation_bundle['target']) if responses is None: logger.warn("Pushing translation for %s of %s returned None" % (translation_bundle['translation_url'], translation_bundle['language'])) continue if responses == 'timeout': logger.warn("Pushing translation for %s of %s returned a timeout error" % (translation_bundle['translation_url'], translation_bundle['language'])) break for response in responses: translation_url_id, app_ids = response all_translation_url_ids.append(translation_url_id) all_app_ids.extend(app_ids) if not only_recent: mongo_bundles.remove({"_id": {"$nin": all_app_ids}, "time": {"$lt": start_time}}) mongo_translation_urls.remove({"_id": {"$nin": all_translation_url_ids}, "time": {"$lt": start_time}}) logger.info("[SYNC]: Sync finished.")
def upgrade(): with app.app_context(): duplicated_bundles = list( db.session.query( TranslationBundle.c.translation_url_id, TranslationBundle.c.language, TranslationBundle.c.target).group_by( TranslationBundle.c.translation_url_id, TranslationBundle.c.language, TranslationBundle.c.target).having( func.count(TranslationBundle.c.id) > 1).all()) translation_url_ids = [ tr_id for tr_id, language, target in duplicated_bundles ] languages = [ language for tr_id, language, target in duplicated_bundles ] targets = [target for tr_id, language, target in duplicated_bundles] all_results = defaultdict(list) for bundle in db.session.query(TranslationBundle).filter( TranslationBundle.c.translation_url_id.in_( translation_url_ids), TranslationBundle.c.language.in_(languages), TranslationBundle.c.target.in_(targets)).all(): all_results[bundle.translation_url_id, bundle.language, bundle.target].append(bundle) all_bundle_ids = [] for key in duplicated_bundles: for bundle in all_results[key][1:]: all_bundle_ids.append(bundle.id) delete_msg_stmt = ActiveTranslationMessage.delete( ActiveTranslationMessage.c.bundle_id.in_(all_bundle_ids)) delete_hist_stmt = TranslationMessageHistory.delete( TranslationMessageHistory.c.bundle_id.in_(all_bundle_ids)) delete_bundle_stmt = TranslationBundle.delete( TranslationBundle.c.id.in_(all_bundle_ids)) connection = op.get_bind() connection.execute(delete_msg_stmt) connection.execute(delete_hist_stmt) connection.execute(delete_bundle_stmt) # ### commands auto generated by Alembic - please adjust! ### op.create_unique_constraint(None, 'TranslationBundles', ['translation_url_id', 'language', 'target'])
def upgrade(): ### commands auto generated by Alembic - please adjust! ### op.add_column('TranslationMessageHistory', sa.Column('category', sa.Unicode(length=255), nullable=True)) op.add_column('TranslationMessageHistory', sa.Column('fmt', sa.Unicode(length=255), nullable=True)) op.add_column('TranslationMessageHistory', sa.Column('from_developer', sa.Boolean(), nullable=True)) op.add_column('TranslationMessageHistory', sa.Column('namespace', sa.Unicode(length=255), nullable=True)) op.add_column('TranslationMessageHistory', sa.Column('position', sa.Integer(), nullable=True)) op.add_column('TranslationMessageHistory', sa.Column('same_tool', sa.Boolean(), nullable=True)) op.add_column('TranslationMessageHistory', sa.Column('tool_id', sa.Unicode(length=255), nullable=True)) op.create_index(u'ix_TranslationMessageHistory_category', 'TranslationMessageHistory', ['category'], unique=False) op.create_index(u'ix_TranslationMessageHistory_fmt', 'TranslationMessageHistory', ['fmt'], unique=False) op.create_index(u'ix_TranslationMessageHistory_from_developer', 'TranslationMessageHistory', ['from_developer'], unique=False) op.create_index(u'ix_TranslationMessageHistory_namespace', 'TranslationMessageHistory', ['namespace'], unique=False) op.create_index(u'ix_TranslationMessageHistory_position', 'TranslationMessageHistory', ['position'], unique=False) op.create_index(u'ix_TranslationMessageHistory_same_tool', 'TranslationMessageHistory', ['same_tool'], unique=False) op.create_index(u'ix_TranslationMessageHistory_tool_id', 'TranslationMessageHistory', ['tool_id'], unique=False) ### end Alembic commands ### with app.app_context(): # Create a list of fields such as active_translation.c.history_id, ... FIELD_NAMES = 'category', 'fmt', 'from_developer', 'namespace', 'position', 'same_tool', 'tool_id' fields = [] for field in ('history_id',) + FIELD_NAMES: fields.append(getattr(active_translation.c, field)) for active_message in db.session.execute(sql.select(fields)): parent_id = active_message[0] # field_values = { 'category' : (category), 'fmt' : (fmt), ... } field_values = dict(zip(FIELD_NAMES, active_message[1:])) while parent_id is not None: update_stmt = translation_history.update().where(translation_history.c.id == parent_id).values(**field_values) db.session.execute(update_stmt) parent_ids = db.session.execute(sql.select([ translation_history.c.parent_translation_id ], translation_history.c.id == parent_id)) parent_ids = list(parent_ids) if len(parent_ids) == 0 or len(parent_ids[0]) == 0: break parent_id = parent_ids[0][0] db.session.commit()
def upgrade(): ### commands auto generated by Alembic - please adjust! ### op.add_column('TranslationMessageHistory', sa.Column('category', sa.Unicode(length=255), nullable=True)) op.add_column('TranslationMessageHistory', sa.Column('fmt', sa.Unicode(length=255), nullable=True)) op.add_column('TranslationMessageHistory', sa.Column('from_developer', sa.Boolean(), nullable=True)) op.add_column( 'TranslationMessageHistory', sa.Column('namespace', sa.Unicode(length=255), nullable=True)) op.add_column('TranslationMessageHistory', sa.Column('position', sa.Integer(), nullable=True)) op.add_column('TranslationMessageHistory', sa.Column('same_tool', sa.Boolean(), nullable=True)) op.add_column('TranslationMessageHistory', sa.Column('tool_id', sa.Unicode(length=255), nullable=True)) op.create_index(u'ix_TranslationMessageHistory_category', 'TranslationMessageHistory', ['category'], unique=False) op.create_index(u'ix_TranslationMessageHistory_fmt', 'TranslationMessageHistory', ['fmt'], unique=False) op.create_index(u'ix_TranslationMessageHistory_from_developer', 'TranslationMessageHistory', ['from_developer'], unique=False) op.create_index(u'ix_TranslationMessageHistory_namespace', 'TranslationMessageHistory', ['namespace'], unique=False) op.create_index(u'ix_TranslationMessageHistory_position', 'TranslationMessageHistory', ['position'], unique=False) op.create_index(u'ix_TranslationMessageHistory_same_tool', 'TranslationMessageHistory', ['same_tool'], unique=False) op.create_index(u'ix_TranslationMessageHistory_tool_id', 'TranslationMessageHistory', ['tool_id'], unique=False) ### end Alembic commands ### with app.app_context(): # Create a list of fields such as active_translation.c.history_id, ... FIELD_NAMES = 'category', 'fmt', 'from_developer', 'namespace', 'position', 'same_tool', 'tool_id' fields = [] for field in ('history_id', ) + FIELD_NAMES: fields.append(getattr(active_translation.c, field)) for active_message in db.session.execute(sql.select(fields)): parent_id = active_message[0] # field_values = { 'category' : (category), 'fmt' : (fmt), ... } field_values = dict(zip(FIELD_NAMES, active_message[1:])) while parent_id is not None: update_stmt = translation_history.update().where( translation_history.c.id == parent_id).values( **field_values) db.session.execute(update_stmt) parent_ids = db.session.execute( sql.select([translation_history.c.parent_translation_id], translation_history.c.id == parent_id)) parent_ids = list(parent_ids) if len(parent_ids) == 0 or len(parent_ids[0]) == 0: break parent_id = parent_ids[0][0] db.session.commit()
def push(self, translation_url, lang, target, recursive = False): if not flask_app.config["ACTIVATE_TRANSLATOR_MONGODB_PUSHES"]: return if lang.startswith('en_'): # Don't send any English text return previous = [] if not recursive: if lang == 'zh_CN': for record in push(self, translation_url, 'zh_ALL', target, recursive=True): previous.append(record) elif lang == 'zh_ALL': for record in push(self, translation_url, 'zh_CN', target, recursive=True): previous.append(record) try: logger.info("[PUSH] Pushing to %s@%s" % (lang, translation_url)) print("[PUSH] Pushing to %s@%s" % (lang, translation_url)) with flask_app.app_context(): translation_bundle = db.session.query(TranslationBundle).filter(TranslationBundle.translation_url_id == TranslationUrl.id, TranslationUrl.url == translation_url, TranslationBundle.language == lang, TranslationBundle.target == target).options(joinedload("translation_url")).first() if translation_bundle is None: if lang == 'zh_CN': translation_bundle = db.session.query(TranslationBundle).filter(TranslationBundle.translation_url_id == TranslationUrl.id, TranslationUrl.url == translation_url, TranslationBundle.language == 'zh_ALL', TranslationBundle.target == target).options(joinedload("translation_url")).first() elif lang == 'zh_ALL': translation_bundle = db.session.query(TranslationBundle).filter(TranslationBundle.translation_url_id == TranslationUrl.id, TranslationUrl.url == translation_url, TranslationBundle.language == 'zh_CN', TranslationBundle.target == target).options(joinedload("translation_url")).first() if translation_bundle is None: return payload = {} max_date = datetime(1970, 1, 1) for message in translation_bundle.active_messages: payload[message.key] = message.value if message.datetime > max_date: max_date = message.datetime data = json.dumps(payload) lang_pack = lang + '_' + target bundle_id = lang_pack + '::' + translation_url bundle = { '_id' : bundle_id, 'url' : translation_url, 'bundle' : lang_pack, 'data' : data, 'time' : max_date } for mongo_translation_urls in all_mongo_translation_urls: try: mongo_translation_urls.update({'_id' : bundle_id, 'time' : { '$lt' : max_date }}, bundle, upsert = True) logger.info("[PUSH]: Updated translation URL bundle %s" % bundle_id) print("[PUSH]: Updated translation URL bundle %s" % bundle_id) except DuplicateKeyError: print("[PUSH]: Ignoring push for translation URL bundle %s (newer date exists already)" % bundle_id) app_bundle_ids = [] for application in translation_bundle.translation_url.apps: app_bundle_id = lang_pack + '::' + application.url app_bundle_ids.append(app_bundle_id) bundle = { '_id' : app_bundle_id, 'spec' : application.url, 'bundle' : lang_pack, 'data' : data, 'time' : max_date } for mongo_bundles in all_mongo_bundles: try: mongo_bundles.update({'_id' : app_bundle_id, 'time' : { '$lt' : max_date }}, bundle, upsert = True) logger.info("[PUSH]: Updated application bundle %s" % app_bundle_id) print("[PUSH]: Updated application bundle %s" % app_bundle_id) except DuplicateKeyError: print("[PUSH]: Ignoring push for application bundle %s (newer date exists already)" % app_bundle_id) previous.append([bundle_id, app_bundle_ids]) return previous except ServerSelectionTimeoutError as exc: logger.warn("[PUSH]: Exception occurred due to server disconnect. NOT RETRYING.", exc_info = True) return 'timeout' except Exception as exc: logger.warn("[PUSH]: Exception occurred. Retrying soon.", exc_info = True) print("[PUSH]: Exception occurred. Retrying soon.") if self is not None: raise self.retry(exc=exc, default_retry_delay=60, max_retries=None)
# 'Buenas' : 1, # } } remaining_texts = [] for translator in TRANSLATORS: current_translations, current_remaining_texts = translator.existing_translations(texts, language, origin_language) for remaining_text in current_remaining_texts: if remaining_text not in translations: remaining_texts.append(remaining_text) for translation_key, translation_values in current_translations.iteritems(): if translation_key in remaining_texts: remaining_texts.remove(translation_key) if translation_key not in translations: translations[translation_key] = {} for value, weight in translation_values.iteritems(): if value not in translations[translation_key]: translations[translation_key] = { value : 0 } translations[translation_key][value] += weight return translations, remaining_texts if __name__ == '__main__': from appcomposer import app with app.app_context(): print existing_translations(["Hello", "Bye", "Good morning", "This was never stored"], 'es') print translate_texts(["Hello", "Bye", "Good morning"], 'es')
def upgrade(): ### commands auto generated by Alembic - please adjust! ### op.add_column('ActiveTranslationMessages', sa.Column('from_developer', sa.Boolean(), nullable=True)) op.create_index(u'ix_ActiveTranslationMessages_from_developer', 'ActiveTranslationMessages', ['from_developer'], unique=False) ### end Alembic commands ### update_stmt = active_translation.update().where( active_translation.c.value == None).values(value="") with app.app_context(): db.session.execute(update_stmt) active_translations = { # bundle_id : { # key : value, # } } # 15256 with app.app_context(): for key, value, bundle_id in db.session.execute( sql.select([ active_translation.c.key, active_translation.c.value, active_translation.c.bundle_id ])): if bundle_id not in active_translations: active_translations[bundle_id] = {} active_translations[bundle_id][key] = value default_bundle_ids = { # translation_url_id : bundle_id # pointing to English } for bundle_id, translation_url_id, target in db.session.execute( sql.select([ bundle.c.id, bundle.c.translation_url_id, bundle.c.target ], bundle.c.language == 'en_ALL')): default_bundle_ids[translation_url_id] = bundle_id active_translation_query = sql.select( [ active_translation.c.id, active_translation.c.taken_from_default, bundle.c.from_developer, bundle.c.translation_url_id, bundle.c.id, active_translation.c.key, active_translation.c.value ], active_translation.c.bundle_id == bundle.c.id, use_labels=True) for row in db.session.execute(active_translation_query): active_msg_id = row[active_translation.c.id] from_default = row[active_translation.c.taken_from_default] bundle_from_developer = row[bundle.c.from_developer] bundle_id = row[bundle.c.id] translation_url_id = row[bundle.c.translation_url_id] current_value = row[active_translation.c.value] current_key = row[active_translation.c.key] default_bundle_id = default_bundle_ids.get(translation_url_id) default_value = active_translations.get(default_bundle_id, {}).get(current_key) if bundle_id == default_bundle_id: from_developer = True else: if bundle_from_developer and not from_default: if default_value is not None and current_value != default_value and current_value: from_developer = True else: from_developer = False from_default = True else: from_developer = False update_stmt = active_translation.update().where( active_translation.c.id == active_msg_id).values( from_developer=from_developer, taken_from_default=from_default) db.session.execute(update_stmt) db.session.commit()
def push(self, translation_url, lang, target): if not flask_app.config["ACTIVATE_TRANSLATOR_MONGODB_PUSHES"]: return try: logger.info("[PUSH] Pushing to %s@%s" % (lang, translation_url)) print("[PUSH] Pushing to %s@%s" % (lang, translation_url)) with flask_app.app_context(): translation_bundle = db.session.query(TranslationBundle).filter( TranslationBundle.translation_url_id == TranslationUrl.id, TranslationUrl.url == translation_url, TranslationBundle.language == lang, TranslationBundle.target == target).options( joinedload("translation_url")).first() if translation_bundle is None: return payload = {} max_date = datetime(1970, 1, 1) for message in translation_bundle.active_messages: payload[message.key] = message.value if message.datetime > max_date: max_date = message.datetime data = json.dumps(payload) lang_pack = lang + '_' + target bundle_id = lang_pack + '::' + translation_url bundle = { '_id': bundle_id, 'url': translation_url, 'bundle': lang_pack, 'data': data, 'time': max_date } try: mongo_translation_urls.update( { '_id': bundle_id, 'time': { '$lt': max_date } }, bundle, upsert=True) logger.info("[PUSH]: Updated translation URL bundle %s" % bundle_id) print("[PUSH]: Updated translation URL bundle %s" % bundle_id) except DuplicateKeyError: print( "[PUSH]: Ignoring push for translation URL bundle %s (newer date exists already)" % bundle_id) app_bundle_ids = [] for application in translation_bundle.translation_url.apps: app_bundle_id = lang_pack + '::' + application.url app_bundle_ids.append(app_bundle_id) bundle = { '_id': app_bundle_id, 'spec': application.url, 'bundle': lang_pack, 'data': data, 'time': max_date } try: mongo_bundles.update( { '_id': app_bundle_id, 'time': { '$lt': max_date } }, bundle, upsert=True) logger.info("[PUSH]: Updated application bundle %s" % app_bundle_id) print("[PUSH]: Updated application bundle %s" % app_bundle_id) except DuplicateKeyError: print( "[PUSH]: Ignoring push for application bundle %s (newer date exists already)" % app_bundle_id) return bundle_id, app_bundle_ids except Exception as exc: logger.warn("[PUSH]: Exception occurred. Retrying soon.", exc_info=True) print("[PUSH]: Exception occurred. Retrying soon.") if self is not None: raise self.retry(exc=exc, default_retry_delay=60, max_retries=None)
remaining_texts = [] for translator in TRANSLATORS: current_translations, current_remaining_texts = translator.existing_translations( texts, language, origin_language) for remaining_text in current_remaining_texts: if remaining_text not in translations: remaining_texts.append(remaining_text) for translation_key, translation_values in current_translations.iteritems( ): if translation_key in remaining_texts: remaining_texts.remove(translation_key) if translation_key not in translations: translations[translation_key] = {} for value, weight in translation_values.iteritems(): if value not in translations[translation_key]: translations[translation_key] = {value: 0} translations[translation_key][value] += weight return translations, remaining_texts if __name__ == '__main__': from appcomposer import app with app.app_context(): print existing_translations( ["Hello", "Bye", "Good morning", "This was never stored"], 'es') print translate_texts(["Hello", "Bye", "Good morning"], 'es')