import glob
import json
import datetime
from bson.json_util import loads
from appcomposer import app
from appcomposer.translator.ops import get_golab_default_user, add_full_translation_to_app
from appcomposer.translator.utils import extract_local_translations_url
files = glob.glob("dump/*")
with app.app_context():
user = get_golab_default_user()
for f in files:
print "Processing...",f,
translation = loads(open(f).read())
app_url = translation['spec']
language, target = translation['bundle'].rsplit('_', 1)
if language == 'all_ALL':
print "Skipping all_ALL"
continue
elif language.startswith('af'):
print "Skipping afghan and afrikaans"
continue
try:
translated_messages = json.loads(json.loads(translation['data']))
except (ValueError, TypeError):
translated_messages = json.loads(translation['data'])
def _add_or_update_app(app_url, metadata_information, repo_app_id, force_reload):
if DEBUG:
logger.debug("Starting %s" % app_url)
print("Starting %s" % app_url)
try:
repo_app = db.session.query(RepositoryApp).filter_by(id=repo_app_id).one()
initial_contents_hash = repo_app.contents_hash
initial_downloaded_hash = repo_app.downloaded_hash
default_user = get_golab_default_user()
translation_url = metadata_information.get('default_translation_url')
original_messages = metadata_information['default_translations']
default_metadata = metadata_information['default_metadata']
for language, translated_messages in metadata_information['original_translations'].iteritems():
add_full_translation_to_app(user_email = default_user.email, app_url = app_url, translation_url = translation_url,
app_metadata = default_metadata,
language = language, target = u'ALL', translated_messages = translated_messages,
original_messages = original_messages, from_developer = True)
namespaces = set([ msg['namespace'] for msg in original_messages.values() if msg['namespace'] ])
processed_languages = []
if namespaces:
pairs = []
for key, msg in original_messages.iteritems():
if msg['namespace']:
pairs.append({
'key' : key,
'namespace' : msg['namespace'],
})
for language_pack in get_bundles_by_key_namespaces(pairs):
cur_language = language_pack['language']
cur_target = language_pack['target']
if cur_target == 'ALL' and cur_language in metadata_information['original_translations']:
# Already processed
continue
processed_languages.append((cur_language, cur_target))
add_full_translation_to_app(user_email = default_user.email, app_url = app_url, translation_url = translation_url,
app_metadata = default_metadata,
language = cur_language, target = cur_target, translated_messages = {},
original_messages = original_messages, from_developer = False)
db_translation_url = db.session.query(TranslationUrl).filter_by(url = translation_url).first()
if db_translation_url:
for translation_bundle in db.session.query(TranslationBundle).filter_by(translation_url = db_translation_url).all():
if translation_bundle.target == u'ALL' and translation_bundle.language in metadata_information['original_translations']:
# Already processed
continue
found = False
for processed_language, processed_target in processed_languages:
if translation_bundle.target == processed_target and translation_bundle.language == processed_language:
found = True
break
if found:
# Already processed
continue
add_full_translation_to_app(user_email = default_user.email, app_url = app_url, translation_url = translation_url,
app_metadata = default_metadata,
language = translation_bundle.language, target = translation_bundle.target, translated_messages = None,
original_messages = original_messages, from_developer = False)
repo_app = db.session.query(RepositoryApp).filter_by(id=repo_app_id).one()
translation_percent = retrieve_translations_percent(translation_url, original_messages)
if translation_percent != repo_app.translation_percent:
repo_app.translation_percent = json.dumps(translation_percent)
repo_app.last_processed_contents_hash = initial_contents_hash
repo_app.last_processed_downloaded_hash = initial_downloaded_hash
repo_app.last_processed_time = datetime.datetime.utcnow()
try:
db.session.commit()
except:
import traceback
traceback.print_exc()
db.session.rollback()
raise
# In the meanwhile, maybe there were changes. Just make 100% sure that the hash is right
update_content_hash(app_url)
except:
import traceback
traceback.print_exc()
raise
def _add_or_update_app(cached_requests,
app_url,
force_reload,
repo_app=None,
task=None):
now = datetime.datetime.utcnow()
if DEBUG:
logger.debug("Starting %s" % app_url)
failing = False
if task is None:
try:
metadata_information = extract_metadata_information(
app_url, cached_requests, force_reload)
except Exception:
if DEBUG_VERBOSE:
print("Error on %s" % app_url)
traceback.print_exc()
logger.warning("Error extracting information from %s" % app_url,
exc_info=True)
metadata_information = {}
failing = True
else:
metadata_information = task.metadata_information or {}
failing = task.failing
if repo_app is not None:
if metadata_information.get('translatable') and len(
metadata_information.get('default_translations', [])) > 0:
repo_app.translatable = True
else:
# If it is translatable but there is no default translation; don't take it into account
repo_app.translatable = False
repo_app.adaptable = metadata_information.get('adaptable', False)
repo_app.original_translations = u','.join(
metadata_information.get('original_translations', {}).keys())
repo_app.last_change = now
repo_app.last_check = now
if failing:
if not repo_app.failing:
# Don't override if it was not failing before
repo_app.failing_since = now
repo_app.failing = True
else:
if repo_app.failing:
repo_app.failing = False
default_user = get_golab_default_user()
if metadata_information.get('translatable'):
translation_url = metadata_information.get('default_translation_url')
original_messages = metadata_information['default_translations']
default_metadata = metadata_information['default_metadata']
for language, translated_messages in metadata_information[
'original_translations'].iteritems():
add_full_translation_to_app(
user=default_user,
app_url=app_url,
translation_url=translation_url,
app_metadata=default_metadata,
language=language,
target=u'ALL',
translated_messages=translated_messages,
original_messages=original_messages,
from_developer=True)
namespaces = set([
msg['namespace'] for msg in original_messages.values()
if msg['namespace']
])
processed_languages = []
if namespaces:
pairs = []
for key, msg in original_messages.iteritems():
if msg['namespace']:
pairs.append({
'key': key,
'namespace': msg['namespace'],
})
for language_pack in get_bundles_by_key_namespaces(pairs):
cur_language = language_pack['language']
cur_target = language_pack['target']
if cur_target == 'ALL' and cur_language in metadata_information[
'original_translations']:
# Already processed
continue
processed_languages.append((cur_language, cur_target))
add_full_translation_to_app(
user=default_user,
app_url=app_url,
translation_url=translation_url,
app_metadata=default_metadata,
language=cur_language,
target=cur_target,
translated_messages={},
original_messages=original_messages,
from_developer=False)
db_translation_url = db.session.query(TranslationUrl).filter_by(
url=translation_url).first()
if db_translation_url:
for translation_bundle in db.session.query(
TranslationBundle).filter_by(
translation_url=db_translation_url).all():
if translation_bundle.target == u'ALL' and translation_bundle.language in metadata_information[
'original_translations']:
# Already processed
continue
found = False
for processed_language, processed_target in processed_languages:
if translation_bundle.target == processed_target and translation_bundle.language == processed_language:
found = True
break
if found:
# Already processed
continue
add_full_translation_to_app(
user=default_user,
app_url=app_url,
translation_url=translation_url,
app_metadata=default_metadata,
language=translation_bundle.language,
target=translation_bundle.target,
translated_messages=None,
original_messages=original_messages,
from_developer=False)
translation_percent = retrieve_translations_percent(
translation_url, original_messages)
if repo_app is not None and translation_percent != repo_app.translation_percent:
repo_app.translation_percent = json.dumps(translation_percent)
try:
db.session.commit()
except:
db.session.rollback()
raise
if active_message.value == '{0}':
reason(bundle, "value={0}")
continue
# if bundle.translation_url.url == 'http://go-lab.gw.utwente.nl/production/conceptmapper/build/languages/conceptmapper_en_ALL.xml' and active_message.key == 'help.helpDialog.content.conceptmapper' and bundle.language == 'es_ALL':
# print "*" * 20
# print "Processing {0}::{1}_{2}::{3} ({4!r}) ({5!r})".format(bundle.translation_url.url, bundle.language, bundle.target, active_message.key, active_message.value, original_messages.get(active_message.key))
# # print not active_message.taken_from_default
# # print active_message.from_developer
# # print active_message.value == original_messages.get(active_message.key, '________this.will.never.be')
# current = active_message.value
# past = original_messages.get(active_message.key, '________this.will.never.be')
# print current.strip() == past.strip()
# print "*" * 20
if not active_message.taken_from_default and active_message.from_developer and active_message.value.strip() == original_messages.get(active_message.key, '________this.will.never.be').strip():
total += 1
print "Processing {0}::{1}_{2}::{3} ({4!r})".format(bundle.translation_url.url, bundle.language, bundle.target, active_message.key, active_message.value)
ph = active_message.history
original_message = original_messages.get(active_message.key, '________this.will.never.be')
new_history = TranslationMessageHistory(bundle = ph.bundle, key = ph.key, value = original_message, user = get_golab_default_user(), datetime = datetime.datetime.utcnow(), parent_translation_id = ph.id, taken_from_default = True, same_tool = ph.same_tool, tool_id = ph.tool_id, fmt = ph.fmt, position = ph.position, category = ph.category, from_developer = ph.from_developer, namespace = ph.namespace)
db.session.add(new_history)
db.session.flush()
db.session.refresh(new_history)
am = active_message
new_active_message = ActiveTranslationMessage(bundle = bundle, key = am.key, value = original_message, history = new_history, datetime = datetime.datetime.utcnow(), taken_from_default = True, position = am.position, category = am.category, from_developer = am.from_developer, namespace = am.namespace, tool_id = am.tool_id, same_tool = am.same_tool, fmt = am.fmt)
db.session.delete(active_message)
db.session.add(new_active_message)
db.session.commit()
print("{0} records processed".format(total))
import glob
import json
import datetime
from bson.json_util import loads
from appcomposer import app
from appcomposer.translator.ops import get_golab_default_user, add_full_translation_to_app
from appcomposer.translator.utils import extract_local_translations_url
files = glob.glob("dump/*")
with app.app_context():
user = get_golab_default_user()
for f in files:
print "Processing...", f,
translation = loads(open(f).read())
app_url = translation['spec']
language, target = translation['bundle'].rsplit('_', 1)
if language == 'all_ALL':
print "Skipping all_ALL"
continue
elif language.startswith('af'):
print "Skipping afghan and afrikaans"
continue
try:
translated_messages = json.loads(json.loads(translation['data']))
except (ValueError, TypeError):
translated_messages = json.loads(translation['data'])
def _add_or_update_app(cached_requests, app_url, force_reload, repo_app = None, task = None):
now = datetime.datetime.utcnow()
if DEBUG:
logger.debug("Starting %s" % app_url)
failing = False
if task is None:
try:
metadata_information = extract_metadata_information(app_url, cached_requests, force_reload)
except Exception:
logger.warning("Error extracting information from %s" % app_url, exc_info = True)
metadata_information = {}
failing = True
else:
metadata_information = task.metadata_information or {}
failing = task.failing
if repo_app is not None:
repo_app.translatable = metadata_information.get('translatable', False)
repo_app.adaptable = metadata_information.get('adaptable', False)
repo_app.original_translations = u','.join(metadata_information.get('original_translations', {}).keys())
repo_app.last_change = now
repo_app.last_check = now
if failing:
repo_app.failing = True
repo_app.failing_since = now
else:
if repo_app.failing:
repo_app.failing = False
default_user = get_golab_default_user()
if metadata_information.get('translatable'):
translation_url = metadata_information.get('default_translation_url')
original_messages = metadata_information['default_translations']
default_metadata = metadata_information['default_metadata']
for language, translated_messages in metadata_information['original_translations'].iteritems():
add_full_translation_to_app(user = default_user, app_url = app_url, translation_url = translation_url,
app_metadata = default_metadata,
language = language, target = u'ALL', translated_messages = translated_messages,
original_messages = original_messages, from_developer = True)
namespaces = set([ msg['namespace'] for msg in original_messages.values() if msg['namespace'] ])
processed_languages = []
if namespaces:
pairs = []
for key, msg in original_messages.iteritems():
if msg['namespace']:
pairs.append({
'key' : key,
'namespace' : msg['namespace'],
})
for language_pack in get_bundles_by_key_namespaces(pairs):
cur_language = language_pack['language']
cur_target = language_pack['target']
if cur_target == 'ALL' and cur_language in metadata_information['original_translations']:
# Already processed
continue
processed_languages.append((cur_language, cur_target))
add_full_translation_to_app(user = default_user, app_url = app_url, translation_url = translation_url,
app_metadata = default_metadata,
language = cur_language, target = cur_target, translated_messages = {},
original_messages = original_messages, from_developer = False)
db_translation_url = db.session.query(TranslationUrl).filter_by(url = translation_url).first()
if db_translation_url:
for translation_bundle in db.session.query(TranslationBundle).filter_by(translation_url = db_translation_url).all():
if translation_bundle.target == u'ALL' and translation_bundle.language in metadata_information['original_translations']:
# Already processed
continue
found = False
for processed_language, processed_target in processed_languages:
if translation_bundle.target == processed_target and translation_bundle.language == processed_language:
found = True
break
if found:
# Already processed
continue
add_full_translation_to_app(user = default_user, app_url = app_url, translation_url = translation_url,
app_metadata = default_metadata,
language = translation_bundle.language, target = translation_bundle.target, translated_messages = None,
original_messages = original_messages, from_developer = False)
translation_percent = retrieve_translations_percent(translation_url, original_messages)
if repo_app is not None and translation_percent != repo_app.translation_percent:
repo_app.translation_percent = json.dumps(translation_percent)
try:
db.session.commit()
except:
db.session.rollback()
raise
def _add_or_update_app(app_url, metadata_information, repo_app_id,
force_reload):
if DEBUG:
logger.debug("Starting %s" % app_url)
print("Starting %s" % app_url)
try:
repo_app = db.session.query(RepositoryApp).filter_by(
id=repo_app_id).one()
initial_contents_hash = repo_app.contents_hash
initial_downloaded_hash = repo_app.downloaded_hash
default_user = get_golab_default_user()
translation_url = metadata_information.get('default_translation_url')
original_messages = metadata_information['default_translations']
default_metadata = metadata_information['default_metadata']
for language, translated_messages in metadata_information[
'original_translations'].iteritems():
add_full_translation_to_app(
user_email=default_user.email,
app_url=app_url,
translation_url=translation_url,
app_metadata=default_metadata,
language=language,
target=u'ALL',
translated_messages=translated_messages,
original_messages=original_messages,
from_developer=True)
namespaces = set([
msg['namespace'] for msg in original_messages.values()
if msg['namespace']
])
processed_languages = []
if namespaces:
pairs = []
for key, msg in original_messages.iteritems():
if msg['namespace']:
pairs.append({
'key': key,
'namespace': msg['namespace'],
})
for language_pack in get_bundles_by_key_namespaces(pairs):
cur_language = language_pack['language']
cur_target = language_pack['target']
if cur_target == 'ALL' and cur_language in metadata_information[
'original_translations']:
# Already processed
continue
processed_languages.append((cur_language, cur_target))
add_full_translation_to_app(
user_email=default_user.email,
app_url=app_url,
translation_url=translation_url,
app_metadata=default_metadata,
language=cur_language,
target=cur_target,
translated_messages={},
original_messages=original_messages,
from_developer=False)
db_translation_url = db.session.query(TranslationUrl).filter_by(
url=translation_url).first()
if db_translation_url:
for translation_bundle in db.session.query(
TranslationBundle).filter_by(
translation_url=db_translation_url).all():
if translation_bundle.target == u'ALL' and translation_bundle.language in metadata_information[
'original_translations']:
# Already processed
continue
found = False
for processed_language, processed_target in processed_languages:
if translation_bundle.target == processed_target and translation_bundle.language == processed_language:
found = True
break
if found:
# Already processed
continue
add_full_translation_to_app(
user_email=default_user.email,
app_url=app_url,
translation_url=translation_url,
app_metadata=default_metadata,
language=translation_bundle.language,
target=translation_bundle.target,
translated_messages=None,
original_messages=original_messages,
from_developer=False)
repo_app = db.session.query(RepositoryApp).filter_by(
id=repo_app_id).one()
translation_percent = retrieve_translations_percent(
translation_url, original_messages)
if translation_percent != repo_app.translation_percent:
repo_app.translation_percent = json.dumps(translation_percent)
repo_app.last_processed_contents_hash = initial_contents_hash
repo_app.last_processed_downloaded_hash = initial_downloaded_hash
repo_app.last_processed_time = datetime.datetime.utcnow()
try:
db.session.commit()
except:
import traceback
traceback.print_exc()
db.session.rollback()
raise
# In the meanwhile, maybe there were changes. Just make 100% sure that the hash is right
update_content_hash(app_url)
except:
import traceback
traceback.print_exc()
raise
