def insert_or_translate(target_file, lc):
#parse target file
target_kv = {}
target_error_lines = []
if not notexist_or_empty_file(target_file):
parsed_strings = strsparser.parse_strings(filename=target_file)
for item in parsed_strings:
k, v, e = item['key'], item['value'], item['error']
# line error
if e:
target_error_lines.append(e)
if not target_error_lines:
target_kv[k] = v
#parsing complete or return.
if target_error_lines:
print '(!) Syntax error - Skip'
return False, None, None, target_error_lines
#base
base_content = base_dict[os.path.basename(target_file)]
base_kv = {}
for item in base_content:
base_kv[item['key']] = item['value']
force_adding_keys = base_kv.keys() if __KEYS_FORCE_TRANSLATE_ALL__ else __KEYS_FORCE_TRANSLATE__
adding_keys = list(((set(base_kv.keys()) - set(target_kv.keys())) | (set(base_kv.keys()) & set(force_adding_keys))) - set(__KEYS_FOLLOW_BASE__))
removing_keys = list(set(target_kv.keys()) - set(base_kv.keys()))
existing_keys = list(set(base_kv.keys()) - (set(adding_keys) | set(removing_keys)))
updated_keys = []
"""
perform translate
"""
translated_kv = {};
if len(adding_keys):
print 'Translating...'
translated_kv = dict(zip(adding_keys, translate_ms([base_kv[k] for k in adding_keys], lc)))
updated_content = []
for item in base_content:
k = item['key']
newitem = dict.fromkeys(item.keys())
newitem['key'] = k
#added
if k in adding_keys:
if k in translated_kv:
newitem['value'] = translated_kv[k]
newitem['comment'] = 'Translated from: {0}'.format(base_kv[k])
print '[Add] "{0}" = "{1}" <- {2}'.format(k, newitem['value'], base_kv[k])
else:
newitem['value'] = target_kv[k]
newitem['comment'] = 'Translate failed from: {0}'.format(base_kv[k])
print '[Error] "{0}" = "{1}" X <- {2}'.format(k, newitem['value'], base_kv[k])
#exists
elif k in existing_keys:
target_value = target_kv.get(k)
if k in __KEYS_FOLLOW_BASE__:
newitem['value'] = base_kv[k]
if target_value != base_kv[k]:
updated_keys.append(k)
else:
newitem['value'] = target_value or base_kv[k]
if not target_value:
updated_keys.append(k)
updated_content.append(newitem)
#removed or wrong
for k in removing_keys:
print '[Remove]', k
if len(adding_keys) or len(removing_keys):
print '(i) Changed Keys: Added {0}, Updated {1}, Removed {2}'.format(len(adding_keys), len(updated_keys), len(removing_keys))
return updated_content and (len(adding_keys)>0 or len(updated_keys)>0 or len(removing_keys)>0), updated_content, translated_kv, target_error_lines
def strings_obj_from_file(file):
return strsparser.parse_strings(filename=file)
def insert_or_translate(target_file, lc):
#parse target file
target_kv = {}
target_error_lines = []
if not notexist_or_empty_file(target_file):
parsed_strings = strsparser.parse_strings(filename=target_file)
for item in parsed_strings:
k, v, e = item['key'], item['value'], item['error']
# line error
if e:
target_error_lines.append(e)
if not target_error_lines:
target_kv[k] = v
#parsing complete or return.
if target_error_lines:
print '(!) Syntax error - Skip'
return False, None, None, target_error_lines
#base
base_content = base_dict[os.path.basename(target_file)]
base_kv = {}
for item in base_content:
base_kv[item['key']] = item['value']
force_adding_keys = base_kv.keys(
) if __KEYS_FORCE_TRANSLATE_ALL__ else __KEYS_FORCE_TRANSLATE__
adding_keys = list(((set(base_kv.keys()) - set(target_kv.keys()))
| (set(base_kv.keys()) & set(force_adding_keys))) -
set(__KEYS_FOLLOW_BASE__))
removing_keys = list(set(target_kv.keys()) - set(base_kv.keys()))
existing_keys = list(
set(base_kv.keys()) - (set(adding_keys) | set(removing_keys)))
updated_keys = []
"""
perform translate
"""
translated_kv = {}
if len(adding_keys):
print 'Translating...'
translated_kv = dict(
zip(adding_keys,
translate_ms([base_kv[k] for k in adding_keys], lc)))
updated_content = []
for item in base_content:
k = item['key']
newitem = dict.fromkeys(item.keys())
newitem['key'] = k
#added
if k in adding_keys:
if k in translated_kv:
newitem['value'] = translated_kv[k]
newitem['comment'] = 'Translated from: {0}'.format(
base_kv[k])
print '[Add] "{0}" = "{1}" <- {2}'.format(
k, newitem['value'], base_kv[k])
else:
newitem['value'] = target_kv[k]
newitem['comment'] = 'Translate failed from: {0}'.format(
base_kv[k])
print '[Error] "{0}" = "{1}" X <- {2}'.format(
k, newitem['value'], base_kv[k])
#exists
elif k in existing_keys:
target_value = target_kv.get(k)
if k in __KEYS_FOLLOW_BASE__:
newitem['value'] = base_kv[k]
if target_value != base_kv[k]:
updated_keys.append(k)
else:
newitem['value'] = target_value or base_kv[k]
if not target_value:
updated_keys.append(k)
updated_content.append(newitem)
#removed or wrong
for k in removing_keys:
print '[Remove]', k
if len(adding_keys) or len(removing_keys):
print '(i) Changed Keys: Added {0}, Updated {1}, Removed {2}'.format(
len(adding_keys), len(updated_keys), len(removing_keys))
return updated_content and (
len(adding_keys) > 0 or len(updated_keys) > 0 or len(removing_keys)
> 0), updated_content, translated_kv, target_error_lines
def strings_obj_from_file(file):
return strsparser.parse_strings(filename=file)
def insert_or_translate(target_file, lc):
#parse target file
target_kv = {}
target_kc = {}
target_error_lines = []
if not notexist_or_empty_file(target_file):
parsed_strings = strsparser.parse_strings(filename=target_file)
for item in parsed_strings:
k, e = item['key'], item['error']
# line error
if e:
target_error_lines.append(e)
if not target_error_lines:
target_kv[k] = item['value']
target_kc[k] = item['comment']
#parsing complete or return.
if target_error_lines:
print '(!) Syntax error - Skip'
return False, None, None, target_error_lines
#base
base_content = base_dict[os.path.basename(target_file)]
base_kv = {}
base_kc = {}
for item in base_content:
k, e = item['key'], item['error']
# line error
if e:
print '(!) WARNING : Syntax error from Base -> ', k, ':' , e
base_kv[k] = item['value']
base_kc[k] = item['comment']
force_adding_keys = base_kv.keys() if __KEYS_FORCE_TRANSLATE_ALL__ else __KEYS_FORCE_TRANSLATE__
adding_keys = list(((set(base_kv.keys()) - set(target_kv.keys())) | (set(base_kv.keys()) & set(force_adding_keys))) - set(__KEYS_FOLLOW_BASE__))
removing_keys = list(set(target_kv.keys()) - set(base_kv.keys()))
existing_keys = list(set(base_kv.keys()) - (set(adding_keys) | set(removing_keys)))
updated_keys = []
"""
perform translate
"""
translated_kv = {}
reversed_matched_ratio_kv = {}
reversed_translated_kv = {}
if len(adding_keys):
print 'Translating...'
translated_kv = dict(zip(adding_keys, translate_ms([base_kv[k] for k in adding_keys], lc)))
if __VERIFY_TRANS_RESULTS__:
print 'Reversing results and matching...'
reversed_translated_kv = dict(zip(adding_keys, translate_ms(translated_kv.values(), 'en')))
for bk in adding_keys:
if bk in reversed_translated_kv:
ratio = fuzz.partial_ratio(base_kv[bk], reversed_translated_kv[bk])
if __IGNORE_UNVERIFIED_RESULTS__ and ratio <= __RATIO_TO_IGNORE_UNVERIFIED_RESULTS__:
translated_kv[k] = base_kv[k] # copy from base set
# adding_keys.remove(bk)
print 'Ignored:', bk, '<- Matching ratio: ', ratio
reversed_matched_ratio_kv[bk] = ratio
updated_content = []
for item in base_content:
k = item['key']
newitem = dict.fromkeys(item.keys())
newitem['key'] = k
target_value, target_comment = target_kv.get(k), target_kc.get(k)
newitem['comment'] = target_comment if __IGNORE_COMMENTS__ else target_comment or base_kc[k]
needs_update_comment = False if __IGNORE_COMMENTS__ else not target_comment and base_kc[k]
#added
if k in adding_keys:
if k in translated_kv:
newitem['value'] = translated_kv[k]
if not newitem['comment']:
newitem['comment'] = 'Translated from: {0}'.format(base_kv[k])
reversed_matched_msg = ''
if k in reversed_matched_ratio_kv:
reversed_matched_msg = Fore.CYAN+"( {} % Matched: \'{}\' <- \'{}\' <- \'{}\' )".format(reversed_matched_ratio_kv[k], reversed_translated_kv[k], newitem['value'], base_kv[k])+Style.RESET_ALL
print '[Add] "{0}" = "{1}" <- {2}'.format(k, newitem['value'], base_kv[k]), reversed_matched_msg
else:
newitem['value'] = target_kv[k]
if not newitem['comment']:
newitem['comment'] = 'Translate failed from: {0}'.format(base_kv[k])
print Fore.RED+'[Error] "{0}" = "{1}" X <- {2}'.format(k, newitem['value'], base_kv[k])+Style.RESET_ALL
#exists
elif k in existing_keys:
if k in __KEYS_FOLLOW_BASE_IF_LENGTH_LONGER__:
if target_value != base_kv[k] and __UNILEN__(target_value) > __UNILEN__(base_kv[k]) or needs_update_comment:
print Fore.YELLOW+'(!) Length of "', target_value, '" is longer than"', base_kv[k], '" as', len(target_value), '>', len(base_kv[k]), Style.RESET_ALL
newitem['value'] = base_kv[k]
updated_keys.append(k)
if not lc in global_result_logs:
global_result_logs[lc] = {}
global_result_logs[lc][k] = (target_value, base_kv[k])
else:
newitem['value'] = target_value or base_kv[k]
elif k in __KEYS_FOLLOW_BASE__:
newitem['value'] = base_kv[k]
if target_value != base_kv[k] or needs_update_comment:
updated_keys.append(k)
else:
newitem['value'] = target_value or base_kv[k]
if not target_value or needs_update_comment:
updated_keys.append(k)
updated_content.append(newitem)
#removed or wrong
for k in removing_keys:
print Fore.RED+'[Remove]', k,Style.RESET_ALL
if len(adding_keys) or len(updated_keys) or len(removing_keys):
print Fore.WHITE + '(i) Changed Keys: Added {0}, Updated {1}, Removed {2}'.format(len(adding_keys), len(updated_keys), len(removing_keys)) ,Style.RESET_ALL
#check verification failed items
target_verified_items = None
if len(reversed_matched_ratio_kv):
#filter(lambda k: reversed_matched_ratio_kv[k] < 1, reversed_matched_ratio_kv)
target_verified_items = {k: {'ratio': reversed_matched_ratio_kv[k], 'original': base_kv[k], 'reversed':reversed_translated_kv[k], 'translated':translated_kv[k] } for k in reversed_matched_ratio_kv.keys()}
return updated_content and (len(adding_keys)>0 or len(updated_keys)>0 or len(removing_keys)>0), updated_content, translated_kv, target_error_lines, target_verified_items
def insert_or_translate(target_file, lc):
#parse target file
target_kv = {}
target_kc = {}
target_error_lines = []
if not notexist_or_empty_file(target_file):
parsed_strings = strsparser.parse_strings(filename=target_file)
for item in parsed_strings:
k, e = item['key'], item['error']
# line error
if e:
target_error_lines.append(e)
if not target_error_lines:
target_kv[k] = item['value']
target_kc[k] = item['comment']
#parsing complete or return.
if target_error_lines:
print '(!) Syntax error - Skip'
return False, None, None, target_error_lines
#base
base_content = base_dict[os.path.basename(target_file)]
base_kv = {}
base_kc = {}
for item in base_content:
k, e = item['key'], item['error']
# line error
if e:
print '(!) WARNING : Syntax error from Base -> ', k, ':', e
base_kv[k] = item['value']
base_kc[k] = item['comment']
force_adding_keys = base_kv.keys(
) if __KEYS_FORCE_TRANSLATE_ALL__ else __KEYS_FORCE_TRANSLATE__
adding_keys = list(((set(base_kv.keys()) - set(target_kv.keys()))
| (set(base_kv.keys()) & set(force_adding_keys))) -
set(__KEYS_FOLLOW_BASE__))
removing_keys = list(set(target_kv.keys()) - set(base_kv.keys()))
existing_keys = list(
set(base_kv.keys()) - (set(adding_keys) | set(removing_keys)))
updated_keys = []
"""
perform translate
"""
translated_kv = {}
reversed_matched_ratio_kv = {}
reversed_translated_kv = {}
if len(adding_keys):
print 'Translating...'
translated_kv = dict(
zip(adding_keys,
translate_ms([base_kv[k] for k in adding_keys], lc)))
if __VERIFY_TRANS_RESULTS__:
print 'Reversing results and matching...'
reversed_translated_kv = dict(
zip(
adding_keys,
translate_ms(
[translated_kv[_ak] for _ak in adding_keys],
'en')))
for bk in adding_keys:
if bk in reversed_translated_kv:
ratio = fuzz.partial_ratio(base_kv[bk],
reversed_translated_kv[bk])
if __IGNORE_UNVERIFIED_RESULTS__ and ratio <= __RATIO_TO_IGNORE_UNVERIFIED_RESULTS__:
translated_kv[k] = base_kv[k] # copy from base set
# adding_keys.remove(bk)
print 'Ignored:', bk, '<- Matching ratio: ', ratio
reversed_matched_ratio_kv[bk] = ratio
updated_content = []
for item in base_content:
k = item['key']
newitem = dict.fromkeys(item.keys())
newitem['key'] = k
target_value, target_comment = target_kv.get(k), target_kc.get(k)
newitem[
'comment'] = target_comment if __IGNORE_COMMENTS__ else target_comment or base_kc[
k]
needs_update_comment = False if __IGNORE_COMMENTS__ else not target_comment and base_kc[
k]
#added
if k in adding_keys:
if k in translated_kv:
newitem['value'] = translated_kv[k]
if not newitem['comment']:
newitem['comment'] = 'Translated from: {0}'.format(
base_kv[k])
reversed_matched_msg = ''
if k in reversed_matched_ratio_kv:
reversed_matched_msg = Fore.CYAN + "( {} % Matched: \'{}\' <- \'{}\' <- \'{}\' )".format(
reversed_matched_ratio_kv[k],
reversed_translated_kv[k], newitem['value'],
base_kv[k]) + Style.RESET_ALL
print '[Add] "{0}" = "{1}" <- {2}'.format(
k, newitem['value'], base_kv[k]), reversed_matched_msg
else:
newitem['value'] = target_kv[k]
if not newitem['comment']:
newitem[
'comment'] = 'Translate failed from: {0}'.format(
base_kv[k])
print Fore.RED + '[Error] "{0}" = "{1}" X <- {2}'.format(
k, newitem['value'], base_kv[k]) + Style.RESET_ALL
#exists
elif k in existing_keys:
if k in __KEYS_FOLLOW_BASE_IF_LENGTH_LONGER__:
if target_value != base_kv[k] and __UNILEN__(
target_value) > __UNILEN__(
base_kv[k]) or needs_update_comment:
print Fore.YELLOW + '(!) Length of "', target_value, '" is longer than"', base_kv[
k], '" as', len(target_value), '>', len(
base_kv[k]), Style.RESET_ALL
newitem['value'] = base_kv[k]
updated_keys.append(k)
if not lc in global_result_logs:
global_result_logs[lc] = {}
global_result_logs[lc][k] = (target_value, base_kv[k])
else:
newitem['value'] = target_value or base_kv[k]
elif k in __KEYS_FOLLOW_BASE__:
newitem['value'] = base_kv[k]
if target_value != base_kv[k] or needs_update_comment:
updated_keys.append(k)
else:
newitem['value'] = target_value or base_kv[k]
if not target_value or needs_update_comment:
updated_keys.append(k)
updated_content.append(newitem)
#removed or wrong
for k in removing_keys:
print Fore.RED + '[Remove]', k, Style.RESET_ALL
if len(adding_keys) or len(updated_keys) or len(removing_keys):
print Fore.WHITE + '(i) Changed Keys: Added {0}, Updated {1}, Removed {2}'.format(
len(adding_keys), len(updated_keys),
len(removing_keys)), Style.RESET_ALL
#check verification failed items
target_verified_items = None
if len(reversed_matched_ratio_kv):
#filter(lambda k: reversed_matched_ratio_kv[k] < 1, reversed_matched_ratio_kv)
target_verified_items = {
k: {
'ratio': reversed_matched_ratio_kv[k],
'original': base_kv[k],
'reversed': reversed_translated_kv[k],
'translated': translated_kv[k]
}
for k in reversed_matched_ratio_kv.keys()
}
return updated_content and (
len(adding_keys) > 0 or len(updated_keys) > 0
or len(removing_keys) > 0
), updated_content, translated_kv, target_error_lines, target_verified_items
