def insert_or_translate(target_file, lc): #parse target file target_kv = {} target_error_lines = [] if not notexist_or_empty_file(target_file): parsed_strings = strsparser.parse_strings(filename=target_file) for item in parsed_strings: k, v, e = item['key'], item['value'], item['error'] # line error if e: target_error_lines.append(e) if not target_error_lines: target_kv[k] = v #parsing complete or return. if target_error_lines: print '(!) Syntax error - Skip' return False, None, None, target_error_lines #base base_content = base_dict[os.path.basename(target_file)] base_kv = {} for item in base_content: base_kv[item['key']] = item['value'] force_adding_keys = base_kv.keys() if __KEYS_FORCE_TRANSLATE_ALL__ else __KEYS_FORCE_TRANSLATE__ adding_keys = list(((set(base_kv.keys()) - set(target_kv.keys())) | (set(base_kv.keys()) & set(force_adding_keys))) - set(__KEYS_FOLLOW_BASE__)) removing_keys = list(set(target_kv.keys()) - set(base_kv.keys())) existing_keys = list(set(base_kv.keys()) - (set(adding_keys) | set(removing_keys))) updated_keys = [] """ perform translate """ translated_kv = {}; if len(adding_keys): print 'Translating...' translated_kv = dict(zip(adding_keys, translate_ms([base_kv[k] for k in adding_keys], lc))) updated_content = [] for item in base_content: k = item['key'] newitem = dict.fromkeys(item.keys()) newitem['key'] = k #added if k in adding_keys: if k in translated_kv: newitem['value'] = translated_kv[k] newitem['comment'] = 'Translated from: {0}'.format(base_kv[k]) print '[Add] "{0}" = "{1}" <- {2}'.format(k, newitem['value'], base_kv[k]) else: newitem['value'] = target_kv[k] newitem['comment'] = 'Translate failed from: {0}'.format(base_kv[k]) print '[Error] "{0}" = "{1}" X <- {2}'.format(k, newitem['value'], base_kv[k]) #exists elif k in existing_keys: target_value = target_kv.get(k) if k in __KEYS_FOLLOW_BASE__: newitem['value'] = base_kv[k] if target_value != base_kv[k]: updated_keys.append(k) else: newitem['value'] = target_value or base_kv[k] if not target_value: updated_keys.append(k) updated_content.append(newitem) #removed or wrong for k in removing_keys: print '[Remove]', k if len(adding_keys) or len(removing_keys): print '(i) Changed Keys: Added {0}, Updated {1}, Removed {2}'.format(len(adding_keys), len(updated_keys), len(removing_keys)) return updated_content and (len(adding_keys)>0 or len(updated_keys)>0 or len(removing_keys)>0), updated_content, translated_kv, target_error_lines
def strings_obj_from_file(file): return strsparser.parse_strings(filename=file)
def insert_or_translate(target_file, lc): #parse target file target_kv = {} target_error_lines = [] if not notexist_or_empty_file(target_file): parsed_strings = strsparser.parse_strings(filename=target_file) for item in parsed_strings: k, v, e = item['key'], item['value'], item['error'] # line error if e: target_error_lines.append(e) if not target_error_lines: target_kv[k] = v #parsing complete or return. if target_error_lines: print '(!) Syntax error - Skip' return False, None, None, target_error_lines #base base_content = base_dict[os.path.basename(target_file)] base_kv = {} for item in base_content: base_kv[item['key']] = item['value'] force_adding_keys = base_kv.keys( ) if __KEYS_FORCE_TRANSLATE_ALL__ else __KEYS_FORCE_TRANSLATE__ adding_keys = list(((set(base_kv.keys()) - set(target_kv.keys())) | (set(base_kv.keys()) & set(force_adding_keys))) - set(__KEYS_FOLLOW_BASE__)) removing_keys = list(set(target_kv.keys()) - set(base_kv.keys())) existing_keys = list( set(base_kv.keys()) - (set(adding_keys) | set(removing_keys))) updated_keys = [] """ perform translate """ translated_kv = {} if len(adding_keys): print 'Translating...' translated_kv = dict( zip(adding_keys, translate_ms([base_kv[k] for k in adding_keys], lc))) updated_content = [] for item in base_content: k = item['key'] newitem = dict.fromkeys(item.keys()) newitem['key'] = k #added if k in adding_keys: if k in translated_kv: newitem['value'] = translated_kv[k] newitem['comment'] = 'Translated from: {0}'.format( base_kv[k]) print '[Add] "{0}" = "{1}" <- {2}'.format( k, newitem['value'], base_kv[k]) else: newitem['value'] = target_kv[k] newitem['comment'] = 'Translate failed from: {0}'.format( base_kv[k]) print '[Error] "{0}" = "{1}" X <- {2}'.format( k, newitem['value'], base_kv[k]) #exists elif k in existing_keys: target_value = target_kv.get(k) if k in __KEYS_FOLLOW_BASE__: newitem['value'] = base_kv[k] if target_value != base_kv[k]: updated_keys.append(k) else: newitem['value'] = target_value or base_kv[k] if not target_value: updated_keys.append(k) updated_content.append(newitem) #removed or wrong for k in removing_keys: print '[Remove]', k if len(adding_keys) or len(removing_keys): print '(i) Changed Keys: Added {0}, Updated {1}, Removed {2}'.format( len(adding_keys), len(updated_keys), len(removing_keys)) return updated_content and ( len(adding_keys) > 0 or len(updated_keys) > 0 or len(removing_keys) > 0), updated_content, translated_kv, target_error_lines
def insert_or_translate(target_file, lc): #parse target file target_kv = {} target_kc = {} target_error_lines = [] if not notexist_or_empty_file(target_file): parsed_strings = strsparser.parse_strings(filename=target_file) for item in parsed_strings: k, e = item['key'], item['error'] # line error if e: target_error_lines.append(e) if not target_error_lines: target_kv[k] = item['value'] target_kc[k] = item['comment'] #parsing complete or return. if target_error_lines: print '(!) Syntax error - Skip' return False, None, None, target_error_lines #base base_content = base_dict[os.path.basename(target_file)] base_kv = {} base_kc = {} for item in base_content: k, e = item['key'], item['error'] # line error if e: print '(!) WARNING : Syntax error from Base -> ', k, ':' , e base_kv[k] = item['value'] base_kc[k] = item['comment'] force_adding_keys = base_kv.keys() if __KEYS_FORCE_TRANSLATE_ALL__ else __KEYS_FORCE_TRANSLATE__ adding_keys = list(((set(base_kv.keys()) - set(target_kv.keys())) | (set(base_kv.keys()) & set(force_adding_keys))) - set(__KEYS_FOLLOW_BASE__)) removing_keys = list(set(target_kv.keys()) - set(base_kv.keys())) existing_keys = list(set(base_kv.keys()) - (set(adding_keys) | set(removing_keys))) updated_keys = [] """ perform translate """ translated_kv = {} reversed_matched_ratio_kv = {} reversed_translated_kv = {} if len(adding_keys): print 'Translating...' translated_kv = dict(zip(adding_keys, translate_ms([base_kv[k] for k in adding_keys], lc))) if __VERIFY_TRANS_RESULTS__: print 'Reversing results and matching...' reversed_translated_kv = dict(zip(adding_keys, translate_ms(translated_kv.values(), 'en'))) for bk in adding_keys: if bk in reversed_translated_kv: ratio = fuzz.partial_ratio(base_kv[bk], reversed_translated_kv[bk]) if __IGNORE_UNVERIFIED_RESULTS__ and ratio <= __RATIO_TO_IGNORE_UNVERIFIED_RESULTS__: translated_kv[k] = base_kv[k] # copy from base set # adding_keys.remove(bk) print 'Ignored:', bk, '<- Matching ratio: ', ratio reversed_matched_ratio_kv[bk] = ratio updated_content = [] for item in base_content: k = item['key'] newitem = dict.fromkeys(item.keys()) newitem['key'] = k target_value, target_comment = target_kv.get(k), target_kc.get(k) newitem['comment'] = target_comment if __IGNORE_COMMENTS__ else target_comment or base_kc[k] needs_update_comment = False if __IGNORE_COMMENTS__ else not target_comment and base_kc[k] #added if k in adding_keys: if k in translated_kv: newitem['value'] = translated_kv[k] if not newitem['comment']: newitem['comment'] = 'Translated from: {0}'.format(base_kv[k]) reversed_matched_msg = '' if k in reversed_matched_ratio_kv: reversed_matched_msg = Fore.CYAN+"( {} % Matched: \'{}\' <- \'{}\' <- \'{}\' )".format(reversed_matched_ratio_kv[k], reversed_translated_kv[k], newitem['value'], base_kv[k])+Style.RESET_ALL print '[Add] "{0}" = "{1}" <- {2}'.format(k, newitem['value'], base_kv[k]), reversed_matched_msg else: newitem['value'] = target_kv[k] if not newitem['comment']: newitem['comment'] = 'Translate failed from: {0}'.format(base_kv[k]) print Fore.RED+'[Error] "{0}" = "{1}" X <- {2}'.format(k, newitem['value'], base_kv[k])+Style.RESET_ALL #exists elif k in existing_keys: if k in __KEYS_FOLLOW_BASE_IF_LENGTH_LONGER__: if target_value != base_kv[k] and __UNILEN__(target_value) > __UNILEN__(base_kv[k]) or needs_update_comment: print Fore.YELLOW+'(!) Length of "', target_value, '" is longer than"', base_kv[k], '" as', len(target_value), '>', len(base_kv[k]), Style.RESET_ALL newitem['value'] = base_kv[k] updated_keys.append(k) if not lc in global_result_logs: global_result_logs[lc] = {} global_result_logs[lc][k] = (target_value, base_kv[k]) else: newitem['value'] = target_value or base_kv[k] elif k in __KEYS_FOLLOW_BASE__: newitem['value'] = base_kv[k] if target_value != base_kv[k] or needs_update_comment: updated_keys.append(k) else: newitem['value'] = target_value or base_kv[k] if not target_value or needs_update_comment: updated_keys.append(k) updated_content.append(newitem) #removed or wrong for k in removing_keys: print Fore.RED+'[Remove]', k,Style.RESET_ALL if len(adding_keys) or len(updated_keys) or len(removing_keys): print Fore.WHITE + '(i) Changed Keys: Added {0}, Updated {1}, Removed {2}'.format(len(adding_keys), len(updated_keys), len(removing_keys)) ,Style.RESET_ALL #check verification failed items target_verified_items = None if len(reversed_matched_ratio_kv): #filter(lambda k: reversed_matched_ratio_kv[k] < 1, reversed_matched_ratio_kv) target_verified_items = {k: {'ratio': reversed_matched_ratio_kv[k], 'original': base_kv[k], 'reversed':reversed_translated_kv[k], 'translated':translated_kv[k] } for k in reversed_matched_ratio_kv.keys()} return updated_content and (len(adding_keys)>0 or len(updated_keys)>0 or len(removing_keys)>0), updated_content, translated_kv, target_error_lines, target_verified_items
def insert_or_translate(target_file, lc): #parse target file target_kv = {} target_kc = {} target_error_lines = [] if not notexist_or_empty_file(target_file): parsed_strings = strsparser.parse_strings(filename=target_file) for item in parsed_strings: k, e = item['key'], item['error'] # line error if e: target_error_lines.append(e) if not target_error_lines: target_kv[k] = item['value'] target_kc[k] = item['comment'] #parsing complete or return. if target_error_lines: print '(!) Syntax error - Skip' return False, None, None, target_error_lines #base base_content = base_dict[os.path.basename(target_file)] base_kv = {} base_kc = {} for item in base_content: k, e = item['key'], item['error'] # line error if e: print '(!) WARNING : Syntax error from Base -> ', k, ':', e base_kv[k] = item['value'] base_kc[k] = item['comment'] force_adding_keys = base_kv.keys( ) if __KEYS_FORCE_TRANSLATE_ALL__ else __KEYS_FORCE_TRANSLATE__ adding_keys = list(((set(base_kv.keys()) - set(target_kv.keys())) | (set(base_kv.keys()) & set(force_adding_keys))) - set(__KEYS_FOLLOW_BASE__)) removing_keys = list(set(target_kv.keys()) - set(base_kv.keys())) existing_keys = list( set(base_kv.keys()) - (set(adding_keys) | set(removing_keys))) updated_keys = [] """ perform translate """ translated_kv = {} reversed_matched_ratio_kv = {} reversed_translated_kv = {} if len(adding_keys): print 'Translating...' translated_kv = dict( zip(adding_keys, translate_ms([base_kv[k] for k in adding_keys], lc))) if __VERIFY_TRANS_RESULTS__: print 'Reversing results and matching...' reversed_translated_kv = dict( zip( adding_keys, translate_ms( [translated_kv[_ak] for _ak in adding_keys], 'en'))) for bk in adding_keys: if bk in reversed_translated_kv: ratio = fuzz.partial_ratio(base_kv[bk], reversed_translated_kv[bk]) if __IGNORE_UNVERIFIED_RESULTS__ and ratio <= __RATIO_TO_IGNORE_UNVERIFIED_RESULTS__: translated_kv[k] = base_kv[k] # copy from base set # adding_keys.remove(bk) print 'Ignored:', bk, '<- Matching ratio: ', ratio reversed_matched_ratio_kv[bk] = ratio updated_content = [] for item in base_content: k = item['key'] newitem = dict.fromkeys(item.keys()) newitem['key'] = k target_value, target_comment = target_kv.get(k), target_kc.get(k) newitem[ 'comment'] = target_comment if __IGNORE_COMMENTS__ else target_comment or base_kc[ k] needs_update_comment = False if __IGNORE_COMMENTS__ else not target_comment and base_kc[ k] #added if k in adding_keys: if k in translated_kv: newitem['value'] = translated_kv[k] if not newitem['comment']: newitem['comment'] = 'Translated from: {0}'.format( base_kv[k]) reversed_matched_msg = '' if k in reversed_matched_ratio_kv: reversed_matched_msg = Fore.CYAN + "( {} % Matched: \'{}\' <- \'{}\' <- \'{}\' )".format( reversed_matched_ratio_kv[k], reversed_translated_kv[k], newitem['value'], base_kv[k]) + Style.RESET_ALL print '[Add] "{0}" = "{1}" <- {2}'.format( k, newitem['value'], base_kv[k]), reversed_matched_msg else: newitem['value'] = target_kv[k] if not newitem['comment']: newitem[ 'comment'] = 'Translate failed from: {0}'.format( base_kv[k]) print Fore.RED + '[Error] "{0}" = "{1}" X <- {2}'.format( k, newitem['value'], base_kv[k]) + Style.RESET_ALL #exists elif k in existing_keys: if k in __KEYS_FOLLOW_BASE_IF_LENGTH_LONGER__: if target_value != base_kv[k] and __UNILEN__( target_value) > __UNILEN__( base_kv[k]) or needs_update_comment: print Fore.YELLOW + '(!) Length of "', target_value, '" is longer than"', base_kv[ k], '" as', len(target_value), '>', len( base_kv[k]), Style.RESET_ALL newitem['value'] = base_kv[k] updated_keys.append(k) if not lc in global_result_logs: global_result_logs[lc] = {} global_result_logs[lc][k] = (target_value, base_kv[k]) else: newitem['value'] = target_value or base_kv[k] elif k in __KEYS_FOLLOW_BASE__: newitem['value'] = base_kv[k] if target_value != base_kv[k] or needs_update_comment: updated_keys.append(k) else: newitem['value'] = target_value or base_kv[k] if not target_value or needs_update_comment: updated_keys.append(k) updated_content.append(newitem) #removed or wrong for k in removing_keys: print Fore.RED + '[Remove]', k, Style.RESET_ALL if len(adding_keys) or len(updated_keys) or len(removing_keys): print Fore.WHITE + '(i) Changed Keys: Added {0}, Updated {1}, Removed {2}'.format( len(adding_keys), len(updated_keys), len(removing_keys)), Style.RESET_ALL #check verification failed items target_verified_items = None if len(reversed_matched_ratio_kv): #filter(lambda k: reversed_matched_ratio_kv[k] < 1, reversed_matched_ratio_kv) target_verified_items = { k: { 'ratio': reversed_matched_ratio_kv[k], 'original': base_kv[k], 'reversed': reversed_translated_kv[k], 'translated': translated_kv[k] } for k in reversed_matched_ratio_kv.keys() } return updated_content and ( len(adding_keys) > 0 or len(updated_keys) > 0 or len(removing_keys) > 0 ), updated_content, translated_kv, target_error_lines, target_verified_items