def parseFile(filename): chunk = list() with open_n_decode(filename, "r", "utf-8") as f: string = prepare(f) jsondata = dict() try: jsondata = loads(string) except: print("Cannot parse " + filename) return [] paths = list_field_paths(jsondata) dialog = dirname(filename).endswith("dialog") for path in paths: for k in files_of_interest.keys(): if filename.endswith(k) or k == "*": for roi in files_of_interest[k]: if roi.match(path) or dialog: val = field_by_path(jsondata, path) if not type(val) is str: print("File: " + filename) print("Type of " + path + " is not a string!") continue if val == "": continue for handler in textHandlers: res = handler(val, filename, '/' + path) if res: chunk += res break break return chunk
def construct_db(assets_dir): print("Scanning assets at " + assets_dir) db = dict() db[""] = dict() foi = list() endings = tuple(files_of_interest.keys()) for subdir, dirs, files in walk(assets_dir): for thefile in files: if thefile.endswith(endings): foi.append(normpath(join(subdir, thefile))) with Pool(8) as p: r = p.imap_unordered(parseFile, foi) for chunk in r: for sec, val, fname, path in chunk: if sec not in db: db[sec] = dict() if val not in db[sec]: db[sec][val] = dict() filename = normpath( relpath(abspath(fname), abspath(assets_dir))) if filename not in db[sec][val]: db[sec][val][filename] = list() if path not in db[sec][val][filename]: insort_left(db[sec][val][filename], path) return db
def construct_db(assets_dir): ## Creating a database of text labels from game assets dir given ## the database has a following structure: ## {"section": { "label" : ## { "files were it used" : [list of fields were it used in file] } } } print("Scanning assets at " + assets_dir) db = dict() db[""] = dict() foi = list() endings = tuple(files_of_interest.keys()) for subdir, dirs, files in walk(assets_dir): for thefile in files: if thefile.endswith(endings): foi.append(normpath(join(subdir, thefile))) with Pool() as p: r = p.imap_unordered(parseFile, foi) for chunk in r: for sec, val, fname, path in chunk: if sec not in db: db[sec] = dict() if val not in db[sec]: db[sec][val] = dict() filename = normpath(relpath(abspath(fname), abspath(assets_dir))) if filename not in db[sec][val]: db[sec][val][filename] = list() if path not in db[sec][val][filename]: insort_left(db[sec][val][filename], path) return db
def parseFile(filename): chunk = list() if basename(filename) not in ignore_filelist: print(basename(filename)) with open_n_decode(filename, "r", "utf_8_sig") as f: try: if basename(filename).endswith('.patch'): chunk.append("patch") if basename(filename) in dict.keys(patch_serialization): string = trans_patch( f, patch_serialization[basename(filename)]) else: string = trans_patch(f) paths = to_a_list(string, 0) else: string = prepare(f) jsondata = loads(string) paths = list_field_paths(jsondata) except: print("Cannot parse " + filename) try: problem_file = open(error_list_file, 'a') problem_file.writelines( filename.replace(root_dir, '') + '\n') problem_file.close() except: pass return [] filename_base = filename if basename(filename).endswith('.patch'): filename = filename.replace('.patch', "") dialog = dirname(filename).endswith("dialog") for i, path in enumerate(paths): for k in files_of_interest.keys(): if filename.endswith(k) or k == "*": for roi in files_of_interest[k]: if roi.match(path) or dialog: if basename(filename_base).endswith('.patch'): val = to_a_list(string, 1)[i] else: val = field_by_path(jsondata, path) if not type(val) is str: print("File: " + filename) print("Type of " + path + " is not a string!") continue if val == "": continue for handler in textHandlers: res = handler(val, filename, '/' + path) if res: chunk += res break break return chunk
def parseFile(filename): chunk = list() if basename(filename) not in ignore_filelist_patch and basename( filename).endswith('.patch'): print(basename(filename)) with open_n_decode(filename, "r", "utf_8_sig") as f: try: if basename(filename) in patchfile_spciallist1: string = trans_patch(f) elif basename(filename) in patchfile_spciallist2: string = trans_patch(f) else: string = trans_patch(f) except: print("Cannot parse " + filename) problem_file = open(pro_list, 'a') problem_file.writelines(filename.replace(root_dir, '') + '\n') problem_file.close() return [] paths = to_a_list(string, 0) dialog = dirname(filename.replace('.patch', '')).endswith("dialog") for i, path in enumerate(paths): for k in files_of_interest.keys(): if filename.replace('.patch', '').endswith(k) or k == "*": for roi in files_of_interest[k]: if roi.match(path) or dialog: val = to_a_list(string, 1)[i] if not type(val) is str: print("File: " + filename) print("Type of " + path + " is not a string!") continue if val == "": continue for handler in textHandlers: res = handler( val, filename.replace('.patch', ''), '/' + path) if res: chunk += res break break return chunk
def construct_db(assets_dir): print("Scanning assets at " + assets_dir) db = [{"": dict()}, {"": dict()}] # db[""] = foi = list() endings = tuple(files_of_interest.keys()) for subdir, dirs, files in walk(assets_dir): for thefile in files: if subdir.replace('\\', '/').replace(root_dir, "") in dir_blacklist: break if thefile.endswith(endings) or thefile.endswith(".patch"): foi.append(normpath(join(subdir, thefile))) with Pool(parse_process_number) as p: r = p.imap_unordered(parseFile, foi) for chunk in r: if chunk == []: continue if chunk[0] == "patch": del chunk[0] chunk_parse(chunk, db[1], assets_dir) else: chunk_parse(chunk, db[0], assets_dir) return db