def parseFile(filename):
chunk = list()
with open_n_decode(filename, "r", "utf-8") as f:
string = prepare(f)
jsondata = dict()
try:
jsondata = loads(string)
except:
print("Cannot parse " + filename)
return []
paths = list_field_paths(jsondata)
dialog = dirname(filename).endswith("dialog")
for path in paths:
for k in files_of_interest.keys():
if filename.endswith(k) or k == "*":
for roi in files_of_interest[k]:
if roi.match(path) or dialog:
val = field_by_path(jsondata, path)
if not type(val) is str:
print("File: " + filename)
print("Type of " + path + " is not a string!")
continue
if val == "":
continue
for handler in textHandlers:
res = handler(val, filename, '/' + path)
if res:
chunk += res
break
break
return chunk
def construct_db(assets_dir):
print("Scanning assets at " + assets_dir)
db = dict()
db[""] = dict()
foi = list()
endings = tuple(files_of_interest.keys())
for subdir, dirs, files in walk(assets_dir):
for thefile in files:
if thefile.endswith(endings):
foi.append(normpath(join(subdir, thefile)))
with Pool(8) as p:
r = p.imap_unordered(parseFile, foi)
for chunk in r:
for sec, val, fname, path in chunk:
if sec not in db:
db[sec] = dict()
if val not in db[sec]:
db[sec][val] = dict()
filename = normpath(
relpath(abspath(fname), abspath(assets_dir)))
if filename not in db[sec][val]:
db[sec][val][filename] = list()
if path not in db[sec][val][filename]:
insort_left(db[sec][val][filename], path)
return db
def construct_db(assets_dir):
## Creating a database of text labels from game assets dir given
## the database has a following structure:
## {"section": { "label" :
## { "files were it used" : [list of fields were it used in file] } } }
print("Scanning assets at " + assets_dir)
db = dict()
db[""] = dict()
foi = list()
endings = tuple(files_of_interest.keys())
for subdir, dirs, files in walk(assets_dir):
for thefile in files:
if thefile.endswith(endings):
foi.append(normpath(join(subdir, thefile)))
with Pool() as p:
r = p.imap_unordered(parseFile, foi)
for chunk in r:
for sec, val, fname, path in chunk:
if sec not in db:
db[sec] = dict()
if val not in db[sec]:
db[sec][val] = dict()
filename = normpath(relpath(abspath(fname), abspath(assets_dir)))
if filename not in db[sec][val]:
db[sec][val][filename] = list()
if path not in db[sec][val][filename]:
insort_left(db[sec][val][filename], path)
return db
def parseFile(filename):
chunk = list()
with open_n_decode(filename, "r", "utf-8") as f:
string = prepare(f)
jsondata = dict()
try:
jsondata = loads(string)
except:
print("Cannot parse " + filename)
return []
paths = list_field_paths(jsondata)
dialog = dirname(filename).endswith("dialog")
for path in paths:
for k in files_of_interest.keys():
if filename.endswith(k) or k == "*":
for roi in files_of_interest[k]:
if roi.match(path) or dialog:
val = field_by_path(jsondata, path)
if not type(val) is str:
print("File: " + filename)
print("Type of " + path + " is not a string!")
continue
if val == "":
continue
for handler in textHandlers:
res = handler(val, filename, '/' + path)
if res:
chunk += res
break
break
return chunk
def construct_db(assets_dir):
## Creating a database of text labels from game assets dir given
## the database has a following structure:
## {"section": { "label" :
## { "files were it used" : [list of fields were it used in file] } } }
print("Scanning assets at " + assets_dir)
db = dict()
db[""] = dict()
foi = list()
endings = tuple(files_of_interest.keys())
for subdir, dirs, files in walk(assets_dir):
for thefile in files:
if thefile.endswith(endings):
foi.append(normpath(join(subdir, thefile)))
with Pool() as p:
r = p.imap_unordered(parseFile, foi)
for chunk in r:
for sec, val, fname, path in chunk:
if sec not in db:
db[sec] = dict()
if val not in db[sec]:
db[sec][val] = dict()
filename = normpath(relpath(abspath(fname), abspath(assets_dir)))
if filename not in db[sec][val]:
db[sec][val][filename] = list()
if path not in db[sec][val][filename]:
insort_left(db[sec][val][filename], path)
return db
def parseFile(filename):
chunk = list()
if basename(filename) not in ignore_filelist:
print(basename(filename))
with open_n_decode(filename, "r", "utf_8_sig") as f:
try:
if basename(filename).endswith('.patch'):
chunk.append("patch")
if basename(filename) in dict.keys(patch_serialization):
string = trans_patch(
f, patch_serialization[basename(filename)])
else:
string = trans_patch(f)
paths = to_a_list(string, 0)
else:
string = prepare(f)
jsondata = loads(string)
paths = list_field_paths(jsondata)
except:
print("Cannot parse " + filename)
try:
problem_file = open(error_list_file, 'a')
problem_file.writelines(
filename.replace(root_dir, '') + '\n')
problem_file.close()
except:
pass
return []
filename_base = filename
if basename(filename).endswith('.patch'):
filename = filename.replace('.patch', "")
dialog = dirname(filename).endswith("dialog")
for i, path in enumerate(paths):
for k in files_of_interest.keys():
if filename.endswith(k) or k == "*":
for roi in files_of_interest[k]:
if roi.match(path) or dialog:
if basename(filename_base).endswith('.patch'):
val = to_a_list(string, 1)[i]
else:
val = field_by_path(jsondata, path)
if not type(val) is str:
print("File: " + filename)
print("Type of " + path +
" is not a string!")
continue
if val == "":
continue
for handler in textHandlers:
res = handler(val, filename, '/' + path)
if res:
chunk += res
break
break
return chunk
def parseFile(filename):
chunk = list()
if basename(filename) not in ignore_filelist_patch and basename(
filename).endswith('.patch'):
print(basename(filename))
with open_n_decode(filename, "r", "utf_8_sig") as f:
try:
if basename(filename) in patchfile_spciallist1:
string = trans_patch(f)
elif basename(filename) in patchfile_spciallist2:
string = trans_patch(f)
else:
string = trans_patch(f)
except:
print("Cannot parse " + filename)
problem_file = open(pro_list, 'a')
problem_file.writelines(filename.replace(root_dir, '') + '\n')
problem_file.close()
return []
paths = to_a_list(string, 0)
dialog = dirname(filename.replace('.patch', '')).endswith("dialog")
for i, path in enumerate(paths):
for k in files_of_interest.keys():
if filename.replace('.patch', '').endswith(k) or k == "*":
for roi in files_of_interest[k]:
if roi.match(path) or dialog:
val = to_a_list(string, 1)[i]
if not type(val) is str:
print("File: " + filename)
print("Type of " + path +
" is not a string!")
continue
if val == "":
continue
for handler in textHandlers:
res = handler(
val, filename.replace('.patch', ''),
'/' + path)
if res:
chunk += res
break
break
return chunk
def construct_db(assets_dir):
print("Scanning assets at " + assets_dir)
db = [{"": dict()}, {"": dict()}]
# db[""] =
foi = list()
endings = tuple(files_of_interest.keys())
for subdir, dirs, files in walk(assets_dir):
for thefile in files:
if subdir.replace('\\', '/').replace(root_dir,
"") in dir_blacklist:
break
if thefile.endswith(endings) or thefile.endswith(".patch"):
foi.append(normpath(join(subdir, thefile)))
with Pool(parse_process_number) as p:
r = p.imap_unordered(parseFile, foi)
for chunk in r:
if chunk == []:
continue
if chunk[0] == "patch":
del chunk[0]
chunk_parse(chunk, db[1], assets_dir)
else:
chunk_parse(chunk, db[0], assets_dir)
return db
