def extract_gif_chunk():
"""
Extract all of the GIF images from gifchunk.bin as PNGs.
gifchunk.bin is just raw gifs that were found lying around in the wdb.
They aren't specifically assigned to any models, and are likely just
loaded into memory when Lego Island loads.
"""
bin_file = open(SETTINGS["gif_path"] + "/gifchunk.bin", "rb")
data = get_formatted_data(bin_file, "wdb", "gifchunk")
for image in data["images"]:
export_gif(image, SETTINGS["gif_path"], bin_file)
def extract_gif_chunk():
"""
Extract all of the GIF images from gifchunk.bin as PNGs.
gifchunk.bin is just raw gifs that were found lying around in the wdb.
They aren't specifically assigned to any models, and are likely just
loaded into memory when Lego Island loads.
"""
bin_file = open(SETTINGS["gif_path"] + "/gifchunk.bin", "rb")
data = get_formatted_data(bin_file, "wdb", "gifchunk")
for image in data["images"]:
export_gif(image, SETTINGS["gif_path"], bin_file)
def extract_wdb():
"""
Open WORLD.WDB and write all the sections as *.bin files.
Open WORLD.WDB as specified by the structure found in the wdb format
file. Export each section of the wdb as a bin file in the folder
hiearchy found in the header of the wdb file.
"""
bin_file = open(SETTINGS["wdb_path"], "rb")
data = get_formatted_data(bin_file, "wdb", "wdb")
for each_group in data["groups"]:
group_title = get_raw(each_group["group_title"], bin_file)
i = -1
for each_subgroup in each_group["subgroups"]:
i += 1
subgroup_title = "sub" + str(i)
for each_sub_item in each_subgroup["sub_items"]:
sub_item_title = get_raw(each_sub_item["sub_item_title"],
bin_file)
item_offset = get_raw(each_sub_item["item_offset"], bin_file)
size_of_item = get_raw(each_sub_item["size_of_item"], bin_file)
directory = create_dir(SETTINGS["bin_path"] + "/" +
group_title[:-1] + "/" +
subgroup_title + "/")
write_file = open(
directory + "/" + sub_item_title[:-1] + ".bin", "wb")
write_file.truncate()
bin_file.seek(item_offset)
write_file.write(bin_file.read(size_of_item))
write_file.close()
#write gif chunk to be extracted by extract_gif_chunk()
directory = create_dir(SETTINGS["gif_path"])
write_file = open(directory + "/gifchunk.bin", "wb")
bin_file.seek(data["gif_chunk_size"][1] + 4)
write_file.write(bin_file.read(get_raw(data["gif_chunk_size"], bin_file)))
write_file.close()
#write model chunk to be extracted by extract_model_chunk()
directory = create_dir(SETTINGS["gif_path"])
write_file = open(directory + "/modelchunk.bin", "wb")
bin_file.seek(data["model_chunk_size"][1] + 4)
write_file.write(bin_file.read(get_raw(data["model_chunk_size"],
bin_file)))
write_file.close()
def extract_model_chunk():
bin_file = open(SETTINGS["gif_path"] + "/modelchunk.bin", "rb")
data = get_formatted_data(bin_file, "wdb", "modelchunk")
bin_file.seek(0)
for binn in data["bins"]:
end_bin_offset = get_raw(binn["end_bin_offset"], bin_file)
size_of_item = end_bin_offset - bin_file.tell()
write_file = open(SETTINGS["bin_path"] + "/" + get_raw(binn["bin_name"], bin_file) + ".bin", "wb")
write_file.truncate()
write_file.write(bin_file.read(size_of_item))
write_file.close()
bin_file.seek(end_bin_offset)
def extract_wdb():
"""
Open WORLD.WDB and write all the sections as *.bin files.
Open WORLD.WDB as specified by the structure found in the wdb format
file. Export each section of the wdb as a bin file in the folder
hiearchy found in the header of the wdb file.
"""
bin_file = open(SETTINGS["wdb_path"], "rb")
data = get_formatted_data(bin_file, "wdb", "wdb")
for each_group in data["groups"]:
group_title = get_raw(each_group["group_title"], bin_file)
i = -1
for each_subgroup in each_group["subgroups"]:
i += 1
subgroup_title = "sub" + str(i)
for each_sub_item in each_subgroup["sub_items"]:
sub_item_title = get_raw(each_sub_item["sub_item_title"], bin_file)
item_offset = get_raw(each_sub_item["item_offset"], bin_file)
size_of_item = get_raw(each_sub_item["size_of_item"], bin_file)
directory = create_dir(SETTINGS["bin_path"] + "/" + group_title[:-1] + "/" + subgroup_title + "/")
write_file = open(directory + "/" + sub_item_title[:-1] + ".bin", "wb")
write_file.truncate()
bin_file.seek(item_offset)
write_file.write(bin_file.read(size_of_item))
write_file.close()
#write gif chunk to be extracted by extract_gif_chunk()
directory = create_dir(SETTINGS["gif_path"])
write_file = open(directory + "/gifchunk.bin", "wb")
bin_file.seek(data["gif_chunk_size"][1]+4)
write_file.write(bin_file.read(get_raw(data["gif_chunk_size"], bin_file)))
write_file.close()
#write model chunk to be extracted by extract_model_chunk()
directory = create_dir(SETTINGS["gif_path"])
write_file = open(directory + "/modelchunk.bin", "wb")
bin_file.seek(data["model_chunk_size"][1]+4)
write_file.write(bin_file.read(get_raw(data["model_chunk_size"], bin_file)))
write_file.close()
def extract_model_chunk():
bin_file = open(SETTINGS["gif_path"] + "/modelchunk.bin", "rb")
data = get_formatted_data(bin_file, "wdb", "modelchunk")
bin_file.seek(0)
for binn in data["bins"]:
end_bin_offset = get_raw(binn["end_bin_offset"], bin_file)
size_of_item = end_bin_offset - bin_file.tell()
write_file = open(
SETTINGS["bin_path"] + "/" + get_raw(binn["bin_name"], bin_file) +
".bin", "wb")
write_file.truncate()
write_file.write(bin_file.read(size_of_item))
write_file.close()
bin_file.seek(end_bin_offset)
def extract_pattern(file_path, pattern):
"""
Attempt to extract data from a .bin file and return the progress made.
There are five steps to extraction:
* Interpret data from the wdb using the wdb format file
* Export the object files using the data
* Export textures
* Export materials
* Export the material file mtl
These steps are represented by the list of five items returned by the
function that signify if that step was successful. '_' means success,
and 'X' means failure.
"""
global MATERIALS
global STATS
bin_file = open(file_path, "rb")
#reset materials to empty
MATERIALS = {}
progress = ["X", "X", "X", "X", "X"]
#INTERPRET FORMAT FROM FILE
try:
data = get_formatted_data(bin_file, "wdb", pattern)
#trace(str(data))
progress[0] = "_"
except:
trace_error()
return progress
#EXPORT OBJ
try:
file_name = get_raw(data["file_name"], bin_file)
#trace(data)
for component in data["components"]:
component_name = get_raw(component["component_name"], bin_file)
if "models" in component:
model_index = len(component["models"])
for model in component["models"]:
model_index -= 1
#determine whether or not to export this LOD model based on SETTINGS
export = False
if SETTINGS["highest_lod_only"]:
if model_index == 0:
export = True
else:
export = True
if export:
if SETTINGS["highest_lod_only"] and not SETTINGS["lod_labels"]:
end_string = ""
else:
end_string = "_lod" + str(model_index)
file_path = file_path.replace("\\", "/")
obj_path = create_dir(SETTINGS["obj_path"] + "/" + file_path[file_path.find("/", 3):file_path.rfind("/")] + "/" + file_name + "/")
export_obj(data, model, bin_file, obj_path + "/" + component_name + end_string)
else:
#no models in this component, only the component header
pass
progress[1] = "_"
except:
trace_error()
return progress
found_materials = []
#EXPORT TEXTURES
try:
num_images = get_raw(data["num_images"], bin_file)
#export textures embedded in this bin group as .png
for image in data["images"]:
#normal gif
export_gif(image, obj_path, bin_file)
#special hidden gif, only seen on isle and isle_hi gifs
if "extra_images" in image:
image["extra_images"][0]["gif_name"] = image["gif_name"]
export_gif(image["extra_images"][0], obj_path, bin_file, pretext="hidden_")
found_materials.append(get_raw(image["gif_name"], bin_file)[:-4])
progress[2] = "_"
except:
trace_error()
return progress
#EXPORT MATERIALS
try:
#export materials without textures as .png, just their rgb on a 4x4 texture
for material in MATERIALS:
if material not in found_materials:
found_materials.append(material)
#write 4x4 png of color c
c = MATERIALS[material]
rows = []
for row in range(4):
rows.append([])
for pixel in range(4):
rows[row].append(c[0])
rows[row].append(c[1])
rows[row].append(c[2])
f = open(obj_path + "/" + material + ".png", "wb")
f.truncate()
w = png.Writer(4, 4)
w.write(f, rows)
f.close()
#statistics for materials
for material in found_materials:
found_duplicate = False
for row in STATS["csv"]["materials"]:
if row[0] == material:
found_duplicate = True
row[2] += 1
if not found_duplicate:
STATS["csv"]["materials"].append([material, "No", 1])
progress[3] = "_"
except:
trace_error()
return progress
#EXPORT MTL FILE
try:
export_mtl(data, obj_path, bin_file)
progress[4] = "_"
except:
trace_error()
return progress
return progress
def main():
bin_file = open(SETTINGS["filename"], "rb")
editor = Editor(bin_file)
overlay = Overlay()
bin_file.seek(0)
if SETTINGS["ignore_pattern"] == False:
data = get_formatted_data(bin_file, SETTINGS["format"], SETTINGS["pattern"])
#tuple_data is (key, (data_string, offset))
nested_values = sorted(get_nested_values(data), key=lambda x: x[1][1])
nested_index = 0
for tuple_data in nested_values:
key = tuple_data[0]
data_string = tuple_data[1][0]
offset = tuple_data[1][1]
new_group = Group(key, data_string, offset, bin_file)
editor.groups.append(new_group)
selected_data = None
while(True):
for event in pygame.event.get():
if event.type == pygame.QUIT:
#editor.save()
sys.exit(0)
if event.type == pygame.KEYDOWN:
INFO["update_display"] = True
if event.key == pygame.K_i:
editor.groups.append(Group("UNDEFINED", "i", selected_data.offset, bin_file))
if event.key == pygame.K_h:
editor.groups.append(Group("UNDEFINED", "h", selected_data.offset, bin_file))
if event.key == pygame.K_b:
editor.groups.append(Group("UNDEFINED", "B", selected_data.offset, bin_file))
if event.key == pygame.K_f:
editor.groups.append(Group("UNDEFINED", "f", selected_data.offset, bin_file))
if event.key == pygame.K_s:
length = get_num()
editor.groups.append(Group("UNDEFINED", "s"+str(length), selected_data.offset, bin_file))
if event.key == pygame.K_BACKSPACE:
for each_group in editor.groups:
if each_group.offset == selected_data.offset:
editor.groups.remove(each_group)
if event.key == pygame.K_g:
offset = get_num()
INFO["y_offset"] = -int(offset/SETTINGS["cells_per_row"])
editor.update_rects()
editor.populate()
if event.key == pygame.K_SPACE:
if nested_index < len(nested_values)-1:
tuple_data = nested_values[nested_index]
key = tuple_data[0]
data_string = tuple_data[1][0]
offset = tuple_data[1][1]
if not data_string.startswith("s"):
new_group = Group(key, data_string, offset, bin_file)
editor.groups.append(new_group)
nested_index += 1
else:
print("NO MORE NESTED VALUES")
if event.type == pygame.MOUSEBUTTONDOWN:
INFO["update_display"] = True
if event.button == 4:
if INFO["y_offset"] < 0:
INFO["y_offset"] += 1
editor.update_rects()
editor.populate()
selected_data = None
elif event.button == 5:
INFO["y_offset"] -= 1
editor.update_rects()
editor.populate()
selected_data = None
#select data with the mouse
mouse_pos = pygame.mouse.get_pos()
mouse_pressed = pygame.mouse.get_pressed()
if mouse_pressed[0]:
for each_data in editor.data:
if each_data.is_in(mouse_pos):
INFO["update_display"] = True
selected_data = each_data
#hover over groups with the mouse
try:
del(overlay.icons["mouse_icon"])
except:
pass
for each_group in editor.groups:
if each_group.is_in(mouse_pos):
INFO["update_overlay"] = True
name_text = main_font.render(each_group.name, True, (50, 50, 50))
name_rect = pygame.surface.Surface((name_text.get_width(), name_text.get_height()))
name_rect.fill((255, 255, 255))
name_rect.blit(name_text, (0, 0))
name_pos = [mouse_pos[0], mouse_pos[1]+10]
if name_pos[0]+name_rect.get_width() > SETTINGS["screen_x"]:
name_pos[0] = SETTINGS["screen_x"]-name_rect.get_width()
overlay.icons["mouse_icon"] = Icon(name_rect, name_pos)
#only redraw editor if required
if INFO["update_display"]:
INFO["update_display"] = False
#draw editor and selected_data
editor.draw()
if selected_data != None:
selected_data.draw(True)
#save a bg_image so that it does not have to be redrawn with every overlay update
INFO["bg_image"] = pygame.surface.Surface((screen.get_width(), screen.get_height()))
INFO["bg_image"].blit(screen, (0, 0))
if INFO["update_overlay"]:
screen.blit(INFO["bg_image"], (0, 0))
overlay.draw()
pygame.display.flip()
def extract_pattern(file_path, pattern):
"""
Attempt to extract data from a .bin file and return the progress made.
There are five steps to extraction:
* Interpret data from the wdb using the wdb format file
* Export the object files using the data
* Export textures
* Export materials
* Export the material file mtl
These steps are represented by the list of five items returned by the
function that signify if that step was successful. '_' means success,
and 'X' means failure.
"""
global MATERIALS
global STATS
bin_file = open(file_path, "rb")
#reset materials to empty
MATERIALS = {}
progress = ["X", "X", "X", "X", "X"]
#INTERPRET FORMAT FROM FILE
try:
data = get_formatted_data(bin_file, "wdb", pattern)
#trace(str(data))
progress[0] = "_"
except:
trace_error()
return progress
#EXPORT OBJ
try:
file_name = get_raw(data["file_name"], bin_file)
#trace(data)
for component in data["components"]:
component_name = get_raw(component["component_name"], bin_file)
if "models" in component:
model_index = len(component["models"])
for model in component["models"]:
model_index -= 1
#determine whether or not to export this LOD model based on SETTINGS
export = False
if SETTINGS["highest_lod_only"]:
if model_index == 0:
export = True
else:
export = True
if export:
if SETTINGS["highest_lod_only"] and not SETTINGS[
"lod_labels"]:
end_string = ""
else:
end_string = "_lod" + str(model_index)
file_path = file_path.replace("\\", "/")
obj_path = create_dir(
SETTINGS["obj_path"] + "/" +
file_path[file_path.find("/", 3):file_path.
rfind("/")] + "/" + file_name + "/")
export_obj(
data, model, bin_file,
obj_path + "/" + component_name + end_string)
else:
#no models in this component, only the component header
pass
progress[1] = "_"
except:
trace_error()
return progress
found_materials = []
#EXPORT TEXTURES
try:
num_images = get_raw(data["num_images"], bin_file)
#export textures embedded in this bin group as .png
for image in data["images"]:
#normal gif
export_gif(image, obj_path, bin_file)
#special hidden gif, only seen on isle and isle_hi gifs
if "extra_images" in image:
image["extra_images"][0]["gif_name"] = image["gif_name"]
export_gif(image["extra_images"][0],
obj_path,
bin_file,
pretext="hidden_")
found_materials.append(get_raw(image["gif_name"], bin_file)[:-4])
progress[2] = "_"
except:
trace_error()
return progress
#EXPORT MATERIALS
try:
#export materials without textures as .png, just their rgb on a 4x4 texture
for material in MATERIALS:
if material not in found_materials:
found_materials.append(material)
#write 4x4 png of color c
c = MATERIALS[material]
rows = []
for row in range(4):
rows.append([])
for pixel in range(4):
rows[row].append(c[0])
rows[row].append(c[1])
rows[row].append(c[2])
f = open(obj_path + "/" + material + ".png", "wb")
f.truncate()
w = png.Writer(4, 4)
w.write(f, rows)
f.close()
#statistics for materials
for material in found_materials:
found_duplicate = False
for row in STATS["csv"]["materials"]:
if row[0] == material:
found_duplicate = True
row[2] += 1
if not found_duplicate:
STATS["csv"]["materials"].append([material, "No", 1])
progress[3] = "_"
except:
trace_error()
return progress
#EXPORT MTL FILE
try:
export_mtl(data, obj_path, bin_file)
progress[4] = "_"
except:
trace_error()
return progress
return progress
def main():
args = parser.parse_args()
file_name = args.file.name
bin_file = args.file
data = get_formatted_data(bin_file, "crp", "crp")
name_of_mod = get_raw(data.get("name_of_mod", ""), bin_file)
if name_of_mod == "":
name_of_mod = file_name[:-4]
output_path = os.path.join(args.output_dir, name_of_mod.decode('utf-8'))
if not os.path.exists(output_path):
os.makedirs(output_path)
end_header_offset = get_raw(data["end_header_offset"], bin_file)
metadata = {}
#go through each file found
for file_header in data["file_headers"]:
file_name = get_raw(file_header["file_name"], bin_file).decode('utf-8')
offset_from_header = get_raw(file_header["offset_from_header"],
bin_file)
file_size = get_raw(file_header["file_size"], bin_file)
#absolute_offset = offset_from_header+end_header_offset+1
absolute_offset = offset_from_header + end_header_offset
bin_file.seek(absolute_offset)
try:
id_string = str(unpack(bin_file, "s", 48)).lower()
except:
id_string = ""
#manually search for PNG header in data
png_header = [137, 80, 78, 71, 13, 10, 26, 10]
bin_file.seek(absolute_offset)
found_header = True
for i in range(8):
if unpack(bin_file, "B") != png_header[i]:
found_header = False
#TEXTURE2D
if "unityengine.texture2d" in id_string:
print("found texture2d")
bin_file.seek(absolute_offset)
dds_string = ""
#find "DDS " in the file to mark the start of the dds
while (True):
value = unpack(bin_file, "B")
if value == 68:
dds_string += "D"
elif value == 83:
dds_string += "S"
elif value == 32:
dds_string += " "
else:
dds_string = ""
if dds_string == "DDS ":
dds_offset = bin_file.tell() - 4
meta_offset = absolute_offset
meta_size = dds_offset - absolute_offset
final_path = os.path.join(output_path, file_name + '.dds')
final_offset = dds_offset
final_size = file_size - meta_size
break
#STEAM PREVIEW PNG (AND RANDOM PNGS)
elif "icolossalframework.importers.image" in id_string or found_header:
print("found png")
bin_file.seek(absolute_offset)
png_string = ""
#find "PNG" in the file to mark the start of the png
while (True):
value = unpack(bin_file, "B")
if value == 137:
png_string += "89"
elif value == 80:
png_string += "50"
elif value == 78:
png_string += "4E"
elif value == 71:
png_string += "47"
else:
png_string = ""
if png_string == "89504E47":
png_offset = bin_file.tell() - 4
meta_offset = absolute_offset
meta_size = png_offset - absolute_offset
final_path = os.path.join(output_path, file_name + ".png")
final_offset = png_offset
final_size = file_size - meta_size
break
#GENERIC
else:
print("found generic")
meta_offset = absolute_offset
meta_size = 0
final_path = os.path.join(output_path, file_name)
final_offset = absolute_offset
final_size = file_size
#add metadata to the metadata dictionary
if meta_size == 0:
metadata[final_path] = ""
else:
bin_file.seek(meta_offset)
metadata[final_path] = unpack(bin_file, "s",
meta_size).decode('utf-8', 'ignore')
#write file
write_file = open(final_path, "wb")
write_file.truncate()
bin_file.seek(final_offset)
write_file.write(bin_file.read(final_size))
write_file.close()
#save the metadata dictionary using json
with open(os.path.join(output_path, "metadata.json"), "w") as f:
json.dump(metadata, f, indent=4, sort_keys=True)