def run(versions=[], runBurger=True, runBE=True, runJarExtractor=True):
install_node_jar_extractor()
install_node_burger_extractor()
utils.fetch_manifest()
_vers = utils.extrapolate_versions(versions)
if not has_dir("output/minecraft-data"):
os.makedirs("output/minecraft-data")
print("Extracting", versions, _vers)
for version in _vers:
if version in ('latest', 'snapshot', 'release'):
version = utils.get_latest_version(version)
print(c.BOLD, "Extracting", version, c.RESET)
good = True
if runBurger:
good = good and run_burger(version)
if runBE:
good = good and run_burger_extractor(version)
if runJarExtractor:
good = good and run_prismarine_jar_extractor(version)
if good:
print(c.OKGREEN, f"{version} was successfully extracted", c.RESET)
else:
print(c.FAIL, f"{version} failed one or more extractions", c.RESET)
pass
def clean_binaries_and_plugins(path):
versioned_files = list(filter(
lambda file_name: re.match(Constants.VERSION_REGEX, file_name),
listdir(path)
))
versions = list(map(
lambda file_name: re.search(Constants.VERSION_REGEX, file_name).group(0),
versioned_files
))
latest_version = get_latest_version(versions)
for file_name in versioned_files:
if latest_version not in file_name:
remove_unless_dry_run(path, file_name)
def check_and_clean(path):
files = listdir(path)
if contains_only_files(files, path):
return
directories = filter_out_nones(files)
latest_version = get_latest_version(directories)
if latest_version is None:
clean_recursively(path, directories)
elif len(directories) == 1:
return
else:
print('Update ' + path.split(Constants.M2_PATH)[1])
for directory_name in directories:
if latest_version in directory_name:
continue
print(directory_name + ' (Has newer version: ' + latest_version +
')')
remove_unless_dry_run(path, directory_name)
def train(epochs, batch_size, world_count, sz=64, version_name=None):
cur_dir = os.getcwd()
res_dir = os.path.abspath(os.path.join(cur_dir, '..', 'res'))
all_models_dir = os.path.abspath(os.path.join(cur_dir, '..', 'models'))
model_dir = utils.check_or_create_local_path('animator', all_models_dir)
utils.delete_empty_versions(model_dir, 1)
no_version = version_name is None
if no_version:
latest = utils.get_latest_version(model_dir)
version_name = f'ver{latest + 1}'
version_dir = utils.check_or_create_local_path(version_name, model_dir)
graph_dir = utils.check_or_create_local_path('graph', model_dir)
graph_version_dir = utils.check_or_create_local_path(
version_name, graph_dir)
previews_dir = utils.check_or_create_local_path('previews', version_dir)
model_save_dir = utils.check_or_create_local_path('models', version_dir)
print('Saving source...')
utils.save_source_to_dir(version_dir)
print('Loading minimap values...')
mm_values = utils.load_minimap_values(res_dir)
print('Loading block images...')
block_images = utils.load_block_images(res_dir)
print('Loading encoding dictionaries...')
block_forward, block_backward = utils.load_encoding_dict(
res_dir, 'blocks_optimized')
print('Building model from scratch...')
animator = build_basic_animator(sz)
animator.compile(loss='mse', optimizer='adam')
translator = load_model(
f'{all_models_dir}\\translator\\ver15\\models\\best_loss.h5')
translator.trainable = False
animator_minimap = Sequential()
animator_minimap.add(animator)
animator_minimap.add(translator)
animator_minimap.compile(loss='mse', optimizer='adam')
print('Saving model images...')
keras.utils.plot_model(animator,
to_file=f'{version_dir}\\animator.png',
show_shapes=True,
show_layer_names=True)
print('Loading worlds...')
x_train = load_minimaps(world_count, f'{res_dir}\\worlds\\', (sz, sz),
block_forward, mm_values)
world_count = x_train.shape[0]
batch_cnt = (world_count - (world_count % batch_size)) // batch_size
# Set up tensorboard
print('Setting up tensorboard...')
tb_manager = TensorboardManager(graph_version_dir, batch_cnt)
for epoch in range(epochs):
# Create directories for current epoch
cur_previews_dir = utils.check_or_create_local_path(
f'epoch{epoch}', previews_dir)
cur_models_dir = utils.check_or_create_local_path(
f'epoch{epoch}', model_save_dir)
print('Shuffling data...')
np.random.shuffle(x_train)
for batch in range(batch_cnt):
minimaps = x_train[batch * batch_size:(batch + 1) * batch_size]
# actual = y_train[minibatch_index * batch_size:(minibatch_index + 1) * batch_size]
# Train animator
# world_loss = animator.train_on_batch(minimaps, actual)
minimap_loss = animator_minimap.train_on_batch(minimaps, minimaps)
tb_manager.log_var('mm_loss', epoch, batch, minimap_loss)
print(f"Epoch = {epoch}/{epochs} :: Batch = {batch}/{batch_cnt} "
f":: MMLoss = {minimap_loss}")
# Save previews and models
if batch == batch_cnt - 1:
print('Saving previews...')
worlds = animator.predict(minimaps)
trained = animator_minimap.predict(minimaps)
for i in range(batch_size):
world_decoded = utils.decode_world_sigmoid(
block_backward, worlds[i])
utils.save_world_preview(
block_images, world_decoded,
f'{cur_previews_dir}\\animated{i}.png')
utils.save_world_minimap(
mm_values, world_decoded,
f'{cur_previews_dir}\\actual{i}.png')
utils.save_rgb_map(utils.decode_world_minimap(trained[i]),
f'{cur_previews_dir}\\trained{i}.png')
mm_decoded = utils.decode_world_minimap(minimaps[i])
utils.save_rgb_map(mm_decoded,
f'{cur_previews_dir}\\target{i}.png')
print('Saving models...')
try:
animator.save(f'{cur_models_dir}\\animator.h5')
animator.save_weights(
f'{cur_models_dir}\\animator.weights')
except ImportError:
print('Failed to save data.')
def main(max_rows, cache, name, version, threshold):
print("Download %r" % name)
pd.set_option("max_rows", max_rows)
expire_after = datetime.timedelta(days=cache)
session = init_session(expire_after)
df = download_index(session)
# df_first = df.groupby('name').last()
# df_last = df.groupby('name').first()
if name not in df['name'].values:
print("%r is not a valid library name" % name)
print()
print("Possible library names are:")
names = find_nearest_names(name, threshold=threshold, df=df)
print(names[0:10])
find_nearest_name = names.iloc[0]
if find_nearest_name.score == 1:
print()
new_name = find_nearest_name['name']
print("Case is important! Autofixing it as %r instead of %r" %
(new_name, name))
name = new_name
input("")
else:
print()
sys.exit("Please correct library name!")
df_all_versions = df[df['name'].str.upper() == name.upper()]
name = df_all_versions.name.unique()[0]
print(df_all_versions.set_index('version'))
print("")
versions = get_versions(name, df)
print("versions: %s" % versions.map(str).values)
if version == 'latest':
version = get_latest_version(name, df)
print()
print("version: %s" % str(version))
url = get_archive_url(name, version, df)
print()
print("Downloading from %r" % url)
assert url[-4:] == '.zip', "URL must finish with .zip"
print()
show_licence_from_archive_url(url)
print()
print("(possible) repository url")
print()
try:
repository_url = repository_url_from_archive_url(url)
print(repository_url)
except Exception as e:
print("Can't find repository url")
print()
traceback.print_exc()
def train(epochs, batch_size, world_count, size, version_name=None):
cur_dir = os.getcwd()
res_dir = os.path.abspath(os.path.join(cur_dir, '..', 'res'))
all_models_dir = os.path.abspath(os.path.join(cur_dir, '..', 'models'))
model_dir = utils.check_or_create_local_path('translator', all_models_dir)
utils.delete_empty_versions(model_dir, 1)
no_version = version_name is None
if no_version:
latest = utils.get_latest_version(model_dir)
version_name = f'ver{latest + 1}'
version_dir = utils.check_or_create_local_path(version_name, model_dir)
graph_dir = utils.check_or_create_local_path('graph', model_dir)
graph_version_dir = utils.check_or_create_local_path(
version_name, graph_dir)
model_save_dir = utils.check_or_create_local_path('models', version_dir)
print('Saving source...')
utils.save_source_to_dir(version_dir)
print('Loading minimap values...')
minimap_values = utils.load_minimap_values(res_dir)
print('Loading encoding dictionaries...')
block_forward, block_backward = utils.load_encoding_dict(
res_dir, 'blocks_optimized')
print('Building model from scratch...')
translator = build_translator(size)
translator.compile(optimizer='adam', loss='mse')
print('Loading worlds...')
x_train, y_train = load_worlds_with_minimaps(world_count,
f'{res_dir}\\worlds\\',
(size, size), block_forward,
minimap_values)
best_loss_callback = keras.callbacks.ModelCheckpoint(
f'{model_save_dir}\\best_loss.h5',
verbose=0,
save_best_only=True,
save_weights_only=False,
mode='min',
period=1,
monitor='loss')
# Create callback for automatically saving latest model so training can be resumed. Saves every epoch
latest_h5_callback = keras.callbacks.ModelCheckpoint(
f'{model_save_dir}\\latest.h5',
verbose=0,
save_best_only=False,
save_weights_only=False,
mode='auto',
period=1)
# Create callback for automatically saving latest weights so training can be resumed. Saves every epoch
latest_weights_callback = keras.callbacks.ModelCheckpoint(
f'{model_save_dir}\\latest.weights',
verbose=0,
save_best_only=False,
save_weights_only=True,
mode='auto',
period=1)
# Create callback for tensorboard
tb_callback = keras.callbacks.TensorBoard(log_dir=graph_version_dir,
batch_size=batch_size,
write_graph=False,
write_grads=True)
callback_list = [
latest_h5_callback, latest_weights_callback, best_loss_callback,
tb_callback
]
translator.fit(x_train,
y_train,
batch_size,
epochs,
callbacks=callback_list)
def train(epochs, batch_size, world_count, version_name=None):
cur_dir = os.getcwd()
res_dir = os.path.abspath(os.path.join(cur_dir, '..', 'res'))
all_models_dir = os.path.abspath(os.path.join(cur_dir, '..', 'models'))
model_dir = utils.check_or_create_local_path('auto_encoder',
all_models_dir)
utils.delete_empty_versions(model_dir, 1)
no_version = version_name is None
if no_version:
latest = utils.get_latest_version(model_dir)
version_name = f'ver{latest + 1}'
version_dir = utils.check_or_create_local_path(version_name, model_dir)
graph_dir = utils.check_or_create_local_path('graph', model_dir)
graph_version_dir = utils.check_or_create_local_path(
version_name, graph_dir)
worlds_dir = utils.check_or_create_local_path('worlds', version_dir)
previews_dir = utils.check_or_create_local_path('previews', version_dir)
model_save_dir = utils.check_or_create_local_path('models', version_dir)
latest_epoch = utils.get_latest_epoch(model_save_dir)
initial_epoch = latest_epoch + 1
print('Saving source...')
utils.save_source_to_dir(version_dir)
# Load block images
print('Loading block images...')
block_images = utils.load_block_images(res_dir)
print('Loading encoding dictionaries...')
block_forward, block_backward = utils.load_encoding_dict(
res_dir, 'blocks_optimized')
# Load model and existing weights
print('Loading model...')
# Try to load full model, otherwise try to load weights
loaded_model = False
if not no_version and latest_epoch != -1:
if os.path.exists(
f'{version_dir}\\models\\epoch{latest_epoch}\\autoencoder.h5'):
print('Found models.')
ae = load_model(
f'{version_dir}\\models\\epoch{latest_epoch}\\autoencoder.h5')
loaded_model = True
elif os.path.exists(
f'{version_dir}\\models\\epoch{latest_epoch}\\autoencoder.weights'
):
print('Found weights.')
ae = autoencoder_model(112)
ae.load_weights(
f'{version_dir}\\models\\epoch{latest_epoch}\\autoencoder.weights'
)
print('Compiling model...')
ae_optim = Adam(lr=0.0001)
ae.compile(loss='binary_crossentropy', optimizer=ae_optim)
loaded_model = True
# Model was not loaded, compile new one
if not loaded_model:
print('Compiling model...')
ae = autoencoder_model(112)
print('Compiling model...')
ae_optim = Adam(lr=0.0001)
ae.compile(loss='binary_crossentropy', optimizer=ae_optim)
if no_version:
# Delete existing worlds and previews if any
print('Checking for old generated data...')
utils.delete_files_in_path(worlds_dir)
utils.delete_files_in_path(previews_dir)
print('Saving model images...')
keras.utils.plot_model(ae,
to_file=f'{version_dir}\\autoencoder.png',
show_shapes=True,
show_layer_names=True)
# Load Data
print('Loading worlds...')
x_train = load_worlds(world_count, f'{res_dir}\\worlds\\', (112, 112),
block_forward)
# Start Training loop
world_count = x_train.shape[0]
batch_cnt = (world_count - (world_count % batch_size)) // batch_size
# Set up tensorboard
print('Setting up tensorboard...')
tb_manager = TensorboardManager(graph_version_dir, batch_cnt)
for epoch in range(initial_epoch, epochs):
# Create directories for current epoch
cur_worlds_cur = utils.check_or_create_local_path(
f'epoch{epoch}', worlds_dir)
cur_previews_dir = utils.check_or_create_local_path(
f'epoch{epoch}', previews_dir)
cur_models_dir = utils.check_or_create_local_path(
f'epoch{epoch}', model_save_dir)
print('Shuffling data...')
np.random.shuffle(x_train)
for batch in range(batch_cnt):
# Get real set of images
world_batch = x_train[batch * batch_size:(batch + 1) * batch_size]
# Train
loss = ae.train_on_batch(world_batch, world_batch)
# Save snapshot of generated images on last batch
if batch == batch_cnt - 1:
# Generate samples
generated = ae.predict(world_batch)
# Save samples
for image_num in range(batch_size):
generated_world = generated[image_num]
decoded_world = utils.decode_world_sigmoid(
block_backward, generated_world)
utils.save_world_data(
decoded_world,
f'{cur_worlds_cur}\\world{image_num}.world')
utils.save_world_preview(
block_images, decoded_world,
f'{cur_previews_dir}\\preview{image_num}.png')
# Save actual worlds
for image_num in range(batch_size):
actual_world = world_batch[image_num]
decoded_world = utils.decode_world_sigmoid(
block_backward, actual_world)
utils.save_world_preview(
block_images, decoded_world,
f'{cur_previews_dir}\\actual{image_num}.png')
# Write loss
tb_manager.log_var('ae_loss', epoch, batch, loss)
print(
f'epoch [{epoch}/{epochs}] :: batch [{batch}/{batch_cnt}] :: loss = {loss}'
)
# Save models
if batch % 100 == 99 or batch == batch_cnt - 1:
print('Saving models...')
try:
ae.save(f'{cur_models_dir}\\autoencoder.h5')
ae.save_weights(f'{cur_models_dir}\\autoencoder.weights')
except ImportError:
print('Failed to save data.')
def train(epochs, batch_size, world_count, version_name=None, initial_epoch=0):
cur_dir = os.getcwd()
res_dir = os.path.abspath(os.path.join(cur_dir, '..', 'res'))
all_models_dir = os.path.abspath(os.path.join(cur_dir, '..', 'models'))
model_dir = utils.check_or_create_local_path('helper', all_models_dir)
utils.delete_empty_versions(model_dir, 1)
no_version = version_name is None
if no_version:
latest = utils.get_latest_version(model_dir)
version_name = f'ver{latest + 1}'
version_dir = utils.check_or_create_local_path(version_name, model_dir)
graph_dir = utils.check_or_create_local_path('graph', model_dir)
graph_version_dir = utils.check_or_create_local_path(
version_name, graph_dir)
worlds_dir = utils.check_or_create_local_path('worlds', version_dir)
previews_dir = utils.check_or_create_local_path('previews', version_dir)
model_save_dir = utils.check_or_create_local_path('models', version_dir)
print('Saving source...')
utils.save_source_to_dir(version_dir)
# Load block images
print('Loading block images...')
block_images = utils.load_block_images(res_dir)
print('Loading encoding dictionaries...')
block_forward, block_backward = utils.load_encoding_dict(
res_dir, 'blocks_optimized')
if no_version:
# Delete existing worlds and previews if any
print('Checking for old generated data...')
utils.delete_files_in_path(worlds_dir)
utils.delete_files_in_path(previews_dir)
# Load model and existing weights
print('Loading models...')
judge = build_judge_model(32)
judge_optimizer = Adam(lr=0.0001)
judge.compile(loss='binary_crossentropy',
optimizer=judge_optimizer,
metrics=['accuracy'])
helper_optimizer = Adam(lr=0.001)
helper = build_helper_model(32)
helper_feedback = build_helper_feedback_model(helper, judge, 32)
helper_feedback.compile(loss='binary_crossentropy',
optimizer=helper_optimizer)
# Load Data
print('Loading worlds...')
x_train = load_worlds(world_count, f'{res_dir}\\worlds\\', (32, 32),
block_forward)
# Start Training loop
world_count = x_train.shape[0]
batch_cnt = (world_count - (world_count % batch_size)) // batch_size
tb_manager = TensorboardManager(graph_version_dir, batch_cnt)
for epoch in range(initial_epoch, epochs):
print(f'Epoch = {epoch} ')
# Create directories for current epoch
cur_worlds_cur = utils.check_or_create_local_path(
f'epoch{epoch}', worlds_dir)
cur_previews_dir = utils.check_or_create_local_path(
f'epoch{epoch}', previews_dir)
cur_models_dir = utils.check_or_create_local_path(
f'epoch{epoch}', model_save_dir)
print('Shuffling data...')
np.random.shuffle(x_train)
for batch in range(batch_cnt):
# Get real set of worlds
world_batch = x_train[batch * batch_size:(batch + 1) * batch_size]
world_batch_masked, world_masks = utils.mask_batch_low(world_batch)
world_masks_reshaped = np.reshape(world_masks[:, :, :, 0],
(batch_size, 32 * 32, 1))
# Get fake set of worlds
noise = np.random.normal(0, 1, size=(batch_size, 128))
generated = helper.predict([world_batch_masked, noise])
real_labels = np.ones((batch_size, 32 * 32, 1))
fake_labels = np.zeros((batch_size, 32 * 32, 1))
masked_labels = 1 - world_masks_reshaped
judge.trainable = True
j_real = judge.train_on_batch([world_batch_masked, world_batch],
real_labels)
j_fake = judge.train_on_batch([world_batch_masked, generated[1]],
fake_labels)
tb_manager.log_var('j_loss_real', epoch, batch, j_real[0])
tb_manager.log_var('j_loss_fake', epoch, batch, j_fake[0])
tb_manager.log_var('j_acc_real', epoch, batch, j_real[1])
tb_manager.log_var('j_acc_fake', epoch, batch, j_fake[1])
judge.trainable = False
h_loss = helper_feedback.train_on_batch(
[world_batch_masked, noise], real_labels)
tb_manager.log_var('h_loss', epoch, batch, h_loss)
print(
f'epoch [{epoch}/{epochs}] :: batch [{batch}/{batch_cnt}] :: fake_loss = {j_fake[0]} :: fake_acc = '
f'{j_fake[1]} :: real_loss = {j_real[0]} :: real_acc = {j_real[1]} :: h_loss = {h_loss}'
)
if batch % 1000 == 999 or batch == batch_cnt - 1:
# Save generated batch
for i in range(batch_size):
actual_world = world_batch_masked[i]
a_decoded = utils.decode_world_sigmoid(
block_backward, actual_world)
utils.save_world_preview(
block_images, a_decoded,
f'{cur_previews_dir}\\actual{i}.png')
gen_world = generated[1][i]
decoded = utils.decode_world_sigmoid(
block_backward, gen_world)
utils.save_world_preview(
block_images, decoded,
f'{cur_previews_dir}\\preview{i}.png')
# Save models
try:
judge.save(f'{cur_models_dir}\\judge.h5')
helper.save(f'{cur_models_dir}\\helper.h5')
judge.save_weights(f'{cur_models_dir}\\judge.weights')
helper.save_weights(f'{cur_models_dir}\\helper.weights')
except ImportError:
print('Failed to save data.')
def clean_data_for_ide(path, files, prefix):
ide_versions = to_versions_map(files, prefix)
latest_version = get_latest_version(ide_versions.keys())
for current_version, current_name in ide_versions.items():
if current_version != latest_version:
remove_unless_dry_run(path, current_name)
if __name__ == "__main__":
parser = argparse.ArgumentParser()
parser.add_argument("-c", "--config", required=True, help="Config file path.")
parser.add_argument("-ch", "--checkpoint", required=False, help="Checkpoint file path.", default=None)
parser.add_argument("-d", "--debug", required=False, type=bool, help="Checkpoint file path.", default=False)
args = parser.parse_args()
with open(args.config, 'r') as stream:
configs = yaml.safe_load(stream)
root_dir = configs['log_dir']
if not os.path.isdir(root_dir):
os.makedirs(root_dir)
version = utils.get_latest_version(root_dir)
version_directory = root_dir + "version_" + str(version)
if not os.path.isdir(version_directory):
os.makedirs(version_directory)
weights_directory = version_directory + "/weights/"
if not os.path.isdir(weights_directory):
os.makedirs(weights_directory)
assert bool, type(args.debug)
debug = args.debug
start = datetime.datetime.now()
configs['start'] = start
configs['version'] = version
def decompile_version(version, client=True, ignoreMappings=False):
print("\n-------")
print(c.BOLD, "Decompiling", version, "this will take a while", c.RESET)
failedBecauseNoMappings = False
# If you crash here, you set an invalid version to decompile
if version == "latest" or version == "snapshot":
version = utils.get_latest_version(typ)
else:
dn = utils.get_date_for_version(version)
# We have Mojang mappings since 1.14.4
# For both 1.14.4 and 1.15, we include both MCP and Mojang mappings (this just makes it easier to diff)
D = "./DecompilerMC"
# DecompilerMC spits out alot of input()s which are annoying, --quiet removes them. Unfortuantely this
# also removes some useful debugging info, so run this script with --verbose to get all the messages.
quiet = '' if verbose else '--quiet'
# uses MCP mappings
isLegacy = False
if utils.mc_version_cmp(version, '1.14.4') < 0:
print(f"{version} < 1.14.4 -- legacy version so using MCP mappings")
isLegacy = True
if isLegacy:
mapping = get_mcp_mappings_for(version)
if not mapping:
print("No MCP mappings for", version)
failedBecauseNoMappings = True
if failedBecauseNoMappings or (utils.mc_version_cmp(version, '1.7.10') <
0):
if not ignoreMappings:
print(
c.FAIL,
f"{version}: no mapping data! Skipping because ignoreMappings is false.",
c.RESET)
return
print(
f"{version}: no mapping data! JAR will only be decompiled. Abort now if you expected deobfuscation."
)
input("Press enter to continue.")
# do not remap and basically just run fernflower
s = '' if client else '--side server'
l = f"python3 {D}/main.py -mcv {version} -na -rmap false --download_mapping false -dj true -rjar false -dec true -dd false -d fernflower {s}"
print("> ", l)
os.system(l)
return
# MCP mappings
if isLegacy or (utils.mc_version_cmp(version, '1.15')
== 0) or (utils.mc_version_cmp(version, '1.14.4') == 0):
mapping = get_mcp_mappings_for(version)
print(f'{version}: Using MCP mappings at', mapping)
os.chdir(D)
s = '' if client else '--side server'
l = f"python3 main.py -mcv {version} -na -rmap false --download_mapping false -dj true -rjar true -dec true -dd false -d fernflower {s} -m ../{mapping} {quiet}"
print(c.WARNING, ">", l, c.RESET)
ret = os.system(l)
if ret:
raise "Decompile failed - non-zero exit code"
print(c.WARNING, f"> move {D}/src/{version} {D}/src/{version}_mcp/",
c.RESET)
utils.move(f"src/{version}/", f"src/{version}_mcp/")
print(
c.OKGREEN,
f"{version} was successfully decompiled to {D}/src/{version}_mcp/",
c.RESET)
os.chdir('..')
# Mojang mappings
if not isLegacy:
print(f"{version}: Using Mojang mappings")
os.chdir(D)
s = '' if client else '--side server'
l = f"python3 main.py -mcv {version} -d fernflower {quiet} {s}"
print(c.WARNING, "> ", l, c.RESET)
os.system(l)
utils.move(f"src/{version}/", f"src/{version}_mojang/")
print(
c.OKGREEN,
f"{version} was successfully decompiled to {D}/src/{version}_mojang/",
c.RESET)
os.chdir('..')
def train(epochs,
batch_size,
world_count,
dict_src_name,
version_name=None,
initial_epoch=0):
cur_dir = os.getcwd()
res_dir = os.path.abspath(os.path.join(cur_dir, '..', 'res'))
all_models_dir = os.path.abspath(os.path.join(cur_dir, '..', 'models'))
model_dir = utils.check_or_create_local_path('pro_classifier',
all_models_dir)
utils.delete_empty_versions(model_dir, 1)
no_version = version_name is None
if no_version:
latest = utils.get_latest_version(model_dir)
version_name = f'ver{latest + 1}'
version_dir = utils.check_or_create_local_path(version_name, model_dir)
graph_dir = utils.check_or_create_local_path('graph', model_dir)
graph_version_dir = utils.check_or_create_local_path(
version_name, graph_dir)
model_save_dir = utils.check_or_create_local_path('models', version_dir)
print('Saving source...')
utils.save_source_to_dir(version_dir)
print('Loading encoding dictionaries...')
block_forward, block_backward = utils.load_encoding_dict(
res_dir, 'blocks_optimized')
print('Building model from scratch...')
c_optim = Adam(lr=0.0001)
size = 64
c = build_classifier(size)
# c = build_resnet50(1)
# c = build_wide_resnet(input_dim=(size, size, 10), nb_classes=1, N=2, k=1, dropout=0.1)
c.summary()
c.compile(loss='binary_crossentropy',
optimizer=c_optim,
metrics=['accuracy'])
print('Loading labels...')
label_dict = utils.load_label_dict(res_dir, dict_src_name)
print('Loading worlds...')
x, y_raw = load_worlds_with_labels(world_count, f'{res_dir}\\worlds\\',
label_dict, (size, size), block_forward)
y = utils.convert_labels_binary(y_raw, epsilon=0)
# Create callback for automatically saving best model based on highest regular accuracy
check_best_acc = keras.callbacks.ModelCheckpoint(
f'{model_save_dir}\\best_acc.h5',
monitor='acc',
verbose=0,
save_best_only=True,
save_weights_only=False,
mode='max',
period=1)
# Create callback for automatically saving latest model so training can be resumed. Saves every epoch
latest_h5_callback = keras.callbacks.ModelCheckpoint(
f'{model_save_dir}\\latest.h5',
verbose=0,
save_best_only=False,
save_weights_only=False,
mode='auto',
period=1)
# Create callback for automatically saving latest weights so training can be resumed. Saves every epoch
latest_weights_callback = keras.callbacks.ModelCheckpoint(
f'{model_save_dir}\\latest.weights',
verbose=0,
save_best_only=False,
save_weights_only=True,
mode='auto',
period=1)
# Create callback for tensorboard
tb_callback = keras.callbacks.TensorBoard(log_dir=graph_version_dir,
batch_size=batch_size,
write_graph=False,
write_grads=True)
callback_list = [
check_best_acc, latest_h5_callback, latest_weights_callback,
tb_callback
]
# Train model
c.fit(x,
y,
batch_size,
epochs,
initial_epoch=initial_epoch,
callbacks=callback_list,
validation_split=0.2)
def train(epochs,
batch_size,
world_count,
latent_dim,
version_name=None,
initial_epoch=0):
cur_dir = os.getcwd()
res_dir = os.path.abspath(os.path.join(cur_dir, '..', 'res'))
all_models_dir = os.path.abspath(os.path.join(cur_dir, '..', 'models'))
model_dir = utils.check_or_create_local_path('gan', all_models_dir)
utils.delete_empty_versions(model_dir, 1)
no_version = version_name is None
if no_version:
latest = utils.get_latest_version(model_dir)
version_name = f'ver{latest + 1}'
version_dir = utils.check_or_create_local_path(version_name, model_dir)
graph_dir = utils.check_or_create_local_path('graph', model_dir)
graph_version_dir = utils.check_or_create_local_path(
version_name, graph_dir)
worlds_dir = utils.check_or_create_local_path('worlds', version_dir)
previews_dir = utils.check_or_create_local_path('previews', version_dir)
model_save_dir = utils.check_or_create_local_path('models', version_dir)
print('Saving source...')
utils.save_source_to_dir(version_dir)
print('Loading block images...')
block_images = utils.load_block_images(res_dir)
print('Loading encoding dictionaries...')
block_forward, block_backward = utils.load_encoding_dict(
res_dir, 'blocks_optimized')
# Load model and existing weights
print('Loading model...')
# Try to load full model, otherwise try to load weights
size = 64
cur_models = f'{model_save_dir}\\epoch{initial_epoch - 1}'
if os.path.exists(f'{cur_models}\\discriminator.h5') and os.path.exists(
f'{cur_models}\\generator.h5'):
print('Building model from files...')
d = load_model(f'{cur_models}\\discriminator.h5')
g = load_model(f'{cur_models}\\generator.h5')
if os.path.exists(f'{cur_models}\\d_g.h5'):
d_on_g = load_model(f'{cur_models}\\d_g.h5')
else:
g_optim = Adam(lr=0.0001, beta_1=0.5)
d_on_g = generator_containing_discriminator(g, d)
d_on_g.compile(loss='binary_crossentropy', optimizer=g_optim)
elif os.path.exists(
f'{cur_models}\\discriminator.weights') and os.path.exists(
f'{cur_models}\\generator.weights'):
print('Building model with weights...')
d_optim = Adam(lr=0.00001)
d = build_discriminator(size)
d.load_weights(f'{cur_models}\\discriminator.weights')
d.compile(loss='binary_crossentropy',
optimizer=d_optim,
metrics=['accuracy'])
g = build_generator(size)
g.load_weights(f'{cur_models}\\generator.weights')
g_optim = Adam(lr=0.0001, beta_1=0.5)
d_on_g = generator_containing_discriminator(g, d)
d_on_g.compile(loss='binary_crossentropy', optimizer=g_optim)
else:
print('Building model from scratch...')
d_optim = Adam(lr=0.00001)
g_optim = Adam(lr=0.0001, beta_1=0.5)
d = build_discriminator(size)
d.compile(loss='binary_crossentropy',
optimizer=d_optim,
metrics=['accuracy'])
d.summary()
g = build_generator(size, latent_dim)
g.summary()
d_on_g = generator_containing_discriminator(g, d)
d_on_g.compile(loss='binary_crossentropy', optimizer=g_optim)
if no_version:
# Delete existing worlds and previews if any
print('Checking for old generated data...')
utils.delete_files_in_path(worlds_dir)
utils.delete_files_in_path(previews_dir)
print('Saving model images...')
keras.utils.plot_model(d,
to_file=f'{version_dir}\\discriminator.png',
show_shapes=True,
show_layer_names=True)
keras.utils.plot_model(g,
to_file=f'{version_dir}\\generator.png',
show_shapes=True,
show_layer_names=True)
# Load Data
print('Loading worlds...')
label_dict = utils.load_label_dict(res_dir, 'pro_labels_b')
x_train = load_worlds_with_label(world_count,
f'{res_dir}\\worlds\\',
label_dict,
1, (size, size),
block_forward,
overlap_x=0.1,
overlap_y=0.1)
world_count = x_train.shape[0]
batch_cnt = (world_count - (world_count % batch_size)) // batch_size
# Set up tensorboard
print('Setting up tensorboard...')
tb_manager = TensorboardManager(graph_version_dir, batch_cnt)
preview_frequency_sec = 5 * 60.0
for epoch in range(initial_epoch, epochs):
# Create directories for current epoch
cur_worlds_dir = utils.check_or_create_local_path(
f'epoch{epoch}', worlds_dir)
cur_previews_dir = utils.check_or_create_local_path(
f'epoch{epoch}', previews_dir)
cur_models_dir = utils.check_or_create_local_path(
f'epoch{epoch}', model_save_dir)
print('Shuffling data...')
np.random.shuffle(x_train)
last_save_time = time.time()
for batch in range(batch_cnt):
# Get real set of images
real_worlds = x_train[batch * batch_size:(batch + 1) * batch_size]
# Get fake set of images
noise = np.random.normal(0, 1, size=(batch_size, latent_dim))
fake_worlds = g.predict(noise)
real_labels = np.ones(
(batch_size,
1)) # np.random.uniform(0.9, 1.1, size=(batch_size,))
fake_labels = np.zeros(
(batch_size,
1)) # np.random.uniform(-0.1, 0.1, size=(batch_size,))
# Train discriminator on real worlds
d.trainable = True
d_loss = d.train_on_batch(real_worlds, real_labels)
acc_real = d_loss[1]
loss_real = d_loss[0]
tb_manager.log_var('d_acc_real', epoch, batch, d_loss[1])
tb_manager.log_var('d_loss_real', epoch, batch, d_loss[0])
# Train discriminator on fake worlds
d_loss = d.train_on_batch(fake_worlds, fake_labels)
d.trainable = False
acc_fake = d_loss[1]
loss_fake = d_loss[0]
tb_manager.log_var('d_acc_fake', epoch, batch, d_loss[1])
tb_manager.log_var('d_loss_fake', epoch, batch, d_loss[0])
# Training generator on X data, with Y labels
# noise = np.random.normal(0, 1, (batch_size, 256))
# Train generator to generate real
g_loss = d_on_g.train_on_batch(noise, real_labels)
tb_manager.log_var('g_loss', epoch, batch, g_loss)
print(
f'epoch [{epoch}/{epochs}] :: batch [{batch}/{batch_cnt}] :: fake_acc = {acc_fake} :: '
f'real_acc = {acc_real} :: fake_loss = {loss_fake} :: real_loss = {loss_real} :: gen_loss = {g_loss}'
)
# Save models
time_since_save = time.time() - last_save_time
if time_since_save >= preview_frequency_sec or batch == batch_cnt - 1:
print('Saving previews...')
for i in range(batch_size):
generated_world = fake_worlds[i]
decoded_world = utils.decode_world_sigmoid(
block_backward, generated_world)
utils.save_world_data(decoded_world,
f'{cur_worlds_dir}\\world{i}.world')
utils.save_world_preview(
block_images, decoded_world,
f'{cur_previews_dir}\\preview{i}.png')
print('Saving models...')
try:
d.save(f'{cur_models_dir}\\discriminator.h5')
g.save(f'{cur_models_dir}\\generator.h5')
d_on_g.save(f'{cur_models_dir}\\d_g.h5')
d.save_weights(f'{cur_models_dir}\\discriminator.weights')
g.save_weights(f'{cur_models_dir}\\generator.weights')
d_on_g.save_weights(f'{cur_models_dir}\\d_g.weights')
except ImportError:
print('Failed to save data.')
last_save_time = time.time()
def train(epochs, batch_size, world_count, version_name=None, initial_epoch=0):
cur_dir = os.getcwd()
res_dir = os.path.abspath(os.path.join(cur_dir, '..', 'res'))
all_models_dir = os.path.abspath(os.path.join(cur_dir, '..', 'models'))
model_dir = utils.check_or_create_local_path('inpainting', all_models_dir)
utils.delete_empty_versions(model_dir, 0)
no_version = version_name is None
if no_version:
latest = utils.get_latest_version(model_dir)
version_name = f'ver{latest}'
version_dir = utils.check_or_create_local_path(version_name, model_dir)
graph_dir = utils.check_or_create_local_path('graph', model_dir)
graph_version_dir = utils.check_or_create_local_path(version_name, graph_dir)
worlds_dir = utils.check_or_create_local_path('worlds', version_dir)
previews_dir = utils.check_or_create_local_path('previews', version_dir)
model_save_dir = utils.check_or_create_local_path('models', version_dir)
print('Saving source...')
utils.save_source_to_dir(version_dir)
# Load block images
print('Loading block images...')
block_images = utils.load_block_images(res_dir)
print('Loading encoding dictionaries...')
block_forward, block_backward = utils.load_encoding_dict(res_dir, 'blocks_optimized')
# Load model
print('Loading model...')
feature_model = auto_encoder.autoencoder_model()
feature_model.load_weights(f'{all_models_dir}\\auto_encoder\\ver15\\models\\epoch28\\autoencoder.weights')
feature_layers = [7, 14, 21]
contextnet = PConvUnet(feature_model, feature_layers, inference_only=False)
unet = contextnet.build_pconv_unet(train_bn=True, lr=0.0001)
unet.summary()
# pconv_unet.load_weights(f'{contextnet_dir}\\ver43\\models\\epoch4\\unet.weights')
if no_version:
# Delete existing worlds and previews if any
print('Checking for old generated data...')
utils.delete_files_in_path(worlds_dir)
utils.delete_files_in_path(previews_dir)
print('Saving model images...')
keras.utils.plot_model(unet, to_file=f'{version_dir}\\unet.png', show_shapes=True,
show_layer_names=True)
# Set up tensorboard
print('Setting up tensorboard...')
tb_writer = tf.summary.FileWriter(logdir=graph_version_dir)
unet_loss_summary = tf.Summary()
unet_loss_summary.value.add(tag='unet_loss', simple_value=None)
# Load Data
x_train = load_worlds(world_count, f'{res_dir}\\worlds\\', (128, 128), block_forward)
# Start Training loop
world_count = x_train.shape[0]
batch_cnt = (world_count - (world_count % batch_size)) // batch_size
for epoch in range(initial_epoch, epochs):
print(f'Epoch = {epoch}')
# Create directories for current epoch
cur_worlds_cur = utils.check_or_create_local_path(f'epoch{epoch}', worlds_dir)
cur_previews_dir = utils.check_or_create_local_path(f'epoch{epoch}', previews_dir)
cur_models_dir = utils.check_or_create_local_path(f'epoch{epoch}', model_save_dir)
print('Shuffling data...')
np.random.shuffle(x_train)
for batch in range(batch_cnt):
# Get real set of images
world_batch = x_train[batch * batch_size:(batch + 1) * batch_size]
world_batch_masked, world_masks = utils.mask_batch_high(world_batch)
if batch % 1000 == 999 or batch == batch_cnt - 1:
# Save model
try:
unet.save(f'{cur_models_dir}\\unet.h5')
unet.save_weights(f'{cur_models_dir}\\unet.weights')
except ImportError:
print('Failed to save data.')
# Save previews
test = unet.predict([world_batch_masked, world_masks])
d0 = utils.decode_world_sigmoid(block_backward, world_batch[0])
utils.save_world_preview(block_images, d0, f'{cur_previews_dir}\\{batch}_orig.png')
d1 = utils.decode_world_sigmoid(block_backward, test[0])
utils.save_world_preview(block_images, d1, f'{cur_previews_dir}\\{batch}_fixed.png')
d2 = utils.decode_world_sigmoid(block_backward, world_batch_masked[0])
utils.save_world_preview(block_images, d2, f'{cur_previews_dir}\\{batch}_masked.png')
loss = unet.train_on_batch([world_batch_masked, world_masks], world_batch)
unet_loss_summary.value[0].simple_value = loss / 1000.0 # Divide by 1000 for better Y-Axis values
tb_writer.add_summary(unet_loss_summary, (epoch * batch_cnt) + batch)
tb_writer.flush()
print(f'epoch [{epoch}/{epochs}] :: batch [{batch}/{batch_cnt}] :: unet_loss = {loss}')