def train(self):
if self.len == 5:
self.model = self.build_model()
if os.path.exists('poetry_models.h5'):
self.model.load_weights('poetry_models.h5')
self.model.fit_generator(
generator=self.generate_poem(),
verbose=True,
steps_per_epoch=128,
epochs=self.epoch,
callbacks=[
ModelCheckpoint('poetry_models.h5',
save_weights_only=True),
LambdaCallback(on_epoch_end=self.get_sample)
])
else:
self.model = self.build_model_7()
if os.path.exists('poetry_models7.h5'):
self.model.load_weights('poetry_models7.h5')
self.model.fit_generator(
generator=self.generate_poem(),
verbose=True,
steps_per_epoch=128,
epochs=self.epoch,
callbacks=[
ModelCheckpoint('poetry_models7.h5',
save_weights_only=True),
LambdaCallback(on_epoch_end=self.get_sample)
])
def train(self, ):
tbCallBack = TensorBoard(log_dir='./mtltd_lstm_logs',
histogram_freq=4,
write_graph=True,
write_images=True)
def modelSave(epoch, logs):
if (epoch + 1) % 4 == 0:
currentDT = datetime.datetime.now()
model_name = './models/single_task_model_' + currentDT.strftime(
"%Y_%m_%d_%H_%m") + '.h5'
self.model.save(model_name)
print("Model is saved")
msCallBack = LambdaCallback(on_epoch_end=modelSave)
# texts_raw_indices, texts_raw_without_aspects_indices, texts_left_indices, texts_left_with_aspects_indices, \
# aspects_indices, texts_right_indices, texts_right_with_aspects_indices, dataset_index, \
# polarities_matrix = \
# read_dataset(types=self.DATASET,
# mode='validate',
# embedding_dim=self.EMBEDDING_DIM,
# max_seq_len=self.MAX_SENTENCE_LENGTH, max_aspect_len=self.MAX_ASPECT_LENGTH)
self.model.fit([self.texts_left_indices, self.texts_right_indices],
self.polarities,
validation_split=0.1,
shuffle=True,
epochs=self.EPOCHS,
batch_size=self.BATCH_SIZE,
verbose=2,
callbacks=[tbCallBack, msCallBack])
def train(self):
tbCallBack = TensorBoard(log_dir='./lstm_logs',
histogram_freq=0,
write_graph=True,
write_images=True)
def modelSave(epoch, logs):
if (epoch + 1) % 5 == 0:
self.model.save('lstm_saved_model.h5')
msCallBack = LambdaCallback(on_epoch_end=modelSave)
texts_raw_indices, texts_raw_without_aspects_indices, texts_left_indices, texts_left_with_aspects_indices, \
aspects_indices, texts_right_indices, texts_right_with_aspects_indices, \
polarities_matrix = \
read_dataset(type=self.DATASET,
mode='test',
embedding_dim=self.EMBEDDING_DIM,
max_seq_len=self.MAX_SEQUENCE_LENGTH, max_aspect_len=self.MAX_ASPECT_LENGTH)
self.model.fit(self.texts_raw_indices,
self.polarities_matrix,
validation_data=(texts_raw_indices, polarities_matrix),
epochs=self.EPOCHS,
batch_size=self.BATCH_SIZE,
callbacks=[tbCallBack])
def __attrs_post_init__(self):
self.weights_history_callback = LambdaCallback(
on_epoch_end=self.add_weight_history)
if self.model == 0:
l_rate = Globals().vp().nn_l_rate
layers = Globals().vp().layers
activation = 'relu'
self.model = self._build_model(layers=layers,
activation=activation,
l_rate=l_rate)
def perform_training(self, mf: ModelFactory,
tuned_model_params: Dict[str, Union[int, float]],
seq: FileSystemSequence,
discovery_path: str) -> Tuple[Model, RunStats]:
log_dir = os.path.join(
self.data_path, f'logs{os.sep}fit{os.sep}',
f'{mf.model_name}_{datetime.now().strftime("%Y%m%d-%H%M%S")}')
os.makedirs(log_dir)
run_stats_path = os.path.join(log_dir, 'run_stats.json')
model_weights_path = os.path.join(log_dir, f'{mf.model_name}_weights')
model_path = os.path.join(log_dir, 'model')
file_writer = tf.summary.create_file_writer(
os.path.join(log_dir, 'validation'))
file_writer.set_as_default()
epochs = 10
model = mf.construct_model(tuned_model_params)
val_cat_acc: List[float] = []
def record_validation(epoch, logs):
seq.set_dataset_type(DatasetType.CROSS_VAL)
results = model.evaluate(seq)
seq.set_dataset_type(DatasetType.TRAINING)
logs['val_loss'] = results[0]
logs['val_categorical_accuracy'] = results[1]
tf.summary.scalar('val_loss', data=results[0], step=epoch)
tf.summary.scalar('val_categorical_accuracy',
data=results[1],
step=epoch)
current_best = max(val_cat_acc) if len(val_cat_acc) > 0 else 0
if current_best < results[1]:
print(f'Saving model weights on epoch {epoch} to {model_path}')
model.save_weights(model_weights_path)
model.save(model_path.replace('/', '\\'), save_format='h5')
val_cat_acc.append(results[1])
callbacks = [
LambdaCallback(on_epoch_end=record_validation),
tf.keras.callbacks.TensorBoard(log_dir=log_dir, histogram_freq=1),
]
print(
f'Started training {mf.model_name} with {seq.get_training_range()[1]} examples'
)
fit = model.fit(seq, epochs=epochs, callbacks=callbacks)
stats = RunStats(mf.model_name, model, seq, epochs, self.batch_size,
tuned_model_params,
mf.get_model_params(tuned_model_params), fit.history,
discovery_path, model_weights_path)
with open(run_stats_path, 'w') as stream:
stream.write(jsonpickle.encode(stats))
print(run_stats_path)
print('Finished')
return model, stats
def main():
df = pd.read_csv("./data/kk_charas.csv", index_col=0)
df = category_to_onehot(df)
df = df.clip(0, 1)
train = df.values
vae = VariationalAutoEncoder(df.shape[1], 800)
encoder = vae.build_encoder()
decoder = vae.build_decoder()
model = vae.build_vae(encoder, decoder)
vae.model_compile(model)
t = datetime.datetime.now().strftime("%Y%m%d_%H%M")
dirpath = os.path.join("./vae_models", t)
if not os.path.exists(dirpath):
os.makedirs(dirpath)
os.makedirs(os.path.join(dirpath, "weights"))
os.makedirs(os.path.join(dirpath, "models"))
def save_model(epoch, logs):
if epoch % 5 != 0:
return
filename = "{:03}_%s.%s".format(epoch)
filepath = os.path.join(dirpath, "models", filename)
with open(filepath % ("encoder", "json"), "w+") as f:
f.write(encoder.to_json(indent=2))
with open(filepath % ("decoder", "json"), "w+") as f:
f.write(decoder.to_json(indent=2))
encoder.save_weights(filepath % ("encoder", "h5"))
decoder.save_weights(filepath % ("decoder", "h5"))
filename = "{:03}.json".format(epoch)
filepath = os.path.join(dirpath, "weights", filename)
with open(filepath, "w+") as f:
w, b = decoder.layers[3].get_weights()
layer_params = {"weights": w.tolist(), "bias": b.tolist()}
json.dump(layer_params, f, indent=2)
save_model_callback = LambdaCallback(on_epoch_end=save_model)
model.fit(train,
train,
epochs=100,
batch_size=200,
callbacks=[save_model_callback])
save_model(100, None)
def train(self):
# tbCallBack = TensorBoard(log_dir='./ram_logs', histogram_freq=0, write_graph=True, write_images=True)
def modelSave(epoch, logs):
if (epoch + 1) % 5 == 0:
self.model.save('lstm_saved_model.h5')
msCallBack = LambdaCallback(on_epoch_end=modelSave)
texts_raw_indices, texts_raw_without_aspects_indices, texts_left_indices, texts_left_with_aspects_indices, \
aspects_indices, texts_right_indices, texts_right_with_aspects_indices, \
polarities_matrix = \
read_dataset(type=self.DATASET,
mode='test',
embedding_dim=self.EMBEDDING_DIM,
max_seq_len=self.MAX_SEQUENCE_LENGTH, max_aspect_len=self.MAX_ASPECT_LENGTH)
self.model.fit([self.texts_raw_indices, self.aspects_indices], self.polarities_matrix,
epochs=self.EPOCHS, batch_size=self.BATCH_SIZE,callbacks=[msCallBack])
scores = self.model.evaluate([texts_raw_indices, aspects_indices], polarities_matrix, verbose=0)
print("Loss :", scores[0], "Accuracy", scores[1]*100)
def train(self, X, y):
tbCallBack = TensorBoard(log_dir='./mtltd_lstm_logs',
histogram_freq=4,
write_graph=True,
write_images=True)
def modelSave(epoch, logs):
if (epoch) % 4 == 0:
currentDT = datetime.datetime.now()
model_name = './models/mtl_absa_saved_model_' + currentDT.strftime(
"%Y_%m_%d_%H_%M") + '.h5'
self.model.save(model_name)
print("Model is saved")
# weightsAndBiases_left = self.model.layers[3].get_weights()
# weightsAndBiases_right = self.model.layers[4].get_weights()
# layer3 = './weights/mtl_absa_saved_model_3_'+currentDT.strftime("%Y_%m_%d_%H_%M")+'.pkl'
# layer4 = './weights/mtl_absa_saved_model_4_' + currentDT.strftime("%Y_%m_%d_%H_%M") + '.pkl'
# np.save(layer3,arr=weightsAndBiases_left)
# np.save(layer4,arr=weightsAndBiases_right)
msCallBack = LambdaCallback(on_epoch_end=modelSave)
# texts_raw_indices, texts_raw_without_aspects_indices, texts_left_indices, texts_left_with_aspects_indices, \
# aspects_indices, texts_right_indices, texts_right_with_aspects_indices, dataset_index, \
# polarities_matrix = \
# read_dataset(types=self.DATASET,
# mode='validate',
# embedding_dim=self.EMBEDDING_DIM,
# max_seq_len=self.MAX_SENTENCE_LENGTH, max_aspect_len=self.MAX_ASPECT_LENGTH)
self.model.fit(X, {
'twitter_output': y,
'rest_output': y,
'general_output': y
},
validation_split=0.1,
shuffle=True,
epochs=self.EPOCHS,
batch_size=self.BATCH_SIZE,
callbacks=[msCallBack],
verbose=0)
def save_predict_image(test_img, exp_dir, model):
def save_predict(image, logdir, model, file_writer_image):
def callback(epoch, logs):
im = tf.expand_dims(image, axis=0)
if epoch <= 1:
with file_writer_image.as_default():
tf.summary.image("Origin", im, step=epoch)
# tf.print('before predict', tf.shape(im))
predict = model.predict(im)
# tf.print('after predict', tf.shape(predict))
# predict = tf.argmax(predict, axis=-1)
predict = VOCColormap(21).colorize(predict)
with file_writer_image.as_default():
tf.summary.image("Predict", predict, step=epoch)
return callback
file_writer_image = tf.summary.create_file_writer(exp_dir + '/image')
return LambdaCallback(
on_epoch_end=save_predict(test_img, exp_dir, model, file_writer_image))
def train(self, npt_exp):
directory = "muscle-formation-diff/data/images/train"
if not path.exists(directory):
directory = "../../data/images/train"
steps = len(listdir(directory)) // self.batch_size
callbacks = NeptuneCallback(neptune_experiment=npt_exp,
n_batch=steps,
images=self.x_test[:20],
img_size=self.img_size)
tf.config.experimental_run_functions_eagerly(True)
# self.vae.add_loss(self.kl_reconstruction_loss)
self.vae.compile(optimizer='adam',
loss=self.vae_loss_function,
metrics=['accuracy'])
train_generator, validation_generator, test_generator = get_generators(
)
history = self.vae.fit(
train_generator,
steps_per_epoch=len(train_generator),
validation_data=validation_generator,
validation_steps=len(validation_generator),
epochs=PARAMS['n_epochs'],
shuffle=PARAMS['shuffle'],
callbacks=[
callbacks,
LambdaCallback(
on_epoch_end=lambda epoch, logs: log_data(logs)),
EarlyStopping(patience=PARAMS['early_stopping'],
monitor='loss',
restore_best_weights=True)
])
# self.vae.compile(optimizer=self.optimizer, loss=self.loss_func)
return self.vae, history
def main(hparams_dict):
# hparams_dict ={ \
# 'epochs' : 5, \
# 'rnn_type' : 'LSTM', \
# 'rnn_size' : 8, \
# 'learning_rate' : 1e-3, \
# 'batch_size' : 64, \
# 'sequence_length' : 24 * 7 * 1, \
# 'steps_per_epoch' : 2, \
# 'layers' : 1, \
# 'dropout' : 0.5, \
# 'warmup_steps' : 50, \
# 'optimizer_type' : 'RMSprop', \
# 'weight_initialization' : False}
epochs = hparams_dict['epochs']
rnn_type = hparams_dict['rnn_type']
rnn_size = hparams_dict['rnn_size']
learning_rate = hparams_dict['learning_rate']
batch_size = hparams_dict['batch_size']
sequence_length = hparams_dict['sequence_length']
steps_per_epoch = hparams_dict['steps_per_epoch']
layers = hparams_dict['layers']
dropout = hparams_dict['dropout']
warmup_steps = hparams_dict['warmup_steps']
optimizer_type = hparams_dict['optimizer_type']
weight_initialization = hparams_dict['weight_initialization']
model_name = '{}_Multi-{}_epoch-{}'.format(
rnn_type, len(target_names), str(epochs))
# creates a generator object from the batch generator
generator = batch_generator(
batch_size=batch_size, sequence_length=sequence_length)
# converts the 1 dimensional vector to 2 dimensional e.g. (2,) >>> (2,1)
validation_data = (np.expand_dims(x_test_scaled, axis=0),
np.expand_dims(y_test_scaled, axis=0))
# creates the Neural Model with keras
model = Sequential()
for layer in range(layers):
if rnn_type == 'LSTM':
model.add(LSTM(units=rnn_size,
return_sequences=True,
input_shape=(None, x_features,)))
if rnn_type == 'GRU':
model.add(GRU(units=rnn_size,
return_sequences=True,
input_shape=(None, x_features,)))
model.add(Dropout(dropout))
dense_layer = Dense(y_features, activation='sigmoid')
model.add(dense_layer)
#Uses a linear activation in case of using weight_initialization
if weight_initialization:
from tensorflow.python.keras.initializers import RandomUniform
# Maybe use lower init-ranges.
init = RandomUniform(minval=-0.05, maxval=0.05)
model.add(Dense(y_features,
activation='linear',
kernel_initializer=init))
# Decides on the type of optimizer based on HPs
if optimizer_type == 'RMSprop':
optimizer = RMSprop(lr=learning_rate)
if optimizer_type == 'Adagrad':
optimizer = Adagrad(lr=learning_rate)
if optimizer_type == 'SGD':
optimizer = SGD(lr=learning_rate)
# compiles the model
model.compile(loss=mse_loss, optimizer=optimizer)
print(model.summary())
# callbacks for save the checkpoint, earlyStopping, the tensorboard log, redusing lr, and a csv file
path_checkpoint = model_name + '.keras'
callback_checkpoint = ModelCheckpoint(filepath=path_checkpoint,
monitor='val_loss',
verbose=1,
save_weights_only=True,
save_best_only=True)
callback_early_stopping = EarlyStopping(monitor='val_loss',
patience=5, verbose=1)
callback_tensorboard = TensorBoard(log_dir='./' + model_name + '_logs/',
histogram_freq=0,
write_graph=False)
callback_reduce_lr = ReduceLROnPlateau(monitor='val_loss',
factor=0.1,
min_lr=1e-4,
patience=0,
verbose=1)
callback_csv_log = CSVLogger('training_log.log')
# saves all the metrics in a list
metrics_history = []
metrics_callback = LambdaCallback(
on_epoch_end=lambda epoch, logs: metrics_history.append(
[logs['loss'], logs['val_loss']])
)
callbacks = [ # callback_early_stopping,
callback_checkpoint,
callback_tensorboard,
callback_reduce_lr,
# callback_csv_log,
metrics_callback
]
# trains the compiled model
model.fit_generator(generator=generator,
epochs=epochs,
steps_per_epoch=steps_per_epoch,
validation_data=validation_data,
callbacks=callbacks)
# in case of testing pretrained model
try:
model.load_weights(path_checkpoint)
except Exception as error:
print("Error trying to load checkpoint.")
print(error)
# evaluates the model on the test data
result = model.evaluate(x=np.expand_dims(x_test_scaled, axis=0),
y=np.expand_dims(y_test_scaled, axis=0))
# stores the training and validation losses in two separate lists
train_loss = [value[0] for value in metrics_history]
val_loss = [value[1] for value in metrics_history]
metrics_history = {'train_loss': train_loss, 'val_loss': val_loss}
return metrics_history
with open('.cnn_labels', 'w') as fw:
json.dump(train_generator.class_indices, fw, indent=4)
# モデルを読み込む
model = cnn_model(train_generator.class_indices)
model.summary()
if os.path.exists(model_path):
model.load_weights(model_path + 'w', by_name=False)
def on_epoch_end(epoch, logs):
model.save_weights(model_path + 'w')
model.save(model_path)
print_callback = LambdaCallback(on_epoch_end=on_epoch_end)
ES = EarlyStopping(monitor='loss',
min_delta=0.001,
patience=10,
verbose=0,
mode='auto')
# if GPUs > 1:
# t_model = multi_gpu_model(model, gpus=GPUs)
# else:
# t_model = model
model.compile(
loss='categorical_crossentropy',
# optimizer=SGD(lr=1e-3, decay=1e-6, momentum=0.9, nesterov=True),
# optimizer=SGD(),
optimizer=Adam(),
Dense(16, activation='relu'
), # Rectified Linear Unit. This layer has only 16 nodes
Dense(1, activation='sigmoid'
) # This will give you a binary classfication output.
])
model.compile(
loss=
'binary_crossentropy', # THis is commonly used in reclassification problems.
optimizer='adam', # THis is a varient of the stocastic gradient dissent.
metrics=['accuracy'])
# print("Model Summery", model.summary())
model.summary()
from tensorflow.python.keras.callbacks import LambdaCallback
simple_log = LambdaCallback(on_epoch_end=lambda e, l: print(
e, end='.')) # THis is to print a '.' at the end of each epoc.
E = 20 # This is the number of Epoc's that we are going to use.
h = model.fit(x_train,
y_train,
validation_split=0.2,
batch_size=512,
epochs=E,
callbacks=[simple_log],
verbose=1)
plt.title('Accuracy Graph')
plt.plot(range(E), h.history['accuracy'], label='Training')
plt.plot(range(E), h.history['val_accuracy'], label='Validation')
plt.legend()
plt.show()
def main_fun(args, ctx):
import numpy
import os
import tensorflow as tf
from tensorflow.python import keras
from tensorflow.python.keras import backend as K
from tensorflow.python.keras.models import Sequential, load_model, save_model
from tensorflow.python.keras.layers import Dense, Dropout
from tensorflow.python.keras.optimizers import RMSprop
from tensorflow.python.keras.callbacks import LambdaCallback, TensorBoard
from tensorflow.python.saved_model import builder as saved_model_builder
from tensorflow.python.saved_model import tag_constants
from tensorflow.python.saved_model.signature_def_utils_impl import predict_signature_def
from tensorflowonspark import TFNode
cluster, server = TFNode.start_cluster_server(ctx)
if ctx.job_name == "ps":
server.join()
elif ctx.job_name == "worker":
def generate_rdd_data(tf_feed, batch_size):
print("generate_rdd_data invoked")
while True:
batch = tf_feed.next_batch(batch_size)
imgs = []
lbls = []
for item in batch:
imgs.append(item[0])
lbls.append(item[1])
images = numpy.array(imgs).astype('float32') / 255
labels = numpy.array(lbls).astype('float32')
yield (images, labels)
with tf.device(
tf.train.replica_device_setter(
worker_device="/job:worker/task:%d" % ctx.task_index,
cluster=cluster)):
IMAGE_PIXELS = 28
batch_size = 100
num_classes = 10
# the data, shuffled and split between train and test sets
if args.input_mode == 'tf':
from tensorflow.python.keras.datasets import mnist
(x_train, y_train), (x_test, y_test) = mnist.load_data()
x_train = x_train.reshape(60000, 784)
x_test = x_test.reshape(10000, 784)
x_train = x_train.astype('float32') / 255
x_test = x_test.astype('float32') / 255
# convert class vectors to binary class matrices
y_train = keras.utils.to_categorical(y_train, num_classes)
y_test = keras.utils.to_categorical(y_test, num_classes)
else: # args.mode == 'spark'
x_train = tf.placeholder(tf.float32,
[None, IMAGE_PIXELS * IMAGE_PIXELS],
name="x_train")
y_train = tf.placeholder(tf.float32, [None, 10],
name="y_train")
model = Sequential()
model.add(Dense(512, activation='relu', input_shape=(784, )))
model.add(Dropout(0.2))
model.add(Dense(512, activation='relu'))
model.add(Dropout(0.2))
model.add(Dense(10, activation='softmax'))
model.summary()
model.compile(loss='categorical_crossentropy',
optimizer=RMSprop(),
metrics=['accuracy'])
saver = tf.train.Saver()
with tf.Session(server.target) as sess:
K.set_session(sess)
def save_checkpoint(epoch, logs=None):
if epoch == 1:
tf.train.write_graph(sess.graph.as_graph_def(),
args.model_dir, 'graph.pbtxt')
saver.save(sess,
os.path.join(args.model_dir, 'model.ckpt'),
global_step=epoch * args.steps_per_epoch)
ckpt_callback = LambdaCallback(on_epoch_end=save_checkpoint)
tb_callback = TensorBoard(log_dir=args.model_dir,
histogram_freq=1,
write_graph=True,
write_images=True)
# add callbacks to save model checkpoint and tensorboard events (on worker:0 only)
callbacks = [ckpt_callback, tb_callback
] if ctx.task_index == 0 else None
if args.input_mode == 'tf':
# train & validate on in-memory data
model.fit(x_train,
y_train,
batch_size=batch_size,
epochs=args.epochs,
verbose=1,
validation_data=(x_test, y_test),
callbacks=callbacks)
else: # args.input_mode == 'spark':
# train on data read from a generator which is producing data from a Spark RDD
tf_feed = TFNode.DataFeed(ctx.mgr)
model.fit_generator(generator=generate_rdd_data(
tf_feed, batch_size),
steps_per_epoch=args.steps_per_epoch,
epochs=args.epochs,
verbose=1,
callbacks=callbacks)
if args.export_dir and ctx.job_name == 'worker' and ctx.task_index == 0:
# save a local Keras model, so we can reload it with an inferencing learning_phase
save_model(model, "tmp_model")
# reload the model
K.set_learning_phase(False)
new_model = load_model("tmp_model")
# export a saved_model for inferencing
builder = saved_model_builder.SavedModelBuilder(
args.export_dir)
signature = predict_signature_def(
inputs={'images': new_model.input},
outputs={'scores': new_model.output})
builder.add_meta_graph_and_variables(
sess=sess,
tags=[tag_constants.SERVING],
signature_def_map={'predict': signature},
clear_devices=True)
builder.save()
if args.input_mode == 'spark':
tf_feed.terminate()
def train(
batch_build_length: int,
batch_size: int,
metadata: dict,
model_config: Dict,
output_dir: str,
test_data: Data,
train_data: Data,
initial_epoch: int = 0,
initial_weights: str = "",
tensorboard_verbose: bool = False,
train_epochs: int = 10,
train_steps_per_epoch_limit: int = None,
validation_freq: int = 1,
verbose_logging: bool = False,
):
if initial_epoch > 0 and initial_weights:
raise ValueError(
"Only one of: initial_epoch or initial_weights - can be set at once"
)
model_arch_name = model_config["model_arch_name"]
weights_best_filename = os.path.join(output_dir, "model_weights.h5")
checkpoint_filename = os.path.join(output_dir, "model_checkpoint.h5")
logdir = os.path.join(output_dir, "tf_logs")
if initial_epoch > 0:
if os.path.exists(checkpoint_filename):
logger.info("Loading model: {}".format(checkpoint_filename))
model = tf.keras.models.load_model(checkpoint_filename)
else:
raise RuntimeError(
"{} not found, you should start from scratch".format(
checkpoint_filename))
else: # initial_epoch == 0
patch_size = model_config["patch_size"]
feature_shape = (
patch_size[0],
patch_size[1],
patch_size[2],
len(model_config["stencil_channels"]),
)
model_make_function = models_collection[model_arch_name]
model_params = deepcopy(model_config)
model_params.update({
"feature_shape": feature_shape,
"voxel_size": tuple(train_data.voxel_size),
})
model = model_make_function(params=model_params)
model.compile(optimizer="adam", loss="mse", metrics=["mse"])
ensure_dir(output_dir)
with open(os.path.join(output_dir, "model.json"), "w") as json_file:
json_file.write(model.to_json())
metadata["model_params"] = model_params
with open(os.path.join(output_dir, "model_metadata.json"),
"w") as json_file:
json.dump(metadata, json_file, indent=4, sort_keys=True)
if initial_weights:
logger.info("Loading weights: {}".format(initial_weights))
model.load_weights(initial_weights)
if os.path.exists(logdir):
shutil.rmtree(logdir)
checkpoint_best_callback = ModelCheckpoint(
filepath=weights_best_filename,
monitor="val_loss",
verbose=1,
save_best_only=True,
save_weights_only=True,
mode="auto",
period=1,
)
checkpoint_all_callback = ModelCheckpoint(
filepath=checkpoint_filename,
verbose=1,
save_best_only=False,
save_weights_only=False,
mode="auto",
period=1,
)
tensorboard_callback = TensorBoard(
log_dir=logdir,
histogram_freq=25 if tensorboard_verbose else 0,
write_grads=tensorboard_verbose,
write_images=tensorboard_verbose,
profile_batch=0,
)
def process_timings(epoch, logs):
this_epoch_train_timings = train_data.get_and_clean_timings()
this_epoch_test_timings = test_data.get_and_clean_timings()
update_timings_dict(TIMINGS_TRAIN, this_epoch_train_timings)
update_timings_dict(TIMINGS_VALIDATE, this_epoch_test_timings)
logger.info("Epoch {}".format(epoch))
logger.info("Data.train timings: {}".format(
pformat(this_epoch_train_timings)))
logger.info("Data.test timings: {}".format(
pformat(this_epoch_test_timings)))
process_timings_callback = LambdaCallback(
on_epoch_begin=lambda epoch, logs: print(),
on_epoch_end=process_timings)
make_batch_function = (make_batch_swap_axes if model_arch_name
in ("planar_first",
"first_baseline") else make_batch)
train_steps_per_epoch = math.ceil(len(train_data) / batch_size)
if train_steps_per_epoch_limit is not None:
train_steps_per_epoch = min(train_steps_per_epoch,
train_steps_per_epoch_limit)
initial_train_data_batch_start_index = (
initial_epoch * train_steps_per_epoch * batch_size) % len(train_data)
initial_data_shuffle_random_seed = (initial_epoch * train_steps_per_epoch *
batch_size) // len(train_data)
callbacks = [
checkpoint_best_callback,
checkpoint_all_callback,
tensorboard_callback,
process_timings_callback,
]
model.fit_generator(
generator=make_batches_gen(
data=train_data,
data_name="train_data",
batch_size=batch_size,
batch_build_length=batch_build_length,
initial_data_batch_start_index=initial_train_data_batch_start_index,
initial_data_shuffle_random_seed=initial_data_shuffle_random_seed,
make_batch_function=make_batch_function,
reshuffle=True,
),
steps_per_epoch=train_steps_per_epoch,
validation_data=make_batches_gen(
data=test_data,
data_name="validation_data",
batch_size=batch_size,
make_batch_function=make_batch_function,
reshuffle=False,
),
validation_freq=validation_freq,
validation_steps=math.ceil(len(test_data) / batch_size),
epochs=train_epochs,
verbose=2,
callbacks=callbacks,
initial_epoch=initial_epoch,
)
x_decoded_mean=outputs,
encoded_sigma=z_log_var,
encoded_mean=z_mean))
opt = PARAMS['optimizer'](PARAMS['learning_rate'])
vae.compile(optimizer=opt) # rmsprop
vae.summary()
train_generator, validation_generator, test_generator = get_generators()
callback = NeptuneCallback(neptune_experiment=npt_exp,
n_batch=steps,
test_generator=test_generator,
img_size=image_size)
callbacks = [
callback,
LambdaCallback(on_epoch_end=lambda epoch, logs: log_data(logs)),
EarlyStopping(patience=PARAMS['early_stopping'],
monitor='loss',
restore_best_weights=True),
]
if PARAMS['scheduler']:
callbacks.append(LearningRateScheduler(lr_scheduler))
vae.fit(train_generator,
steps_per_epoch=len(train_generator),
validation_data=validation_generator,
validation_steps=len(validation_generator),
epochs=PARAMS['n_epochs'],
shuffle=PARAMS['shuffle'],
callbacks=callbacks)
def train_model(self):
if self.train_mode == 'train':
model = self.get_compiled_model()
self.save_model(model)
plot_model(model, to_file=self.model_dir + '/model.png', show_shapes=True)
elif self.train_mode == 'retrain':
model = self.load_model()
else:
raise ValueError('Wrong TRAIN_MODE value')
# Callbacks
# def step_decay(epoch):
# initial_lrate = 0.1
# drop = 0.5
# epochs_drop = 10.0
# lrate = initial_lrate * math.pow(drop, math.floor((1 + epoch) / epochs_drop))
# return lrate
#
# def exp_decay(epoch):
# initial_lrate = 0.1
# k = 0.1
# lrate = initial_lrate * math.exp(-k * epoch)
# return lrate
#
# lrate = LearningRateScheduler(step_decay)
def on_epoch_end(epoch, logs):
if epoch % EPOCH_CHEK == 0:
if not os.path.exists(MODEL_DIR + '/images_per_epoch'):
os.makedirs(MODEL_DIR + '/images_per_epoch')
predicted_images = model.predict(test_images)
for i in range(0, self.num_saved_imgs):
predicted_img_path = os.path.join(MODEL_DIR, 'images_per_epoch',
str(i).zfill(1) + '_image_' + str(epoch).zfill(3)
+ 'epoch' + '.png')
predicted_img = predicted_images[i].reshape(IMG_HEIGHT, IMG_WIDTH)
plt.imsave(predicted_img_path, predicted_img, cmap="gray")
pltfig = LambdaCallback(on_epoch_end=on_epoch_end)
csv_logger = CSVLogger(os.path.join(self.model_dir, 'logs.csv'), separator=',', append=False)
weight_saver = ModelCheckpoint(filepath=self.model_dir + '/weights.h5',
verbose=1,
save_best_only=True,
save_weights_only=True)
early_stop = EarlyStopping(monitor='val_loss', min_delta=0.001, patience=150)
run_name = self.network_type + "#code=" + str(CODE_SIZE)
wandb.init(project="tavr",
dir=self.model_dir,
name=run_name,
sync_tensorboard=True)
wb = WandbCallback(monitor='val_loss',
mode='min',
save_weights_only=False,
save_model=False,
log_evaluation=False,
verbose=1)
callbacks = [early_stop, weight_saver, csv_logger, wb, pltfig]
# Model training
if IS_AUGMENTATION:
datagen_args = dict(
# rotation_range=90,
# width_shift_range=0.20,
# height_shift_range=0.20,
# shear_range=10,
# zoom_range=[0.75, 1.25],
# brightness_range=(0.5, 1.5),
horizontal_flip=True,
vertical_flip=True,
fill_mode='constant',
cval=0
)
else:
datagen_args = dict()
datagen = tf.keras.preprocessing.image.ImageDataGenerator(**datagen_args)
model.fit_generator(datagen.flow(x=train_images, y=train_images, batch_size=self.batch_size),
validation_data=(val_images, val_images),
steps_per_epoch=len(train_images) // self.batch_size,
epochs=self.epochs,
callbacks=callbacks)
def __call__(self):
return LambdaCallback(on_epoch_end=self.log_sampling)
kernel_initializer='random_uniform',
name="Dense_fc_2"))
model.add(Dense(10, activation='softmax', name="Ausgabe"))
model.compile(loss=losses.categorical_crossentropy,
optimizer=optimizers.Adadelta(),
metrics=["accuracy", "mse", metrics.categorical_accuracy])
# Wird aufgerufen, wenn das Training beginnt
def train_begin():
url = 'http://localhost:9000/publish/train/begin'
post_fields = {"model": model.to_json()}
request = requests.post(url, data=post_fields)
lambda_cb = LambdaCallback(on_train_begin=train_begin())
remote_cb = RemoteMonitor(root='http://localhost:9000',
path="/publish/epoch/end/",
send_as_json=True)
#tf_cb = callbacks.TensorBoard(log_dir='./logs', histogram_freq=0, batch_size=32, write_graph=True, write_grads=True, write_images=True, embeddings_freq=0, embeddings_layer_names=None, embeddings_metadata=None, embeddings_data=None)
model.fit(train_data,
train_labels,
batch_size=64,
epochs=100,
verbose=1,
validation_split=0.33,
callbacks=[lambda_cb, remote_cb])
from tensorflow.python.keras.layers import Embedding, Dense, GlobalAveragePooling1D
model = Sequential([
Embedding(10000, 16),
GlobalAveragePooling1D(),
Dense(16, activation = 'relu'),
Dense(1, activation = 'sigmoid')
])
model.compile(
optimizer = 'adam',
loss = 'binary_crossentropy',
metrics = ['acc']
)
model.summary()
from tensorflow.python.keras.callbacks import LambdaCallback
simple_logging = LambdaCallback(on_epoch_end = lambda e, l: print(e, end='.'))
E = 20
h = model.fit(
x_train, y_train,
validation_split = 0.2,
epochs = E,
callbacks = [simple_logging],
verbose = False
)
# Commented out IPython magic to ensure Python compatibility.
import matplotlib.pyplot as plt
# %matplotlib inline
plt.plot(range(E), h.history['acc'], label = 'Training')
plt.plot(range(E), h.history['val_acc'], label = 'Validation')
plt.legend()
def train(params):
batch_size = params.batch_size
sample_rate = 0.8
seed = 2019
save_path = os.path.join(params.model_path, "%s.h5" % params.net)
pretrain_path = save_path
input_size, loss_weight, loss_func = gen_traning_params(params.net)
models, graph, session = BuildModel(params.net,
lr=params.lr,
pretrain_path=pretrain_path,
is_train=True)
print(pretrain_path)
with session.as_default(), graph.as_default():
models.compile(optimizer=Adam(lr=params.lr),
loss=loss_func,
loss_weights=loss_weight,
metrics={"class": accuracy})
log_dir = params.log
def get_weights(epoch, loggs):
print(epoch, K.get_value(models.optimizer.lr))
callbacks = [
ModelCheckpoint(save_path,
monitor='val_class_accuracy',
verbose=1,
save_best_only=True,
mode='max'),
TensorBoard(log_dir=log_dir, batch_size=batch_size, write_images=True),
ReduceLROnPlateau(monitor='val_class_accuracy',
factor=0.1,
patience=5,
min_lr=0.0000001),
LambdaCallback(on_epoch_end=get_weights),
]
dataframe = load_data("./data/", ptn=params.net)
trainset, testset = train_test_split(dataframe,
train_size=sample_rate,
test_size=1 - sample_rate,
random_state=seed)
print(trainset.shape, testset.shape)
train_gen = DataGenetator(trainset,
input_size=input_size,
batch_size=batch_size)
validation_gen = DataGenetator(testset,
input_size=input_size,
batch_size=batch_size,
is_training=False)
with session.as_default(), graph.as_default():
history = models.fit_generator(
train_gen,
workers=params.worker,
use_multiprocessing=(params.worker > 1),
steps_per_epoch=len(trainset) / batch_size,
epochs=80,
callbacks=callbacks,
validation_data=validation_gen,
validation_steps=len(testset) / batch_size)
def train(**kwargs):
"""
Train model with specified parameters. For example parameters / parameters used for the paper check
"run_experiments.py".
:param with_aux: Flag, whether auxiliary (SSAL) classifiers should be used.
:param aux_weight: Weight of the auxiliary loss compared to the base loss. List, if multiple aux losses.
:param aux_depth: List of positions for aux nets. A position is either a list of 2 integers or just one integer.
List of two integers encode major and minor block position of aux net. It is placed in the major block after the
given minor block. Minor 0 means typically before the first minor block. Minor -1 is the last position of a
major block, if existent. It is typically behind the last minor block, but before pooling, if existent at the
end or after the major block. A single integer defaults to the last position in the specified major block.
:param batch_size: Batch size.
:param epochs: Number of epochs of training.
:param learning_rate: Maximum learning rate for the training.
:param weight_decay: Weight decay for the training.
:param momentum: Momentum for the training.
:param num_coarse_classes: Number of coarse categories in the auxiliary classifier. List, if multiple aux
classifiers. Must fit with the grouping specified in grouping parameter.
:param save_model: Flag, whether model should be saved after training (in models folder). See param use_pretrained.
:param max_epoch: Epoch in which learning rate reaches its peak (for slanted triangular learning rate).
:param use_pretrained: Path to .h5 model to load.
:param optimize: If true, use split of train set as validation set.
:param network: Current options: 'resnet50', 'WRN', 'DenseNet'
:param grouping: Identifier of the grouping to be loaded from txt to array. Labels file should list the coarse label
for each fine label and be readable by numpy.loadtxt. File 'cifar100_' + <grouping> + '.txt' must exist
in labels folder.
:param aux_layouts: List of layouts for each aux network. A aux network layout is a list of blocks. A block is also
a list. The first element defines the type of block. Currently implemented is 'inception' for an inception block
and 'cbr' for a CBR layer. The following elements are optional and can further adjust the block's layout.
Example: '[[['cbr'],['inception']],[['inception']]]' contains one CBR layer + one inception block in the first
aux network and just one inception block in the second one.
:param aux_weight_decay: If None, copied from weight_decay
:param se_net: Include squeeze & excitation blocks. Currently only for WRN.
:param mean_std_norm: If True, normalize dataset by mean and stddev of training set, else normalize images to [0,1].
:param label_smoothing: If True, smooth ground truth logits s.t. the true label only has a value of 0.9.
:return: None
"""
params = {
'with_aux': False,
'aux_weight': 0.3,
'aux_depth': [],
'batch_size': 256,
'epochs': 1,
'learning_rate': 0.01,
'weight_decay': None,
'aux_weight_decay': None,
'momentum': 0.95,
'num_coarse_classes': 20,
'exp_combination_factor': 1.0,
'save_model': False,
'use_pretrained': None,
'max_epoch': 10,
'grouping': 'default',
'optimize': False,
'network': None,
'aux_layouts': None,
'wide_width': 10,
'wide_depth': 28,
'se_net': False,
'dense_depth': 100,
'dense_growth': 12,
'nesterov': False,
'mean_std_norm': False,
'label_smoothing': False
}
params.update(kwargs)
# create explicit session as possible solution against memory leak during meta parameter exploration
cfg = tf.ConfigProto()
cfg.gpu_options.allow_growth = True
tf.Session(config=cfg)
grouping = params['grouping']
if not isinstance(grouping, list):
grouping = [grouping]
hyper = params
aux_depth = params['aux_depth']
if isinstance(aux_depth, int):
aux_depth = [aux_depth]
aux_depth = aux_depth.copy()
for i in range(len(aux_depth)):
if not isinstance(aux_depth[i], list):
aux_depth[i] = (int(aux_depth[i]), -1)
with_aux = hyper['with_aux']
num_auxs = len(aux_depth)
max_epoch = params[
'max_epoch'] # epoch in which learning rate reaches it's maximum value
batch_size = hyper['batch_size']
epochs = hyper['epochs']
aux_weight = hyper['aux_weight']
if not isinstance(aux_weight, list):
aux_weight = [aux_weight]
aux_weight_decay = params['aux_weight_decay']
if aux_weight_decay is None:
aux_weight_decay = params['weight_decay']
learning_rate = hyper['learning_rate'] # maximum learning rate
num_coarse_classes = hyper['num_coarse_classes']
if not isinstance(num_coarse_classes, list):
num_coarse_classes = [num_coarse_classes]
aux_layouts = hyper['aux_layouts']
if with_aux:
if not isinstance(aux_layouts[0][0], list):
raise TypeError(
'Bad aux_layouts format. Expect list with 2-element list for each SSAL branch'
)
if aux_layouts is not None:
# repeat last aux_layout if more positions specified than layouts
if len(aux_layouts) < len(aux_depth):
while len(aux_layouts) < len(aux_depth):
aux_layouts.append(aux_layouts[-1])
num_classes = 100
# load cifar100 data and create train set and test set
(x_train, y_train), (x_test, y_test) = cifar100.load_data()
if params['optimize']:
len_val = round(0.85 * len(y_train))
x_test = x_train[len_val:]
x_train = x_train[:len_val]
y_test = y_train[len_val:]
y_train = y_train[:len_val]
print('x_train:', x_train.shape)
print('y_train:', y_train.shape)
print('x_test:', x_test.shape)
print('y_test:', y_test.shape)
# (x_train_c, y_train_c), (x_test_c, y_test_c) = cifar100.load_data(label_mode='coarse') # only used to map classes
steps_per_epoch = int(len(x_train) / batch_size)
# logs custom combined accuracy
combined_accuracy_log = []
# create coarse labels
cats = []
y_train_c = []
y_test_c = []
if with_aux:
cats, y_train_c, y_test_c = create_coarse_data(y_train, y_test,
'cifar100', grouping)
# prepares output for categorical crossentropy
if not params['label_smoothing']:
y_train = keras.utils.to_categorical(y_train, num_classes)
y_test = keras.utils.to_categorical(y_test, num_classes)
if with_aux:
for i in range(0, num_auxs):
y_train_c[i] = keras.utils.to_categorical(
y_train_c[i], num_coarse_classes[i])
y_test_c[i] = keras.utils.to_categorical(
y_test_c[i], num_coarse_classes[i])
else:
y_train = to_categorical_smooth(y_train, num_classes, temperature=0.1)
y_test = to_categorical_smooth(y_test, num_classes, temperature=0.1)
if with_aux:
for i in range(0, num_auxs):
y_train_c[i] = to_categorical_smooth(y_train_c[i],
num_coarse_classes[i],
temperature=0.1)
y_test_c[i] = to_categorical_smooth(y_test_c[i],
num_coarse_classes[i],
temperature=0.1)
# normalize pixel values
if not params['mean_std_norm']:
x_train = x_train.astype('float32')
x_test = x_test.astype('float32')
x_train /= 255
x_test /= 255
else:
x_train = x_train.astype('float32')
x_test = x_test.astype('float32')
avg = numpy.average(x_train, axis=(0, 1, 2))
stddev = numpy.std(x_train, axis=(0, 1, 2))
x_train = (x_train - avg) / stddev
x_test = (x_test - avg) / stddev
weight_decay = params['weight_decay']
use_pretrained = params['use_pretrained']
if use_pretrained is not None:
# load .h5 model
model = load_model(use_pretrained)
else:
if params['network'] == 'resnet50':
if with_aux:
model = ResnetBuilder.build_resnet_50(
x_train.shape[1:],
num_classes,
aux_depth=aux_depth,
num_coarse_classes=num_coarse_classes,
aux_layouts=aux_layouts)
else:
model = ResnetBuilder.build_resnet_50(x_train.shape[1:],
num_classes)
elif params['network'] == 'WRN':
depth = params['wide_depth']
width = params['wide_width']
if with_aux:
model = WideResidualNetwork(
depth=depth,
width=width,
input_shape=x_train.shape[1:],
classes=num_classes,
activation='softmax',
aux_depth=aux_depth,
num_coarse_classes=num_coarse_classes,
aux_layouts=aux_layouts,
aux_weight_decay=aux_weight_decay,
se_net=params['se_net'],
aux_init='he_normal')
else:
model = WideResidualNetwork(depth=depth,
width=width,
input_shape=x_train.shape[1:],
classes=num_classes,
activation='softmax',
aux_depth=[],
num_coarse_classes=[],
aux_layouts=[],
se_net=params['se_net'])
if weight_decay is not None:
# manually add weight decay as loss
for layer in model.layers:
if len(layer.losses) == 0:
if isinstance(layer,
keras.layers.Conv2D) or isinstance(
layer, keras.layers.Dense):
layer.add_loss(
keras.regularizers.l2(weight_decay)(
layer.kernel))
if hasattr(layer,
'bias_regularizer') and layer.use_bias:
layer.add_loss(
keras.regularizers.l2(weight_decay)(
layer.bias))
elif params['network'] == 'DenseNet':
depth = params['dense_depth']
growth = params['dense_growth']
decay = weight_decay if weight_decay is not None else 0
# 100-12, 250-24, 190-40
if with_aux:
model = DenseNet(
input_shape=x_train.shape[1:],
depth=depth, # L parameter / Depth parameter
growth_rate=growth, # k parameter
bottleneck=True, # True for DenseNet-B
reduction=
0.5, # 1-theta (1-compression), >0.0 for DenseNet-C
subsample_initial_block=False, # Keep false for CIFAR
weight_decay=decay,
classes=num_classes,
aux_depth=aux_depth,
num_coarse_classes=num_coarse_classes,
aux_layouts=aux_layouts,
initialization='orthogonal',
aux_init='he_normal')
else:
model = DenseNet(
input_shape=x_train.shape[1:],
depth=depth, # L parameter / Depth parameter
growth_rate=growth, # k parameter
bottleneck=True, # True for DenseNet-B
reduction=
0.5, # 1-theta (1-compression), >0.0 for DenseNet-C
subsample_initial_block=False, # Keep false for CIFAR
weight_decay=decay,
classes=num_classes,
aux_depth=[],
num_coarse_classes=[],
aux_layouts=[],
initialization='orthogonal')
else:
raise NotImplementedError("Unknown Model: " +
str(params['network']))
# stochastic gradient descent
sgd = SGD(lr=hyper['learning_rate'],
momentum=hyper['momentum'],
nesterov=params['nesterov'])
print('Free parameters:', model.count_params())
# plot_model(model, to_file='resnet50_1aux_model.pdf', show_shapes=True)
update_lr = LearningRateScheduler(lambda epoch, lr: lr_scheduler(
epoch, epochs, learning_rate, max_epoch))
if with_aux:
model.compile(optimizer=sgd,
loss='categorical_crossentropy',
loss_weights=[1.0] + aux_weight,
metrics=['accuracy'])
else:
model.compile(optimizer=sgd,
loss='categorical_crossentropy',
metrics=['accuracy'])
log_call = LambdaCallback(
on_epoch_end=lambda epoch, logs: log_combined_acc(
model, x_test, y_test, cats, num_classes,
combined_accuracy_log, params['exp_combination_factor']),
on_train_end=lambda logs: classification_analysis(
model, x_test, y_test, cats, num_classes
)) # called during training after each epoch
start_time = time.time()
datagen = ImageDataGenerator(width_shift_range=5,
height_shift_range=5,
horizontal_flip=True,
fill_mode='reflect')
# train
if with_aux:
model.fit(generator_for_multi_outputs(datagen,
x_train,
[y_train] + y_train_c,
batch_size=batch_size),
epochs=epochs,
validation_data=(x_test, [y_test] + y_test_c),
callbacks=[update_lr, log_call],
verbose=1,
steps_per_epoch=steps_per_epoch,
shuffle=True)
else:
model.fit(datagen.flow(x_train, y_train, batch_size=batch_size),
epochs=epochs,
shuffle=True,
validation_data=(x_test, y_test),
callbacks=[update_lr],
steps_per_epoch=steps_per_epoch,
verbose=1)
end_time = time.time()
print('duration', end_time - start_time, 's')
if params['save_model']:
model.save('../models/cifar100_model.h5')
# evaluate and print results
if with_aux:
score = model.evaluate(x_test, [y_test] + y_test_c,
batch_size=batch_size)
print('Test acc (fine):', score[2 + num_auxs])
for i in range(0, num_auxs):
print('Test acc (SSAL ' + str(i) + '):', score[3 + num_auxs + i])
log_combined_acc(
model,
x_test,
y_test,
cats,
num_classes, [],
exp_combination_factor=params['exp_combination_factor']
) # also print combined acc!
else:
score = model.evaluate(x_test, y_test, batch_size=batch_size)
print('Test acc:', score[1])
del model