def main():
# load data from the data files
jpn_data = get_data(jpn_txt_path)
en_data = get_data(en_txt_path)
#train_jpn, val_jpn, train_en, val_en = train_test_split(jpn_data,
# en_data,
# test_size=TR_TE_RATIO)
JPN_MAX_LEN = get_max_len(jpn_data)
EN_MAX_LEN = get_max_len(en_data)
# include [BOS] and [EOS] to each max len above
JPN_MAX_LEN += 2
EN_MAX_LEN += 2
test_jpn_data = [
"今日は夜ごはん何にしようかな？", "ここ最近暑い日がずっと続きますね。", "来年は本当にオリンピックが開催されるでしょうか？",
"将来の夢はエンジニアになることです。", "子供のころはあの公園でたくさん遊んだなー。", "今日は早く帰りたいな。",
"明日は父の日だ。", "試験勉強はなかなか大変です。", "来年はおいしいお店に行きたいです。",
"あそこの家にはまだ誰か住んでいますか？"
]
#test_en_data = [[""],
# [""],
# [""],
# [""],
# [""]]
# preprocess for the train dataset
train_dataset = tf.data.Dataset.from_tensor_slices((jpn_data, en_data))
train_dataset = train_dataset.map(tf_encode)
train_dataset = train_dataset.cache()
train_dataset = train_dataset.shuffle(
len(jpn_data)).padded_batch(BATCH_SIZE)
train_dataset = train_dataset.prefetch(AUTOTUNE)
## preprocess for the validation dataset
#val_dataset = tf.data.Dataset.from_tensor_slices((val_jpn, val_en))
#val_dataset = val_dataset.map(tf_encode)
#val_dataset = val_dataset.padded_batch(BATCH_SIZE)
# preprocess for the test data
#test_dataset = tf.data.Dataset.from_tensor_slices((test_jpn_data, test_en_data))
#test_dataset = test_dataset.map(tf_encode)
#test_dataset = test_dataset.cache()
#test_dataset = test_dataset.padded_batch(len(test_jpn_data))
#test_dataset = test_dataset.prefetch(AUTOTUNE)
# instantiate the Transformer model
transformer = Transformer(num_layers=num_layers,
d_model=d_model,
num_heads=num_heads,
dff=dff,
input_vocab_size=jpn_vocab_size,
target_vocab_size=en_vocab_size,
pe_input=JPN_MAX_LEN,
pe_target=EN_MAX_LEN)
# set learning rate, optimizer, loss and matrics
learning_rate = CustomSchedule(d_model)
optimizer = Adam(learning_rate, beta_1=0.9, beta_2=0.98, epsilon=1e-9)
loss_object = SparseCategoricalCrossentropy(from_logits=True,
reduction="none")
def loss_function(label, pred):
mask = tf.math.logical_not(tf.math.equal(label, 0))
loss_ = loss_object(label, pred)
mask = tf.cast(mask, dtype=loss_.dtype)
loss_ *= mask
return tf.reduce_sum(loss_) / tf.reduce_sum(mask)
train_loss = Mean(name="train_loss")
train_accuracy = SparseCategoricalAccuracy(name="train_accuracy")
"""
The @tf.function trace-compiles train_step into a TF graph for faster
execution. The function specializes to the precise shape of the argument
tensors. To avoid re-tracing due to the variable sequence lengths or
variable batch sizes(usually the last batch is smaller), use input_signature
to specify more generic shapes.
"""
train_step_signature = [
tf.TensorSpec(shape=(None, None), dtype=tf.int64),
tf.TensorSpec(shape=(None, None), dtype=tf.int64)
]
@tf.function(input_signature=train_step_signature)
def train_step(inp, tar):
tar_inp = tar[:, :-1]
tar_label = tar[:, 1:]
training = True
enc_padding_mask, combined_mask, dec_padding_mask = create_masks(
inp, tar_inp)
with tf.GradientTape() as tape:
predictions, _ = transformer(inp, tar_inp, training,
enc_padding_mask, combined_mask,
dec_padding_mask)
loss = loss_function(tar_label, predictions)
gradients = tape.gradient(loss, transformer.trainable_variables)
optimizer.apply_gradients(
zip(gradients, transformer.trainable_variables))
train_loss(loss)
train_accuracy(tar_label, predictions)
# set the checkpoint and the checkpoint manager
ckpt = tf.train.Checkpoint(epoch=tf.Variable(0),
transformer=transformer,
optimizer=optimizer)
ckpt_manager = tf.train.CheckpointManager(ckpt, ckpt_path, max_to_keep=5)
# if a checkpoint exists, restore the latest checkpoint.
if ckpt_manager.latest_checkpoint:
ckpt.restore(ckpt_manager.latest_checkpoint)
print("Latest checkpoint restored.")
# set up summary writers
current_time = datetime.datetime.now().strftime("%Y%m%d-%H%M%S")
log_dir = os.path.join(log_path, current_time, "train")
#test_log_dir = os.path.join(log_path, current_time, "validation")
summary_writer = tf.summary.create_file_writer(log_dir)
#test_summary_writer = tf.summary.create_file_writer(test_log_dir)
for ckpt.epoch in range(EPOCHS):
start = time.time()
ckpt.epoch.assign_add(1)
train_loss.reset_states()
train_accuracy.reset_states()
# inp: Japanese, tar: English
for (batch, (inp, tar)) in enumerate(train_dataset):
train_step(inp, tar)
if batch % 100 == 0:
print("Epoch {} Batch {} Loss {:.4f} Accuracy {:.4f}".format(
ckpt.epoch, batch, train_loss.result(),
train_accuracy.result()))
# output the training log for every epoch
print("Epoch {} Loss {:.4f} Accuracy {:.4f}".format(
ckpt.epoch, train_loss.result(), train_accuracy.result()))
print("Time taken for 1 epoch: {:.3f} secs\n".format(time.time() -
start))
# check how the model performs for every epoch
test_summary_log = test_translate(test_jpn_data, EN_MAX_LEN,
transformer)
with summary_writer.as_default():
tf.summary.scalar("loss", train_loss.result(), step=ckpt.epoch)
tf.summary.scalar("accuracy",
train_accuracy.result(),
step=ckpt.epoch)
tf.summary.text("test_text", test_summary_log, step=ckpt.epoch)
if (ckpt.epoch) % 5 == 0:
ckpt_save_path = ckpt_manager.save()
print("Saving checkpoint for epoch {} at {}".format(
ckpt.epoch, ckpt_save_path))
test_size=0.3,
shuffle=True,
random_state=42)
# It is suggested to use [-1, 1] input for GAN training
x_train = (x_train.astype('float32') - 127.5) / 127.5
x_test = (x_test.astype('float32') - 127.5) / 127.5
# Get image size
img_size = x_train[0].shape
# Get number of classes
n_classes = len(np.unique(y_train))
# %% ---------------------------------- Hyperparameters ----------------------------------------------------------------
optimizer = Adam(lr=0.0002, beta_1=0.5, beta_2=0.9)
latent_dim = 128
# trainRatio === times(Train D) / times(Train G)
trainRatio = 5
# %% ---------------------------------- Models Setup -------------------------------------------------------------------
# Build Generator/Decoder
def decoder():
# weight initialization
init = RandomNormal(stddev=0.02)
noise_le = Input((latent_dim, ))
x = Dense(4 * 4 * 256)(noise_le)
x = LeakyReLU(alpha=0.2)(x)
callbacks_list = [
checkpoint,
LearningRateScheduler(lr_scheduler, verbose=1), csv_logger
]
def my_loss_fn(y_true, y_pred):
squared_difference = tf.square(y_true - y_pred)
return tf.reduce_mean(squared_difference, axis=-1) # Note the `axis=-1`
def custom_loss(y_true, y_pred):
loss1 = binary_crossentropy(y_true, d3.output)
loss2 = binary_crossentropy(y_true, d4.output)
loss3 = binary_crossentropy(y_true, d5.output)
loss4 = binary_crossentropy(y_true, d345.output)
return (loss1 + loss2 + loss3 + loss4) / 4.0
opt = Adam(lr=1e-2)
d345.compile(loss=custom_loss, optimizer=opt)
# train the model
H = d345.fit(X_train,
Y_train,
validation_data=(X_test, Y_test),
epochs=EPOCHS,
batch_size=BS,
callbacks=callbacks_list)
headModel = Dropout(0.5)(headModel)
headModel = Dense(2, activation="softmax")(headModel)
# place the head FC model on top of the base model (this will become
# the actual model we will train)
model = Model(inputs=baseModel.input, outputs=headModel)
print('savepoint 7')
# loop over all layers in the base model and freeze them so they will
# *not* be updated during the first training process
for layer in baseModel.layers:
layer.trainable = False
# compile our model
print("[INFO] compiling model...")
opt = Adam(lr=INIT_LR, decay=INIT_LR / EPOCHS)
model.compile(loss="binary_crossentropy", optimizer=opt, metrics=["accuracy"])
print('savepoint 8')
# train the head of the network
print("[INFO] training head...")
H = model.fit(aug.flow(trainX, trainY, batch_size=BS),
steps_per_epoch=len(trainX) // BS,
validation_data=(testX, testY),
validation_steps=len(testX) // BS,
epochs=EPOCHS)
print('savepoint 9')
# make predictions on the testing set
print("[INFO] evaluating network...")
predIdxs = model.predict(testX, batch_size=BS)
return drunet(channel)
if __name__ == '__main__':
test_x_files, test_y_files = data_prepare_fault.get_train_files()
if not os.path.exists(model_dir):
os.makedirs(model_dir)
if os.path.exists(model_path):
print("get_model_from_load")
model = get_model_from_load()
else:
print("get_model_from_network")
model = get_model_from_network(1)
# compile the model
model.compile(optimizer=Adam(), loss=['mse'], metrics=[mean_squared_error, peak_sifnal_to_noise])
checkpointer = keras.callbacks.ModelCheckpoint('./models/model_{epoch:03d}.hdf5',
verbose=1, save_weights_only=False)
file_size = 8
files_len = len(test_x_files)
test_len = int(files_len*0.95)
history = model.fit(
data_prepare_fault.train_datagen(test_x_files[:test_len], test_y_files[:test_len], file_size),
steps_per_epoch=test_len // file_size,
epochs=10,
validation_data=data_prepare_fault.get_val_data(test_x_files[test_len:], test_y_files[test_len:]),
callbacks=[checkpointer])
plt.plot(history.history['loss'])
plt.title('model accuracy')
plt.ylabel('accuracy')
plt.xlabel('epoch')
def custom_ffcnn(model_config,
input_shape,
metrics,
n_classes,
mixed_precision=False,
output_bias=None):
'''
Defines a feedforward convolutional neural network model with residual connections for multiclass image classification.
:param model_config: A dictionary of parameters associated with the model architecture
:param input_shape: The shape of the model input
:param metrics: Metrics to track model's performance
:param n_classes: # of classes in data
:param mixed_precision: Whether to use mixed precision (use if you have GPU with compute capacity >= 7.0)
:param output_bias: bias initializer of output layer
:return: a Keras Model object with the architecture defined in this method
'''
# Set hyperparameters
nodes_dense0 = model_config['NODES_DENSE0']
nodes_dense1 = model_config['NODES_DENSE1']
lr = model_config['LR']
dropout = model_config['DROPOUT']
l2_lambda = model_config['L2_LAMBDA']
if model_config['OPTIMIZER'] == 'sgd':
optimizer = SGD(learning_rate=lr, momentum=0.9)
else:
optimizer = Adam(learning_rate=lr)
init_filters = model_config['INIT_FILTERS']
filter_exp_base = model_config['FILTER_EXP_BASE']
n_blocks = model_config['BLOCKS']
kernel_size = eval(model_config['KERNEL_SIZE'])
max_pool_size = eval(model_config['MAXPOOL_SIZE'])
strides = eval(model_config['STRIDES'])
pad = kernel_size[0] // 2
print("MODEL CONFIG: ", model_config)
if mixed_precision:
tf.train.experimental.enable_mixed_precision_graph_rewrite(optimizer)
if output_bias is not None:
output_bias = Constant(output_bias) # Set initial output bias
# Input layer
X_input = Input(input_shape)
X = X_input
X = ZeroPadding2D((pad, pad))(X)
# Add blocks of convolutions and max pooling
for i in range(n_blocks):
filters = init_filters * (2**i)
X = Conv2D(filters=filters,
kernel_size=kernel_size,
strides=strides,
padding='same',
name='conv2d_block' + str(i) + '_0',
kernel_initializer='he_uniform',
activation='relu',
activity_regularizer=l2(l2_lambda))(X)
X = Conv2D(filters=filters,
kernel_size=kernel_size,
strides=strides,
padding='same',
name='conv2d_block' + str(i) + '_1',
kernel_initializer='he_uniform',
activation='relu',
activity_regularizer=l2(l2_lambda))(X)
X = BatchNormalization(axis=3, name='bn_block' + str(i))(X)
X = MaxPool2D(max_pool_size, padding='same',
name='maxpool' + str(i))(X)
# Model head
X = GlobalAveragePooling2D(name='gloval_avgpool')(X)
X = Dropout(dropout)(X)
X = Dense(nodes_dense0,
kernel_initializer='he_uniform',
activity_regularizer=l2(l2_lambda),
activation='relu',
name='fc0')(X)
X = Dropout(dropout)(X)
X = Dense(nodes_dense1,
kernel_initializer='he_uniform',
activity_regularizer=l2(l2_lambda),
activation='relu',
name='fc1')(X)
X = Dense(n_classes, bias_initializer=output_bias)(X)
Y = Activation('softmax', dtype='float32', name='output')(X)
# Set model loss function, optimizer, metrics.
model = Model(inputs=X_input, outputs=Y)
model.summary()
model.compile(loss='categorical_crossentropy',
optimizer=optimizer,
metrics=metrics)
return model
activation='relu',
kernel_regularizer=regularizers.l2(0.0001)))
model.add(BatchNormalization())
model.add(MaxPooling2D(2, 2))
model.add(Dropout(0.5))
model.add(Flatten())
model.add(
Dense(32, activation='relu', kernel_regularizer=regularizers.l2(0.0001)))
model.add(BatchNormalization())
model.add(Dropout(0.8))
model.add(Dense(1, activation='sigmoid'))
model.compile(optimizer=Adam(0.0001),
loss='binary_crossentropy',
metrics=['acc'])
va = EarlyStopping(monitor='val_loss', mode='min', verbose=1, patience=50)
output = model.fit_generator(train_set,
steps_per_epoch=200 / batchsize,
epochs=60,
validation_data=valid_set,
validation_steps=94 / batchsize)
plt.figure(figsize=(6, 3))
plt.plot(output.history['acc'])
plt.plot(output.history['val_acc'])
plt.title('classifier accuracy')
plt.ylabel('accuracy')
def main(args=None):
# parse arguments
if args is None:
args = sys.argv[1:]
args = parse_args(args)
# create the generators
train_generator, validation_generator = create_generators(args)
num_classes = train_generator.num_classes()
num_anchors = train_generator.num_anchors
# optionally choose specific GPU
if args.gpu:
os.environ['CUDA_VISIBLE_DEVICES'] = args.gpu
# K.set_session(get_session())
model, prediction_model = efficientdet(
args.phi,
num_classes=num_classes,
num_anchors=num_anchors,
weighted_bifpn=args.weighted_bifpn,
freeze_bn=args.freeze_bn,
detect_quadrangle=args.detect_quadrangle)
# load pretrained weights
if args.snapshot:
if args.snapshot == 'imagenet':
model_name = 'efficientnet-b{}'.format(args.phi)
file_name = '{}_weights_tf_dim_ordering_tf_kernels_autoaugment_notop.h5'.format(
model_name)
file_hash = WEIGHTS_HASHES[model_name][1]
weights_path = keras.utils.get_file(file_name,
BASE_WEIGHTS_PATH + file_name,
cache_subdir='models',
file_hash=file_hash)
model.load_weights(weights_path, by_name=True)
else:
print('Loading model, this may take a second...')
model.load_weights(args.snapshot, by_name=True)
# freeze backbone layers
if args.freeze_backbone:
# 227, 329, 329, 374, 464, 566, 656
for i in range(1, [227, 329, 329, 374, 464, 566, 656][args.phi]):
model.layers[i].trainable = False
if args.gpu and len(args.gpu.split(',')) > 1:
model = keras.utils.multi_gpu_model(model,
gpus=list(
map(int, args.gpu.split(','))))
# compile model
model.compile(
optimizer=Adam(lr=1e-3),
loss={
'regression':
smooth_l1_quad() if args.detect_quadrangle else smooth_l1(),
'classification':
focal()
},
)
# print(model.summary())
# create the callbacks
callbacks = create_callbacks(
model,
prediction_model,
validation_generator,
args,
)
if not args.compute_val_loss:
validation_generator = None
elif args.compute_val_loss and validation_generator is None:
raise ValueError(
'When you have no validation data, you should not specify --compute-val-loss.'
)
# start training
return model.fit_generator(generator=train_generator,
steps_per_epoch=args.steps,
initial_epoch=0,
epochs=args.epochs,
verbose=1,
callbacks=callbacks,
workers=args.workers,
use_multiprocessing=args.multiprocessing,
max_queue_size=args.max_queue_size,
validation_data=validation_generator)
BatchNormalization()
model.add(Dropout(0.2))
model.add(Dense(10))
# model.add(Convolution2D(10,3,3, border_mode='same'))
# model.add(GlobalAveragePooling2D())
model.add(Activation('softmax'))
model.summary()
# In[42]:
model.compile(optimizer=Adam(), loss='categorical_crossentropy',
metrics=['accuracy']
)
# In[44]:
h = model.fit(X_train, Y_train, epochs=2)
# In[64]:
test_loss, test_accuracy = model.evaluate(X_test, Y_test)
print(test_accuracy)
def fit_path_mod(self, train_data, test_data, epochs=50, lr=1e-3,\
checkpoint_period = 0, save_mod=True):
"""
Trains the path model
Parameters
----------
train_data : dictionary
training data dictionary.
test_data : dictionary
test data dictionary.
epochs: int
number of training epochs
lr: scalar
learning rate
checkpoint_period: int
period in epochs for saving the model checkpoints.
A value of 0 indicates that checkpoints are not be saved.
save_mod: boolean
Indicates if model is to be saved
"""
# Get the link state
ls_tr = train_data['link_state']
ls_ts = test_data['link_state']
los_tr = (ls_tr == LinkState.los_link)
los_ts = (ls_ts == LinkState.los_link)
# Extract the links that are in LOS or NLOS
Itr = np.where(ls_tr != LinkState.no_link)[0]
Its = np.where(ls_ts != LinkState.no_link)[0]
# Fit and transform the condition data
Utr = self.transform_cond(\
train_data['dvec'][Itr], train_data['rx_type'][Itr],\
los_tr[Itr], fit=True)
Uts = self.transform_cond(\
test_data['dvec'][Its], test_data['rx_type'][Its],\
los_ts[Its])
# Fit and transform the data
Xtr = self.transform_data(\
train_data['dvec'][Itr],\
train_data['nlos_pl'][Itr,:self.npaths_max],\
train_data['nlos_ang'][Itr,:self.npaths_max,:],\
train_data['nlos_dly'][Itr,:self.npaths_max], fit=True)
Xts = self.transform_data(\
test_data['dvec'][Its],\
test_data['nlos_pl'][Its,:self.npaths_max],\
test_data['nlos_ang'][Its,:self.npaths_max,:],\
test_data['nlos_dly'][Its,:self.npaths_max])
# Save the pre-processor
if save_mod:
self.save_path_preproc()
# Create the checkpoint callback
batch_size = 100
if (checkpoint_period > 0):
save_freq = checkpoint_period*int(np.ceil(Xtr.shape[0]/batch_size))
if not os.path.exists(self.model_dir):
os.makedirs(self.model_dir)
cp_path = os.path.join(self.model_dir, 'path_weights.{epoch:03d}.h5')
callbacks = [tf.keras.callbacks.ModelCheckpoint(\
filepath=cp_path, save_weights_only=True,save_freq=save_freq)]
else:
callbacks = []
# Fit the model
opt = Adam(lr=lr)
self.path_mod.vae.compile(opt)
self.path_hist = self.path_mod.vae.fit(\
[Xtr,Utr], batch_size=batch_size, epochs=epochs,\
validation_data=([Xts,Uts],None),\
callbacks=callbacks)
# Save the history
hist_path = os.path.join(self.model_dir, 'path_train_hist.p')
with open(hist_path,'wb') as fp:
pickle.dump(self.path_hist.history, fp)
# Save the weights model
if save_mod:
self.save_path_model()
np.random.seed(seed=11)
with open('series_85177_500_stride50.pkl', 'rb') as f:
segments = pickle.load(f)
segments = zscore(segments).astype(np.float32) # standardize
deep_model = Sequential(name="LSTM-autoencoder")
deep_model.add(CuDNNGRU(50, input_shape=(500, 1), return_sequences=False))
#deep_model.add(CuDNNGRU(100, return_sequences=False))
deep_model.add(Dense(20, activation=None))
deep_model.add(RepeatVector(500))
#deep_model.add(CuDNNGRU(100, return_sequences=True))
deep_model.add(CuDNNGRU(50, return_sequences=True))
deep_model.add(TimeDistributed(Dense(1)))
deep_model.compile(optimizer=Adam(lr=5e-3, clipnorm=1.0), loss='mse')
#deep_model.load_weights("model_weights/lstm_autoencoder_2020-01-09_18-20-21.h5")
training_time_stamp = datetime.datetime.now(
tz=pytz.timezone('Europe/London')).strftime("%Y-%m-%d_%H-%M-%S")
CB = EarlyStopping(monitor='val_loss',
min_delta=1e-4,
patience=100,
verbose=1,
mode='auto')
MC = ModelCheckpoint(
'model_weights/lstm_autoencoder_{}.h5'.format(training_time_stamp),
monitor='val_loss',
mode="auto",
# load the MNIST dataset
print("[INFO] loading MNIST dataset...")
((trainX, _), (testX, _)) = mnist.load_data()
# add a channel dimension to every image in the dataset, then scale
# the pixel intensities to the range [0, 1]
trainX = np.expand_dims(trainX, axis=-1)
testX = np.expand_dims(testX, axis=-1)
trainX = trainX.astype("float32") / 255.0
testX = testX.astype("float32") / 255.0
# construct our convolutional autoencoder
print("[INFO] building autoencoder...")
(encoder, decoder, autoencoder) = ConvAutoencoder.build(28, 28, 1)
opt = Adam(lr=1e-3)
autoencoder.compile(loss="mse", optimizer=opt)
# train the convolutional autoencoder
H = autoencoder.fit(trainX,
trainX,
validation_data=(testX, testX),
epochs=EPOCHS,
batch_size=BS)
# construct a plot that plots and saves the training history
N = np.arange(0, EPOCHS)
plt.style.use("ggplot")
plt.figure()
plt.plot(N, H.history["loss"], label="train_loss")
plt.plot(N, H.history["val_loss"], label="val_loss")
# ========================== Create dataset =======================
feature_columns, train, test = create_criteo_dataset(file=file,
embed_dim=embed_dim,
read_part=read_part,
sample_num=sample_num,
test_size=test_size)
train_X, train_y = train
test_X, test_y = test
# ============================Build Model==========================
mirrored_strategy = tf.distribute.MirroredStrategy()
with mirrored_strategy.scope():
model = PNN(feature_columns, hidden_units, dnn_dropout)
model.summary()
# =========================Compile============================
model.compile(loss=binary_crossentropy,
optimizer=Adam(learning_rate=learning_rate),
metrics=[AUC()])
# ============================model checkpoint======================
# check_path = 'save/pnn_weights.epoch_{epoch:04d}.val_loss_{val_loss:.4f}.ckpt'
# checkpoint = tf.keras.callbacks.ModelCheckpoint(check_path, save_weights_only=True,
# verbose=1, period=5)
# ===========================Fit==============================
model.fit(
train_X,
train_y,
epochs=epochs,
callbacks=[
EarlyStopping(monitor='val_loss',
patience=2,
restore_best_weights=True)
], # checkpoint
def __init__(self, env,
actor=None,
critic=None,
buffer=None,
action_bound_range=1,
max_buffer_size = 100000,
batch_size = 64,
replay = 'uniform',
max_time_steps = 1000,
tow = 0.001,
discount_factor = 0.99,
explore_time = 1000,
actor_learning_rate = 0.0001,
critic_learning_rate = 0.001,
dtype = 'float32',
n_episodes = 1000,
verbose = True,
plot = False,
model_save_freq = 10):
'''env , # Gym environment with continous action space
actor(None), # Tensorflow/keras model
critic (None), # Tensorflow/keras model
buffer (None), # pre-recorded buffer
action_bound_range=1,
max_buffer_size =10000, # maximum transitions to be stored in buffer
batch_size =64, # batch size for training actor and critic networks
max_time_steps = 1000 ,# no of time steps per epoch
tow = 0.001, # for soft target update
discount_factor = 0.99,
explore_time = 1000, # time steps for random actions for exploration
actor_learning_rate = 0.0001,
critic_learning_rate = 0.001
dtype = 'float32',
n_episodes = 1000 ,# no of episodes to run
reward_plot = True ,# (bool) to plot reward progress per episode
model_save = 1) # epochs to save models and buffer'''
#############################################
# --------------- Parametres-----------------#
#############################################
self.model_save_freq = model_save_freq
self.max_buffer_size = max_buffer_size
self.batch_size = batch_size
self.T = max_time_steps ## Time limit for a episode
self.tow = tow ## Soft Target Update
self.gamma = discount_factor ## discount factor
# self.target_update_freq = 10 ## frequency for updating target weights
self.explore_time = explore_time
self.act_learning_rate = actor_learning_rate
self.critic_learning_rate = critic_learning_rate
self.dflt_dtype = dtype
self.n_episodes = n_episodes
self.action_bound_range = action_bound_range
self.plot = plot
self.verbose = verbose
self.actor_opt = Adam(self.act_learning_rate)
self.critic_opt = Adam(self.critic_learning_rate)
self.r, self.l, self.qlss = [], [], []
self.env = env
self.observ_min = self.env.observation_space.low
self.observ_max = self.env.observation_space.high
action_dim = 1
state_dim = len(env.reset())
if buffer is not None:
print('using loaded models')
self.buffer = buffer
self.actor = actor
self.critic = critic
else:
if replay == 'prioritized': self.buffer = Prioritized_experience_replay(max_buffer_size, batch_size, dtype)
else : self.buffer = Replay_Buffer( max_buffer_size, batch_size, dtype)
self.actor = _actor_network(state_dim, action_dim,action_bound_range).model()
self.critic = _critic_network(state_dim, action_dim).model()
self.actor_target = _actor_network(state_dim, action_dim,action_bound_range).model()
self.actor_target.set_weights(self.actor.get_weights())
self.critic_target = _critic_network(state_dim, action_dim).model()
self.critic.compile(loss='mse', optimizer=self.critic_opt)
self.critic_target.set_weights(self.critic.get_weights())
def xception(model_config,
input_shape,
metrics,
n_classes,
mixed_precision=False,
output_bias=None):
'''
Defines a model based on a pretrained Xception for multiclass US classification.
:param model_config: A dictionary of parameters associated with the model architecture
:param input_shape: The shape of the model input
:param metrics: Metrics to track model's performance
:param n_classes: # of classes in data
:param mixed_precision: Whether to use mixed precision (use if you have GPU with compute capacity >= 7.0)
:param output_bias: bias initializer of output layer
:return: a Keras Model object with the architecture defined in this method
'''
# Set hyperparameters
nodes_dense0 = model_config['NODES_DENSE0']
nodes_dense1 = model_config['NODES_DENSE1']
lr = model_config['LR']
dropout = model_config['DROPOUT']
l2_lambda = model_config['L2_LAMBDA']
optimizer = Adam(learning_rate=lr)
frozen_layers = model_config['FROZEN_LAYERS']
print("MODEL CONFIG: ", model_config)
if mixed_precision:
tf.train.experimental.enable_mixed_precision_graph_rewrite(optimizer)
if output_bias is not None:
output_bias = Constant(output_bias) # Set initial output bias
# Start with pretrained Xception
X_input = Input(input_shape, name='input')
base_model = Xception(include_top=False,
weights='imagenet',
input_shape=input_shape,
input_tensor=X_input)
# Freeze desired conv layers set in config.yml
for layers in range(len(frozen_layers)):
layer2freeze = frozen_layers[layers]
print('Freezing layer: ' + str(layer2freeze))
base_model.layers[layer2freeze].trainable = False
# Add regularization to Xception conv layers
for layer_idx in model_config['L2_LAYERS']:
if base_models.layers[layer_idx].trainable:
setattr(base_models.layers[layer_idx], 'activity_regularizer',
l2(l2_lambda))
print('Adding regularization to: ' +
str(base_models.layers[layer_idx]))
X = base_model.output
# Add custom top layers
X = GlobalAveragePooling2D()(X)
X = Dropout(dropout)(X)
X = Dense(n_classes, bias_initializer=output_bias, name='logits')(X)
Y = Activation('softmax', dtype='float32', name='output')(X)
# Set model loss function, optimizer, metrics.
model = Model(inputs=X_input, outputs=Y)
model.summary()
model.compile(loss='categorical_crossentropy',
optimizer=optimizer,
metrics=metrics)
return model
verbose=1,
save_best_only=True,
save_weights_only=True)
]
batchSize = 25
data = MiccaiDataset('/datasets/miccai/dataset',
255, (256, 256),
batch_size=batchSize)
config = tf.ConfigProto()
config.gpu_options.allow_growth = True
sess = tf.Session(config=config)
set_session(sess)
net = Unet12()
model = net.build_unet(tf.keras.layers.Input((256, 256, 3)))
model.summary()
model.compile(loss='binary_crossentropy',
optimizer=Adam(0.001),
metrics=['accuracy'])
datagen_train = data.image_train_generator()
datagen_val = data.image_val_generator()
model.fit_generator(datagen_train,
epochs=45,
steps_per_epoch=data.data_train_len(),
callbacks=callbacks_l,
validation_steps=data.data_val_len(),
validation_data=datagen_val)
model.save_weights(save_path)
def custom_resnet(model_config,
input_shape,
metrics,
n_classes,
mixed_precision=False,
output_bias=None):
'''
Defines a deep convolutional neural network model with residual connections for multiclass image classification.
:param model_config: A dictionary of parameters associated with the model architecture
:param input_shape: The shape of the model input
:param metrics: Metrics to track model's performance
:param n_classes: # of classes in data
:param mixed_precision: Whether to use mixed precision (use if you have GPU with compute capacity >= 7.0)
:param output_bias: bias initializer of output layer
:return: a Keras Model object with the architecture defined in this method
'''
# Set hyperparameters
nodes_dense0 = model_config['NODES_DENSE0']
nodes_dense1 = model_config['NODES_DENSE1']
lr = model_config['LR']
dropout = model_config['DROPOUT']
l2_lambda = model_config['L2_LAMBDA']
optimizer = Adam(learning_rate=lr)
init_filters = model_config['INIT_FILTERS']
filter_exp_base = model_config['FILTER_EXP_BASE']
res_blocks = model_config['RES_BLOCKS']
kernel_size = eval(model_config['KERNEL_SIZE'])
max_pool_size = eval(model_config['MAXPOOL_SIZE'])
strides = eval(model_config['STRIDES'])
print("MODEL CONFIG: ", model_config)
pad = kernel_size[0] // 2
if mixed_precision:
tf.train.experimental.enable_mixed_precision_graph_rewrite(optimizer)
if output_bias is not None:
output_bias = Constant(output_bias) # Set initial output bias
# Input layer
X_input = Input(input_shape)
X = X_input
X = ZeroPadding2D((pad, pad))(X)
# Initialize the model with a convolutional layer
X = Conv2D(init_filters, (7, 7),
strides=strides,
name='conv0',
kernel_initializer='he_uniform')(X)
X = BatchNormalization(axis=3, name='bn_conv0')(X)
X = LeakyReLU()(X)
X = MaxPool2D(max_pool_size, padding='same', name='maxpool0')(X)
# Add residual blocks
for i in range(res_blocks):
f1 = f2 = init_filters * (filter_exp_base**i)
f3 = init_filters * (filter_exp_base**(i + 2))
X = convolutional_block(X,
kernel_size=kernel_size,
filters=[f1, f2, f3],
stage=(i + 1),
block='a',
s=strides)
X = identity_block(X,
kernel_size=kernel_size,
filters=[f1, f2, f3],
stage=(i + 1),
block='b')
X = identity_block(X,
kernel_size=kernel_size,
filters=[f1, f2, f3],
stage=(i + 1),
block='c')
# Add fully connected layers
X = AveragePooling2D(strides, name='avgpool0')(X)
X = Flatten()(X)
X = Dropout(dropout)(X)
X = Dense(nodes_dense0,
kernel_initializer='he_uniform',
activity_regularizer=l2(l2_lambda),
name='fc0')(X)
X = LeakyReLU()(X)
X = Dropout(dropout)(X)
X = Dense(n_classes, bias_initializer=output_bias)(X)
Y = Activation('softmax', dtype='float32', name='output')(X)
# Set model loss function, optimizer, metrics.
model = Model(inputs=X_input, outputs=Y)
model.summary()
model.compile(loss='categorical_crossentropy',
optimizer=optimizer,
metrics=metrics)
return model
def optimizer(self): # pylint: disable=no-self-use
if self.learning_rate:
return Adam(lr=self.learning_rate)
else:
return Adam()
def resnet101v2(model_config,
input_shape,
metrics,
n_classes,
mixed_precision=False,
output_bias=None):
'''
Defines a model based on a pretrained ResNet50V2 for multiclass US classification.
:param model_config: A dictionary of parameters associated with the model architecture
:param input_shape: The shape of the model input
:param metrics: Metrics to track model's performance
:param n_classes: # of classes in data
:param mixed_precision: Whether to use mixed precision (use if you have GPU with compute capacity >= 7.0)
:param output_bias: bias initializer of output layer
:return: a Keras Model object with the architecture defined in this method
'''
# Set hyperparameters
nodes_dense0 = model_config['NODES_DENSE0']
nodes_dense1 = model_config['NODES_DENSE1']
lr = model_config['LR']
dropout = model_config['DROPOUT']
l2_lambda = model_config['L2_LAMBDA']
optimizer = Adam(learning_rate=lr)
print("MODEL CONFIG: ", model_config)
if mixed_precision:
tf.train.experimental.enable_mixed_precision_graph_rewrite(optimizer)
if output_bias is not None:
output_bias = Constant(output_bias) # Set initial output bias
# Start with pretrained ResNet101V2
X_input = Input(input_shape, name='input')
base_model = ResNet101V2(include_top=False,
weights='imagenet',
input_shape=input_shape,
input_tensor=X_input)
X = base_model.output
# Add custom top layers
X = GlobalAveragePooling2D()(X)
X = Dropout(dropout)(X)
X = Dense(nodes_dense0,
kernel_initializer='he_uniform',
activation='relu',
activity_regularizer=l2(l2_lambda))(X)
X = Dropout(dropout)(X)
X = Dense(nodes_dense1,
kernel_initializer='he_uniform',
activation='relu',
activity_regularizer=l2(l2_lambda))(X)
X = Dense(n_classes, bias_initializer=output_bias)(X)
Y = Activation('softmax', dtype='float32', name='output')(X)
# Set model loss function, optimizer, metrics.
model = Model(inputs=X_input, outputs=Y)
model.summary()
model.compile(loss='categorical_crossentropy',
optimizer=optimizer,
metrics=metrics)
return model
def train(model,
n_classes=1,
batch_size=1,
sample_shape=(128, 128, 128),
epochs=10,
save_model=True,
validate=True,
train_name="",
custom_train_loop=True,
train_debug=False,
reduce_lr=False,
start_lr=5e-4,
dataset_load_method=None,
shuffle_order=True,
normalise_input=True,
remove_outliers=True,
transform_angle=False,
transform_position="normal",
skip_empty=True,
examples_per_load=1,
use_optimizer="adam",
mean_loss_of_batch="",
schedule_drop="",
schedule_epochs_drop="",
notes="",
**model_kwargs):
start_time = time.perf_counter()
debug = ""
if train_debug:
debug = "debug_"
get_position, get_slice = False, False
commit_id = ""
if model == "tiny":
from Segmentation.model.vnet_tiny import VNet_Tiny
vnet = VNet_Tiny(1, n_classes, **model_kwargs)
model_path = "Segmentation/model/vnet_tiny.py"
commit_id = get_git_file_short_hash(model_path)
elif model == "small":
from Segmentation.model.vnet_small import VNet_Small
vnet = VNet_Small(1, n_classes, **model_kwargs)
model_path = "Segmentation/model/vnet_small.py"
commit_id = get_git_file_short_hash(model_path)
elif model == "small_relative":
from Segmentation.model.vnet_small_relative import VNet_Small_Relative
vnet = VNet_Small_Relative(1, n_classes, **model_kwargs)
get_position = True
model_path = "Segmentation/model/vnet_small_relative.py"
commit_id = get_git_file_short_hash(model_path)
elif model == "slice":
from Segmentation.model.vnet_slice import VNet_Slice
vnet = VNet_Slice(1, n_classes, **model_kwargs)
get_slice = True
model_path = "Segmentation/model/vnet_slice.py"
commit_id = get_git_file_short_hash(model_path)
elif model == "large":
from Segmentation.model.vnet_large import VNet_Large
vnet = VNet_Large(1, n_classes, **model_kwargs)
model_path = "Segmentation/model/vnet_large.py"
commit_id = get_git_file_short_hash(model_path)
elif model == "large_relative":
from Segmentation.model.vnet_large_relative import VNet_Large_Relative
vnet = VNet_Large_Relative(1, n_classes, **model_kwargs)
get_position = True
model_path = "Segmentation/model/vnet_large_relative.py"
commit_id = get_git_file_short_hash(model_path)
else:
raise NotImplementedError()
if not os.path.exists('ckpt'):
os.makedirs('ckpt')
if not os.path.exists('ckpt/checkpoints'):
os.makedirs('ckpt/checkpoints')
now = datetime.now()
now_time = now.strftime('%Y_%m_%d-%H_%M_%S')
if not os.path.exists(
f'ckpt/checkpoints/{debug}train_session_{now_time}_{model}'):
os.makedirs(
f'ckpt/checkpoints/{debug}train_session_{now_time}_{model}')
if custom_train_loop:
if not os.path.exists(
f'ckpt/checkpoints/{debug}train_session_{now_time}_{model}/progress'
):
os.makedirs(
f'ckpt/checkpoints/{debug}train_session_{now_time}_{model}/progress'
)
setup_gpu()
steps = None
vsteps = None
if dataset_load_method == "tf":
get_slice = False
if model == "slice":
get_slice = True
from Segmentation.utils.data_loader_3d_tf import get_dataset
tdataset, steps = get_dataset(file_path="t",
sample_shape=sample_shape,
remove_outliers=remove_outliers,
transform_angle=False,
transform_position=transform_position,
get_slice=get_slice,
get_position=get_position,
skip_empty=skip_empty)
vdataset, vsteps = get_dataset(file_path="v",
sample_shape=sample_shape,
remove_outliers=remove_outliers,
transform_angle=False,
transform_position=False,
get_slice=get_slice,
get_position=get_position,
skip_empty=skip_empty)
tdataset = tdataset.repeat().batch(batch_size).prefetch(
tf.data.experimental.AUTOTUNE)
vdataset = vdataset.repeat().batch(batch_size).prefetch(
tf.data.experimental.AUTOTUNE)
elif dataset_load_method is None:
get_slice = False
if model == "slice":
get_slice = True
from Segmentation.utils.data_loader_3d import VolumeGenerator
tdataset = VolumeGenerator(batch_size,
sample_shape,
"t",
shuffle_order=shuffle_order,
normalise_input=normalise_input,
remove_outliers=remove_outliers,
transform_angle=False,
transform_position=transform_position,
get_slice=get_slice,
get_position=get_position,
skip_empty=skip_empty,
examples_per_load=examples_per_load,
train_debug=train_debug)
vdataset = VolumeGenerator(batch_size,
sample_shape,
"v",
shuffle_order=shuffle_order,
normalise_input=normalise_input,
remove_outliers=remove_outliers,
transform_angle=False,
transform_position=False,
get_slice=get_slice,
get_position=get_position,
skip_empty=skip_empty,
examples_per_load=examples_per_load,
train_debug=train_debug)
from Segmentation.utils.losses import bce_dice_loss, dice_loss
from Segmentation.utils.losses import tversky_loss, precision, recall
loss_name = ""
if n_classes == 1:
loss_func = dice_loss
loss_name = 'dice loss'
from tensorflow.keras.losses import binary_crossentropy
cross_entropy = binary_crossentropy
cross_entropy_name = 'binary_crossentropy'
else:
loss_func = tversky_loss
loss_name = "tversky loss"
from tensorflow.keras.losses import categorical_crossentropy
cross_entropy = categorical_crossentropy
cross_entropy_name = 'categorical_crossentropy'
metrics = {}
metrics['loss'] = {
'loss func': loss_func,
'history': [],
'val history': []
}
metrics[cross_entropy_name] = {
'loss func': cross_entropy,
'history': [],
'val history': []
}
metrics['bce_dice_loss'] = {
'loss func': bce_dice_loss,
'history': [],
'val history': []
}
metrics['precision'] = {
'loss func': precision,
'history': [],
'val history': []
}
metrics['recall'] = {'loss func': recall, 'history': [], 'val history': []}
save_models_callback = tf.keras.callbacks.ModelCheckpoint(
filepath=
f'ckpt/checkpoints/{debug}train_session_{now_time}_{model}/chkp/model_'
+ '{epoch}',
verbose=1)
if custom_train_loop:
if use_optimizer == "adam":
optimizer = Adam(learning_rate=start_lr)
elif use_optimizer == "adadelta":
optimizer = tf.keras.optimizers.Adadelta(learning_rate=1e-4,
rho=0.95)
elif use_optimizer == "adam_schedule":
assert dataset_load_method is None, "Need to be using data generator loader"
assert schedule_drop != "", "Schedule drop needs to be set when using schedule"
assert schedule_epochs_drop != "", "Schedule epochs drop needs to be set when using schedule"
steps_per_epoch = int(
len(VolumeGenerator.get_paths("t")) / batch_size)
lr_schedule = LearningRateSchedule(
steps_per_epoch=steps_per_epoch,
initial_learning_rate=start_lr,
drop=schedule_drop,
epochs_drop=schedule_epochs_drop)
optimizer = Adam(learning_rate=lr_schedule)
if mean_loss_of_batch == "":
mean_loss_of_batch = False
for epoch in range(epochs):
epoch_time = time.perf_counter()
metrics_epoch_avg = {}
for metric in metrics:
metrics_epoch_avg[metric] = {
'loss': tf.keras.metrics.Mean(),
'val_loss': tf.keras.metrics.Mean()
}
for x, y in tdataset:
with tf.GradientTape() as tape:
y_ = vnet(x, training=True)
loss_value = loss_func(y_true=y, y_pred=y_)
if mean_loss_of_batch:
loss_value = tf.reduce_mean(loss_value)
grads = tape.gradient(loss_value, vnet.trainable_variables)
optimizer.apply_gradients(zip(grads, vnet.trainable_variables))
for idx, metric in enumerate(metrics):
if idx == 0: # perform different action for loss_func since it has already been calculated
metrics_epoch_avg[metric]['loss'](loss_value)
continue
value = metrics[metric]['loss func'](
y_true=y,
y_pred=y_) # calculate the training metrics value
metrics_epoch_avg[metric]['loss'](
value) # store the value in keras.metrics.Mean
clr = ""
if use_optimizer == "adam_schedule":
clr = f", lr: {optimizer.get_config()['learning_rate']['config']['current_learning_rate']: .07f}"
# should be in seperate function
y_ = vnet(x, training=False)
if get_position:
x = x[0]
slice_idx = int(x.shape[3] / 2)
store_x = x[0, :, :, slice_idx, 0]
if get_slice:
store_y = y[0, :, :, 0]
store_y_pred = y_[0, :, :, 0]
else:
store_y = y[0, :, :, slice_idx, 0]
store_y_pred = y_[0, :, :, slice_idx, 0]
vidx = 0
for x, y in vdataset:
y_ = vnet(x, training=False)
for metric in metrics:
value = metrics[metric]['loss func'](
y_true=y,
y_pred=y_) # calculate the training metrics value
metrics_epoch_avg[metric]['val_loss'](
value) # store the value in keras.metrics.Mean
if vidx == 0: # should be in seperate function
if get_position:
x = x[0]
store_x_val_0 = x[0, :, :, slice_idx, 0]
if get_slice:
store_y_val_0 = y[0, :, :, 0]
store_y_pred_val_0 = y_[0, :, :, 0]
else:
store_y_val_0 = y[0, :, :, slice_idx, 0]
store_y_pred_val_0 = y_[0, :, :, slice_idx, 0]
vidx += 1
# should be in seperate function
if get_position:
x = x[0]
store_x_val = x[0, :, :, slice_idx, 0]
if get_slice:
store_y_val = y[0, :, :, 0]
store_y_pred_val = y_[0, :, :, 0]
else:
store_y_val = y[0, :, :, slice_idx, 0]
store_y_pred_val = y_[0, :, :, slice_idx, 0]
eloss_str = f" epoch: {epoch:3d}"
for metric in metrics:
epoch_loss_avg = metrics_epoch_avg[metric]['loss'].result()
epoch_val_loss_avg = metrics_epoch_avg[metric][
'val_loss'].result()
metrics[metric]['history'].append(epoch_loss_avg.numpy())
metrics[metric]['val history'].append(
epoch_val_loss_avg.numpy())
eloss_str += f", {metric}: {epoch_loss_avg: .5f}, val {metric}: {epoch_val_loss_avg: .5f}"
print(f"{time.perf_counter() - epoch_time:3.0f} s" + eloss_str)
f, axes = plt.subplots(
3, 3) # convert into for loop and seperate function
axes[0, 0].imshow(store_x, cmap="gray")
axes[0, 0].set_title("train raw image")
axes[0, 1].imshow(store_y, cmap="gray")
axes[0, 1].set_title("train y")
axes[0, 2].imshow(store_y_pred, cmap="gray")
axes[0, 2].set_title("train y pred")
axes[1, 0].imshow(store_x_val, cmap="gray")
axes[1, 0].set_title("val raw image")
axes[1, 1].imshow(store_y_val, cmap="gray")
axes[1, 1].set_title("val y")
axes[1, 2].imshow(store_y_pred_val, cmap="gray")
axes[1, 2].set_title("val y pred")
axes[2, 0].imshow(store_x_val_0, cmap="gray")
axes[2, 0].set_title("val raw image")
axes[2, 1].imshow(store_y_val_0, cmap="gray")
axes[2, 1].set_title("val y")
axes[2, 2].imshow(store_y_pred_val_0, cmap="gray")
axes[2, 2].set_title("val y pred")
for a in axes:
for ax in a:
ax.xaxis.set_visible(False)
ax.yaxis.set_visible(False)
f.tight_layout(rect=[0, 0.01, 1, 0.93])
f.suptitle(
f"{model}: {train_name}\nepoch: {epoch}, loss: {metrics['loss']['history'][-1]: .5f}, val loss: {metrics['loss']['val history'][-1]: .5f}{clr}"
)
plt.savefig(
f"ckpt/checkpoints/{debug}train_session_{now_time}_{model}/progress/train_{epoch:04d}_{now_time}"
)
plt.close('all')
# print(f"plot saved: {time.perf_counter() - epoch_time:.0f}") plots are very fast to save, take ~1 second
vnet.save_weights(
f"ckpt/checkpoints/{debug}train_session_{now_time}_{model}/chkp/ckpt_{epoch:04d}_{now_time}.cktp"
)
else:
metric_list = []
for idx, metric in enumerate(metrics):
if idx == 0:
continue
metric_list.append(metrics[metric]['loss func'])
vnet.compile(optimizer=Adam(lr=start_lr),
loss=loss_func,
metrics=metric_list,
experimental_run_tf_function=True)
callbacks = [save_models_callback]
if reduce_lr:
callbacks.append(
tf.keras.callbacks.ReduceLROnPlateau(monitor='val_loss',
factor=0.5,
patience=10,
min_lr=1e-7,
verbose=1))
if validate:
h = vnet.fit(x=tdataset,
validation_data=vdataset,
callbacks=callbacks,
epochs=epochs,
verbose=1,
steps_per_epoch=steps,
validation_steps=vsteps)
metrics['loss']['val history'] = h.history['val_loss']
metrics[cross_entropy_name]['val history'] = h.history[
f'val_{cross_entropy_name}']
metrics['bce_dice_loss']['val history'] = h.history[
f'val_bce_dice_loss']
else:
h = vnet.fit(x=tdataset,
callbacks=callbacks,
epochs=epochs,
verbose=1)
metrics['loss']['history'] = h.history['loss']
metrics[cross_entropy_name]['history'] = h.history[
f'{cross_entropy_name}']
metrics['bce_dice_loss']['history'] = h.history[f'dice_loss']
time_taken = time.perf_counter() - start_time
roll_period = 5
if roll_period > (epochs - 1):
roll_period = epochs - 1
f, (ax1, ax2, ax3) = plt.subplots(3, 1, sharex=True)
ax1.plot(metrics['loss']['history'], label="loss")
ax1.plot(running_mean(metrics['loss']['history'], roll_period),
label="loss roll")
if validate:
ax1.plot(metrics['loss']['val history'], label="val loss")
ax1.plot(running_mean(metrics['loss']['val history'], roll_period),
label="val loss roll")
ax1.set_xlabel("epoch")
ax1.set_ylabel(loss_name)
ax1.set_title("loss: dice")
ax1.legend()
ax2.plot(metrics[cross_entropy_name]['history'], label="loss")
ax2.plot(running_mean(metrics[cross_entropy_name]['history'], roll_period),
label=f"roll loss")
if validate:
ax2.plot(metrics[cross_entropy_name]['val history'], label=f"val loss")
ax2.plot(running_mean(metrics[cross_entropy_name]['val history'],
roll_period),
label=f"val roll loss")
ax2.set_xlabel("epoch")
ax2.set_ylabel("cross entropy")
ax2.legend()
ax3.plot(metrics['bce_dice_loss']['history'], label="loss")
ax3.plot(running_mean(metrics['bce_dice_loss']['history'], roll_period),
label=f"roll loss")
if validate:
ax3.plot(metrics['bce_dice_loss']['val history'], label=f"val loss")
ax3.plot(running_mean(metrics['bce_dice_loss']['val history'],
roll_period),
label=f"val roll loss")
ax3.set_xlabel("epoch")
ax3.set_ylabel("bce + dice loss")
ax3.legend()
f.tight_layout(rect=[0, 0.01, 1, 0.93])
min_loss = min(metrics['loss']['history'])
min_val_loss = min(metrics['loss']['val history'])
f.suptitle(
f"{model}: {train_name}\nTime: {time_taken/60:.1f} mins, Min Loss: {min_loss:.5f}, Min Val Loss: {min_val_loss:.5f}"
)
plt.savefig(
f"ckpt/checkpoints/{debug}train_session_{now_time}_{model}/train_result_{now_time}"
)
min_roll_loss = min(running_mean(metrics['loss']['history'], roll_period))
min_roll_val_loss = min(
running_mean(metrics['loss']['val history'], roll_period))
min_ce = min(metrics[cross_entropy_name]['history'])
min_val_ce = min(metrics[cross_entropy_name]['val history'])
min_roll_ce = min(
running_mean(metrics[cross_entropy_name]['history'], roll_period))
min_roll_val_ce = min(
running_mean(metrics[cross_entropy_name]['val history'], roll_period))
min_dce = min(metrics['bce_dice_loss']['history'])
min_val_dce = min(metrics['bce_dice_loss']['val history'])
min_roll_dce = min(
running_mean(metrics['bce_dice_loss']['history'], roll_period))
min_roll_val_dce = min(
running_mean(metrics['bce_dice_loss']['val history'], roll_period))
if custom_train_loop:
filenames = glob(
f"ckpt/checkpoints/{debug}train_session_{now_time}_{model}/progress/*"
)
filenames.sort()
images = []
for filename in filenames:
images.append(imageio.imread(filename))
imageio.mimsave(
f'ckpt/checkpoints/{debug}train_session_{now_time}_{model}/progress.gif',
images)
print(f"time taken: {time_taken:.1f}")
df_losses = pd.DataFrame(
data={
'Loss': metrics['loss']['history'],
'Loss Val': metrics['loss']['val history'],
'BCE': metrics[cross_entropy_name]['history'],
'BCE Val': metrics[cross_entropy_name]['val history'],
'BCE + Dice': metrics['bce_dice_loss']['history'],
'BCE + Dice Val': metrics['bce_dice_loss']['val history'],
'Precision': metrics['precision']['history'],
'Precision Val': metrics['precision']['val history'],
'Recall': metrics['recall']['history'],
'Recall Val': metrics['recall']['val history'],
})
df_losses.to_csv(
f"ckpt/checkpoints/{debug}train_session_{now_time}_{model}/vnet_losses.csv",
index_label="epoch")
train_cols = {
'Model': [model],
'Input Shape': [sample_shape],
'Learning Rate': [start_lr],
'Debug': [debug],
'Loss': [loss_name],
'Optimizer': [use_optimizer],
'Num. Epochs': [epochs],
'Examples/Epoch': [examples_per_load],
'Min Loss': [min_loss],
'Min Val Loss': [min_val_loss],
'Min Roll Loss': [min_roll_loss],
'Min Roll Val Loss': [min_roll_val_loss],
'Min BCE': [min_ce],
'Min Val BCE': [min_val_ce],
'Min Roll BCE': [min_roll_ce],
'Min Roll Val BCE': [min_roll_val_ce],
'Min BCE + Dice Loss': [min_dce],
'Min Val BCE + Dice Loss': [min_val_dce],
'Min Roll BCE + Dice Loss': [min_roll_dce],
'Min Roll Val BCE + Dice Loss': [min_roll_val_dce],
'Fit': [custom_train_loop],
'Mean loss of batch': [mean_loss_of_batch],
'Schedule Drop': [schedule_drop],
'Schedule Epochs Drop': [schedule_epochs_drop],
'Train Duration': [time_taken],
'Shuffle Order': [shuffle_order],
'Normalise Input': [normalise_input],
'Remove Outliers': [remove_outliers],
'Skip Empty': [skip_empty],
'transform_angle': [transform_angle],
'transform_position': [transform_position],
'Commit ID': [commit_id],
'Train Name': [f"{debug}train_session_{now_time}_{model}"],
'Model Params': [vnet.params],
'Notes': [notes],
}
df_experiments = pd.DataFrame(data=train_cols)
df_experiments.to_csv("vnet_train_experiments.csv",
index=False,
header=False,
mode='a')
return time_taken
)
# Defining our DQN
dqn = DQNAgent(
model=model,
nb_actions=18,
policy=policy,
memory=memory,
nb_steps_warmup=1000,
gamma=0.5,
target_model_update=1,
delta_clip=0.01,
enable_double_dqn=True,
)
dqn.compile(Adam(lr=0.00025), metrics=["mae"])
# Training
env_player.play_against(
env_algorithm=dqn_training,
opponent=opponent,
env_algorithm_kwargs={
"dqn": dqn,
"nb_steps": NB_TRAINING_STEPS
},
)
model.save("model_%d" % NB_TRAINING_STEPS)
# Evaluation
print("Results against random player:")
env_player.play_against(
#model = SVBRDF_branched()
model = SVBRDF_debugged(9)
learning_rate = 0.00002
#sample = 'E:\workspace_ms_zhiyuan\Data_Deschaintre18\Train_smaller'
train_path = '/vol/bitbucket/zz6117/Data_Deschaintre18/trainBlended'
test_path = '/vol/bitbucket/zz6117/Data_Deschaintre18/testBlended'
print('load_data')
ds = svbrdf_gen(train_path, 8)
#sample_ds = svbrdf_gen(sample,8)
test_ds = svbrdf_gen(test_path, 8)
print(ds.element_spec)
print('finish_loading')
opt = Adam(lr=learning_rate)
model.compile(optimizer=opt, loss=rendering_loss, metrics=['accuracy'])
hitory = model.fit(ds,
verbose=2,
steps_per_epoch=2000,
epochs=8,
callbacks=[tensorboard_callback]) #24884 DisplayCallback()
model.save('/vol/bitbucket/zz6117/DNNreimplement/Model_trained/Model_saved_1')
new_model = tf.keras.models.load_model(
'/vol/bitbucket/zz6117/DNNreimplement/Model_trained/Model_saved_1',
custom_objects={'rendering_loss': rendering_loss},
compile=False)
#tf.saved_model.save(model,'/vol/bitbucket/zz6117/DNNreimplement/Model_trained/Model_saved_1')
#new_model = tf.saved_model.load('/vol/bitbucket/zz6117/DNNreimplement/Model_trained/Model_saved_1')
plt.show()
if __name__ == "__main__":
data = CATDOGDataset()
train_dataset = data.get_train_set()
test_dataset = data.get_test_set()
val_dataset = data.get_val_set()
model_name = f"catvsdog_YesNorm_YesAug_4block"
model = build_model(data.img_shape, data.num_classes)
model.compile(loss=categorical_crossentropy,
optimizer=Adam(learning_rate=LEARNING_RATE),
metrics=["accuracy"])
model.summary()
model_log_dir = os.path.join(LOGS_DIR, f"model_{model_name}")
tb_callback = TensorBoard(log_dir=model_log_dir,
histogram_freq=0,
profile_batch=0,
write_graph=False)
history = model.fit(train_dataset,
epochs=EPOCHS,
batch_size=data.batch_size,
verbose=1,
plt.imshow(predictions[i, :, :, 0] * 127.5 + 127.5)
plt.axis('off')
plt.savefig('{}/image_at_epoch_{:05d}.png'.format(folder, epoch))
plt.close(fig)
# Save generator model
filename = '{}/generator_model_{:05d}.h5'.format(MODELS_FOLDER, epoch)
model.save(filename)
if __name__ == '__main__':
train_dataset = get_dataset()
generator = make_generator_model()
discriminator = make_discriminator_model()
cross_entropy = BinaryCrossentropy(from_logits=True)
G_optimizer = Adam(1e-4)
D_optimizer = Adam(1e-4)
checkpoint = tf.train.Checkpoint(generator_otimizer=G_optimizer,
discriminator_optimizer=D_optimizer,
generator=generator,
discriminator=discriminator)
#train(train_dataset, EPOCHS)
train_forever(train_dataset)
checkpoint.restore(tf.train.latest_checkpoint(checkpoint_dir))
def create_model(
self,
lr,
type="vanilla",
rescale_value=255.0,
):
""" Builds the DQN Agent architecture.
Source:https://cs.corp.google.com/piper///depot/google3/third_party/py/
dopamine/agents/dqn/dqn_agent.py?q=DQN&dr=CSs&l=15
This initializes the model as per the specifications mentioned in the
DQN paper by Deepmind. This is a sequential model implemention of
tf.keras. The compiled model is returned by the Method.
Args:
Returns:
Model: Compiled Model
"""
#with tf.device('/gpu:0'):
self.image_frames = Input(shape=(self.height, self.width,
self.channels))
#self.normalize = Lambda(lambda input: input/255.0)
self.conv1 = Conv2D(
filters=32,
kernel_size=(8, 8),
strides=(4, 4),
activation="relu",
name="conv1")(Lambda(lambda input: input / float(rescale_value))(
self.image_frames))
self.conv2 = Conv2D(filters=64,
kernel_size=(4, 4),
strides=(2, 2),
activation="relu",
name="conv2")(self.conv1)
self.conv3 = Conv2D(filters=64,
kernel_size=(3, 3),
strides=(1, 1),
activation="relu",
name="conv3")(self.conv2)
self.flattened = Flatten(name="flattened")(self.conv3)
self.fully_connected_1 = Dense(units=512,
activation="relu",
name="fully_connected_1")(
self.flattened)
self.q_values = Dense(units=self.actions,
activation="linear",
name="q_values")(self.fully_connected_1)
self.model = Model(inputs=[self.image_frames], outputs=[self.q_values])
self.optimizer = Adam(lr=lr)
if self.loss == "huber":
self.loss = huber_loss
K.get_session().run(tf.global_variables_initializer())
def reward(y_true, y_pred):
return self.reward_tensor
self.model.compile(optimizer=self.optimizer,
loss=self.loss,
metrics=["mse", reward])
return self.model
validation_generator = DataGenerator(segments[int(np.floor(len(segments)*0.95)):], errors[int(np.floor(len(errors)*0.95)):], batch_size=1024)
y_in = Input(shape=(256,1))
y_err = Input(shape=(256,1))
# h_enc = Conv1D(32, 2, activation='relu')(y_in)
# h_enc = Conv1D(32, 8, activation='relu')(h_enc)
# h_enc = CuDNNGRU(512, return_sequences=True)(y_in)
# h_enc = CuDNNGRU(256, return_sequences=True)(y_in)
h_enc = CuDNNLSTM(256, return_sequences=False)(y_in)
# h_enc = Dense(256)(h_enc)
h_enc = Dense(8, activation=None, name='bottleneck')(h_enc)
# h_enc = BatchNormalization()(h_enc)
h_dec = RepeatVector(256)(h_enc)
h_dec = CuDNNLSTM(256, return_sequences=True)(h_dec)
# h_dec = CuDNNGRU(256, return_sequences=True)(h_dec)
h_dec = TimeDistributed(Dense(1))(h_dec)
model = Model(inputs=[y_in, y_err], outputs=h_dec)
model.compile(optimizer=Adam(clipvalue=0.5), loss=chi2(y_err))
# model.load_weights("../../model_weights/model_2020-03-11_20-34-12.h5")
training_time_stamp = datetime.datetime.now(tz=pytz.timezone('Europe/London')).strftime("%Y-%m-%d_%H-%M-%S")
CB = EarlyStopping(monitor='val_loss', min_delta=5e-5, patience=100, verbose=1, mode='auto')
MC = ModelCheckpoint('../../model_weights/model_{}.h5'.format(training_time_stamp), monitor='val_loss', mode="auto", save_best_only=True, verbose=1)
history = model.fit_generator(training_generator, epochs=8000, verbose=2, callbacks = [MC, CB], validation_data=validation_generator)
np.savetxt("training_history/loss_history-{}.txt".format(training_time_stamp), [np.asarray(history.history["loss"]), np.asarray(history.history["val_loss"])], delimiter=",")
#%%
# =============================================================================
# Build classification model - GRU
# =============================================================================
from tensorflow.keras.layers import GRU, Activation
model = Sequential()
model.add(
GRU(50,
input_shape=(X_train.shape[1], X_train.shape[2]),
return_sequences=True))
model.add(GRU(1, return_sequences=False))
model.add(Activation('sigmoid'))
model.summary()
adam = Adam(lr=learningRate)
model.compile(loss='binary_crossentropy', optimizer=adam, metrics=['accuracy'])
chk = ModelCheckpoint(os.path.join(os.getcwd(), 'data', 'best_model.hdf5'),
monitor='val_accuracy',
save_best_only=True,
mode='max',
verbose=1)
history = model.fit(X_train,
y_train,
epochs=150,
batch_size=128,
callbacks=[chk],
validation_data=(X_test, y_test),
verbose=0)
#%%
def mobilenetv2(model_config,
input_shape,
metrics,
n_classes,
mixed_precision=False,
output_bias=None):
'''
Defines a model based on a pretrained MobileNetV2 for multiclass US classification.
:param model_config: A dictionary of parameters associated with the model architecture
:param input_shape: The shape of the model input
:param metrics: Metrics to track model's performance
:param n_classes: # of classes in data
:param mixed_precision: Whether to use mixed precision (use if you have GPU with compute capacity >= 7.0)
:param output_bias: bias initializer of output layer
:return: a Keras Model object with the architecture defined in this method
'''
# Set hyperparameters
nodes_dense0 = model_config['NODES_DENSE0']
nodes_dense1 = model_config['NODES_DENSE1']
lr = model_config['LR']
dropout = model_config['DROPOUT']
l2_lambda = model_config['L2_LAMBDA']
if model_config['OPTIMIZER'] == 'sgd':
optimizer = SGD(learning_rate=lr, momentum=0.9)
else:
optimizer = Adam(learning_rate=lr)
print("MODEL CONFIG: ", model_config)
if mixed_precision:
tf.train.experimental.enable_mixed_precision_graph_rewrite(optimizer)
if output_bias is not None:
output_bias = Constant(output_bias) # Set initial output bias
# Start with pretrained MobileNetV2
X_input = Input(input_shape, name='input')
base_model = MobileNetV2(include_top=False,
weights='imagenet',
input_shape=input_shape,
input_tensor=X_input)
'''
for layer in base_model.layers[:30]:
layer.trainable = False
for layer in base_model.layers[30:]:
layer.trainable = True
if 'keras.layers.Conv2D' in layer._keras_api_names:
setattr(layer, 'activity_regularizer', l2(l2_lambda))
print("Trainable layer with regularization added", layer.name)
'''
X = base_model.output
# Add custom top layers
X = BatchNormalization()(X)
X = GlobalAveragePooling2D()(X)
X = Dropout(dropout)(X)
X = Dense(nodes_dense0,
activation='relu',
activity_regularizer=l2(l2_lambda))(X)
#X = LeakyReLU()(X)
X = BatchNormalization()(X)
#X = Dropout(dropout)(X)
#X = Dense(nodes_dense1, activity_regularizer=l2(l2_lambda))(X)
#X = LeakyReLU()(X)
X = Dropout(dropout)(X)
X = Dense(n_classes, bias_initializer=output_bias)(X)
Y = Activation('softmax', dtype='float32', name='output')(X)
# Set model loss function, optimizer, metrics.
model = Model(inputs=X_input, outputs=Y)
model.summary()
model.compile(loss='categorical_crossentropy',
optimizer=optimizer,
metrics=metrics)
return model
#model.add(layers.UpSampling2D((2,2)))
#model.add(layers.UpSampling2D((2,2)))
model.add(base_model)
model.add(layers.Flatten())
#model.add(layers.BatchNormalization())
#model.add(layers.Dense(128, activation='relu'))
#model.add(layers.Dropout(0.5))
#model.add(layers.BatchNormalization())
#model.add(layers.Dense(64, activation='relu'))
#model.add(layers.Dropout(0.5))
#model.add(layers.BatchNormalization())
model.add(layers.Dense(
10, activation='softmax')) #, kernel_initializer='he_uniform'))
model.compile(loss='categorical_crossentropy',
optimizer=Adam(lr=lr_schedule(0)),
metrics=['accuracy'])
#model.fit(x_train, y_train, epochs=1, batch_size=20, validation_data=(x_test, y_test))
#def get_flops(model):
# run_meta = tf.RunMetadata()
# opts = tf.profiler.ProfileOptionBuilder.float_operation()
#
# # We use the Keras session graph in the call to the profiler.
# flops = tf.profiler.profile(graph=K.get_session().graph,
# run_meta=run_meta, cmd='op', options=opts)
#
# return flops.total_float_ops # Prints the "flops" of the model.
#
#pdb.set_trace()
def r2udensenet():
inputs = Input((image_row, image_col, image_depth))
conv1 = Conv2D(32, (3, 3), activation='relu', padding='same')(inputs)
reconv1 = rec_layer(conv1, 32)
concinter1 = concatenate([inputs, reconv1], axis=3)
conv1 = Conv2D(32, (3, 3), activation='relu', padding='same')(concinter1)
reconv1 = rec_layer(conv1, 32)
conc1 = concatenate([inputs, reconv1], axis=3)
pool1 = MaxPooling2D(pool_size=(2, 2))(conc1)
conv2 = Conv2D(64, (3, 3), activation='relu', padding='same')(pool1)
reconv2 = rec_layer(conv2, 64)
concinter2 = concatenate([conv2, reconv2], axis=3)
conv2 = Conv2D(64, (3, 3), activation='relu', padding='same')(concinter2)
reconv2 = rec_layer(conv2, 64)
conc2 = concatenate([pool1, reconv2], axis=3)
pool2 = MaxPooling2D(pool_size=(2, 2))(conc2)
conv3 = Conv2D(128, (3, 3), activation='relu', padding='same')(pool2)
reconv3 = rec_layer(conv3, 128)
concinter3 = concatenate([conv3, reconv3], axis=3)
conv3 = Conv2D(128, (3, 3), activation='relu', padding='same')(concinter3)
reconv3 = rec_layer(conv3, 128)
conc3 = concatenate([pool2, reconv3], axis=3)
pool3 = MaxPooling2D(pool_size=(2, 2))(conc3)
conv4 = Conv2D(256, (3, 3), activation='relu', padding='same')(pool3)
reconv4 = rec_layer(conv4, 256)
concinter3 = concatenate([conv4, reconv4], axis=3)
conv4 = Conv2D(256, (3, 3), activation='relu', padding='same')(concinter3)
reconv4 = rec_layer(conv4, 256)
conc4 = concatenate([pool3, reconv4], axis=3)
drop4 = Dropout(0.5)(conc4)
pool4 = MaxPooling2D(pool_size=(2, 2))(drop4)
conv5 = Conv2D(512, (3, 3), activation='relu', padding='same')(pool4)
reconv5 = rec_layer(conv5, 512)
concinter3 = concatenate([conv5, reconv5], axis=3)
conv5 = Conv2D(512, (3, 3), activation='relu', padding='same')(concinter3)
reconv5 = rec_layer(conv5, 512)
conc5 = concatenate([pool4, reconv5], axis=3)
drop5 = Dropout(0.5)(conc5)
up6 = concatenate([
Conv2DTranspose(256,
(2, 2), strides=(2, 2), padding='same')(drop5), conv4
],
axis=3)
conv6 = Conv2D(256, (3, 3), activation='relu', padding='same')(up6)
reconv6 = rec_layer(conv6, 256)
concinter6 = concatenate([conv6, reconv6], axis=3)
conv6 = Conv2D(256, (3, 3), activation='relu', padding='same')(concinter6)
reconv6 = rec_layer(conv6, 256)
conc6 = concatenate([up6, reconv6], axis=3)
up7 = concatenate([
Conv2DTranspose(128,
(2, 2), strides=(2, 2), padding='same')(conc6), conv3
],
axis=3)
conv7 = Conv2D(128, (3, 3), activation='relu', padding='same')(up7)
reconv7 = rec_layer(conv7, 128)
concinter7 = concatenate([conv7, reconv7], axis=3)
conv7 = Conv2D(128, (3, 3), activation='relu', padding='same')(concinter7)
reconv7 = rec_layer(conv7, 128)
conc7 = concatenate([up7, reconv7], axis=3)
up8 = concatenate([
Conv2DTranspose(64,
(2, 2), strides=(2, 2), padding='same')(conc7), conv2
],
axis=3)
conv8 = Conv2D(64, (3, 3), activation='relu', padding='same')(up8)
reconv8 = rec_layer(conv8, 64)
concinter8 = concatenate([conv8, reconv8], axis=3)
conv8 = Conv2D(64, (3, 3), activation='relu', padding='same')(concinter8)
reconv8 = rec_layer(conv8, 64)
conc8 = concatenate([up8, reconv8], axis=3)
up9 = concatenate([
Conv2DTranspose(32,
(2, 2), strides=(2, 2), padding='same')(conc8), conv1
],
axis=3)
conv9 = Conv2D(32, (3, 3), activation='relu', padding='same')(up9)
reconv9 = rec_layer(conv9, 32)
concinter9 = concatenate([conv9, reconv9], axis=3)
conv9 = Conv2D(32, (3, 3), activation='relu', padding='same')(concinter9)
reconv9 = rec_layer(conv9, 32)
conc9 = concatenate([up9, reconv9], axis=3)
conv10 = Conv2D(1, (1, 1), activation='sigmoid')(conc9)
#reconv10 = rec_layer(conv10, 1)
model = Model(inputs=[inputs], outputs=[conv10])
#model.summary()
#model.plot()
model.compile(optimizer=Adam(lr=2e-4),
loss='binary_crossentropy',
metrics=[dice_coef])
pretrained_weights = None
if (pretrained_weights):
model.load_weights(pretrained_weights)
return model
