from tensorflow.python.keras.datasets import mnist
from tensorflow.python.keras import backend
from tensorflow.python.keras import callbacks
from util import *
from modelConfig import *
from time import time
conf = tf.compat.v1.ConfigProto(intra_op_parallelism_threads=1,
inter_op_parallelism_threads=1)
sess = tf.compat.v1.Session(config=conf)
#callbacks
epoch = getEpoch()
tensorboard = callbacks.TensorBoard(log_dir=f'../Logs/{name}')
checkpoint = callbacks.ModelCheckpoint(f"../Models/{name}-{epoch}.h5",
monitor='val_loss',
mode='auto',
period=10)
# Decoder - Sequential #
decoder = Sequential([
Dense(intermediate_dim, input_dim=latent_dim, activation='relu'),
Dense(original_dim, activation='sigmoid')
])
x_pred = decoder(z)
vae = Model(inputs=[x, eps], outputs=x_pred)
vae.compile(optimizer='rmsprop', loss=nll)
def train_model_retinanet(model,
dataset,
backbone,
expt='',
test_size=.1,
n_epoch=10,
batch_size=1,
num_gpus=None,
include_masks=False,
panoptic=False,
panoptic_weight=0.1,
transforms=['watershed'],
transforms_kwargs={},
anchor_params=None,
pyramid_levels=['P3', 'P4', 'P5', 'P6', 'P7'],
mask_size=(28, 28),
optimizer=SGD(lr=0.01, decay=1e-6, momentum=0.9, nesterov=True),
log_dir='/data/tensorboard_logs',
model_dir='/data/models',
model_name=None,
sigma=3.0,
alpha=0.25,
gamma=2.0,
score_threshold=0.01,
iou_threshold=0.5,
max_detections=100,
weighted_average=True,
lr_sched=rate_scheduler(lr=0.01, decay=0.95),
rotation_range=0,
flip=True,
shear=0,
zoom_range=0,
compute_map=True,
seed=None,
**kwargs):
"""Train a RetinaNet model from the given backbone
Adapted from:
https://github.com/fizyr/keras-retinanet &
https://github.com/fizyr/keras-maskrcnn
"""
is_channels_first = K.image_data_format() == 'channels_first'
if model_name is None:
todays_date = datetime.datetime.now().strftime('%Y-%m-%d')
data_name = os.path.splitext(os.path.basename(dataset))[0]
model_name = '{}_{}_{}'.format(todays_date, data_name, expt)
model_path = os.path.join(model_dir, '{}.h5'.format(model_name))
loss_path = os.path.join(model_dir, '{}.npz'.format(model_name))
train_dict, test_dict = get_data(dataset, seed=seed, test_size=test_size)
channel_axis = 1 if is_channels_first else -1
n_classes = model.layers[-1].output_shape[channel_axis]
if panoptic:
n_semantic_classes = [layer.output_shape[channel_axis]
for layer in model.layers if 'semantic' in layer.name]
# the data, shuffled and split between train and test sets
print('X_train shape:', train_dict['X'].shape)
print('y_train shape:', train_dict['y'].shape)
print('X_test shape:', test_dict['X'].shape)
print('y_test shape:', test_dict['y'].shape)
print('Output Shape:', model.layers[-1].output_shape)
print('Number of Classes:', n_classes)
if num_gpus is None:
num_gpus = train_utils.count_gpus()
if num_gpus >= 1e6:
batch_size = batch_size * num_gpus
model = train_utils.MultiGpuModel(model, num_gpus)
print('Training on {} GPUs'.format(num_gpus))
# evaluation of model is done on `retinanet_bbox`
if include_masks:
prediction_model = model
else:
prediction_model = retinanet_bbox(
model,
nms=True,
anchor_params=anchor_params,
num_semantic_heads=len(n_semantic_classes),
panoptic=panoptic,
class_specific_filter=False)
retinanet_losses = losses.RetinaNetLosses(sigma=sigma, alpha=alpha, gamma=gamma,
iou_threshold=iou_threshold,
mask_size=mask_size)
def semantic_loss(n_classes):
def _semantic_loss(y_pred, y_true):
return panoptic_weight * losses.weighted_categorical_crossentropy(
y_pred, y_true, n_classes=n_classes)
return _semantic_loss
loss = {
'regression': retinanet_losses.regress_loss,
'classification': retinanet_losses.classification_loss
}
if include_masks:
loss['masks'] = retinanet_losses.mask_loss
if panoptic:
# Give losses for all of the semantic heads
for layer in model.layers:
if 'semantic' in layer.name:
n_classes = layer.output_shape[channel_axis]
loss[layer.name] = semantic_loss(n_classes)
model.compile(loss=loss, optimizer=optimizer)
if num_gpus >= 2:
# Each GPU must have at least one validation example
if test_dict['y'].shape[0] < num_gpus:
raise ValueError('Not enough validation data for {} GPUs. '
'Received {} validation sample.'.format(
test_dict['y'].shape[0], num_gpus))
# When using multiple GPUs and skip_connections,
# the training data must be evenly distributed across all GPUs
num_train = train_dict['y'].shape[0]
nb_samples = num_train - num_train % batch_size
if nb_samples:
train_dict['y'] = train_dict['y'][:nb_samples]
train_dict['X'] = train_dict['X'][:nb_samples]
# this will do preprocessing and realtime data augmentation
datagen = image_generators.RetinaNetGenerator(
# fill_mode='constant', # for rotations
rotation_range=rotation_range,
shear_range=shear,
zoom_range=zoom_range,
horizontal_flip=flip,
vertical_flip=flip)
datagen_val = image_generators.RetinaNetGenerator(
# fill_mode='constant', # for rotations
rotation_range=0,
shear_range=0,
zoom_range=0,
horizontal_flip=0,
vertical_flip=0)
# if 'vgg' in backbone or 'densenet' in backbone:
# compute_shapes = make_shapes_callback(model)
# else:
# compute_shapes = guess_shapes
compute_shapes = guess_shapes
train_data = datagen.flow(
train_dict,
seed=seed,
include_mask_transforms=len(transforms) > 0,
include_masks=include_masks,
panoptic=panoptic,
transforms=transforms,
transforms_kwargs=transforms_kwargs,
pyramid_levels=pyramid_levels,
anchor_params=anchor_params,
compute_shapes=compute_shapes,
batch_size=batch_size)
val_data = datagen_val.flow(
test_dict,
seed=seed,
include_mask_transforms=len(transforms) > 0,
include_masks=include_masks,
panoptic=panoptic,
transforms=transforms,
transforms_kwargs=transforms_kwargs,
pyramid_levels=pyramid_levels,
anchor_params=anchor_params,
compute_shapes=compute_shapes,
batch_size=batch_size)
tensorboard_callback = callbacks.TensorBoard(
log_dir=os.path.join(log_dir, model_name))
# fit the model on the batches generated by datagen.flow()
loss_history = model.fit_generator(
train_data,
steps_per_epoch=train_data.y.shape[0] // batch_size,
epochs=n_epoch,
validation_data=val_data,
validation_steps=val_data.y.shape[0] // batch_size,
callbacks=[
callbacks.LearningRateScheduler(lr_sched),
callbacks.ModelCheckpoint(
model_path, monitor='val_loss', verbose=1,
save_best_only=True, save_weights_only=num_gpus >= 2),
tensorboard_callback,
callbacks.ReduceLROnPlateau(
monitor='loss', factor=0.1,
patience=10, verbose=1,
mode='auto', min_delta=0.0001,
cooldown=0, min_lr=0),
RedirectModel(
Evaluate(val_data,
iou_threshold=iou_threshold,
score_threshold=score_threshold,
max_detections=max_detections,
tensorboard=tensorboard_callback,
weighted_average=weighted_average),
prediction_model),
])
model.save_weights(model_path)
np.savez(loss_path, loss_history=loss_history.history)
if compute_map:
average_precisions = evaluate(
val_data,
prediction_model,
iou_threshold=iou_threshold,
score_threshold=score_threshold,
max_detections=max_detections,
)
# print evaluation
total_instances = []
precisions = []
for label, (average_precision, num_annotations) in average_precisions.items():
print('{:.0f} instances of class'.format(num_annotations),
label, 'with average precision: {:.4f}'.format(average_precision))
total_instances.append(num_annotations)
precisions.append(average_precision)
if sum(total_instances) == 0:
print('No test instances found.')
else:
print('mAP using the weighted average of precisions among classes: {:.4f}'.format(
sum([a * b for a, b in zip(total_instances, precisions)]) / sum(total_instances)))
print('mAP: {:.4f}'.format(sum(precisions) / sum(x > 0 for x in total_instances)))
return model
def train_model_sample(model,
dataset,
expt='',
test_size=.1,
n_epoch=10,
batch_size=32,
num_gpus=None,
transform=None,
window_size=None,
balance_classes=True,
max_class_samples=None,
log_dir='/data/tensorboard_logs',
model_dir='/data/models',
model_name=None,
focal=False,
gamma=0.5,
optimizer=SGD(lr=0.01, decay=1e-6, momentum=0.9, nesterov=True),
lr_sched=rate_scheduler(lr=0.01, decay=0.95),
rotation_range=0,
flip=False,
shear=0,
zoom_range=0,
seed=None,
**kwargs):
is_channels_first = K.image_data_format() == 'channels_first'
if model_name is None:
todays_date = datetime.datetime.now().strftime('%Y-%m-%d')
data_name = os.path.splitext(os.path.basename(dataset))[0]
model_name = '{}_{}_{}'.format(todays_date, data_name, expt)
model_path = os.path.join(model_dir, '{}.h5'.format(model_name))
loss_path = os.path.join(model_dir, '{}.npz'.format(model_name))
train_dict, test_dict = get_data(dataset, test_size=test_size, seed=seed)
n_classes = model.layers[-1].output_shape[1 if is_channels_first else -1]
# the data, shuffled and split between train and test sets
print('X_train shape:', train_dict['X'].shape)
print('y_train shape:', train_dict['y'].shape)
print('X_test shape:', test_dict['X'].shape)
print('y_test shape:', test_dict['y'].shape)
print('Output Shape:', model.layers[-1].output_shape)
print('Number of Classes:', n_classes)
def loss_function(y_true, y_pred):
if isinstance(transform, str) and transform.lower() == 'disc':
return losses.discriminative_instance_loss(y_true, y_pred)
if focal:
return losses.weighted_focal_loss(
y_true, y_pred, gamma=gamma, n_classes=n_classes)
return losses.weighted_categorical_crossentropy(
y_true, y_pred, n_classes=n_classes)
if num_gpus is None:
num_gpus = train_utils.count_gpus()
if num_gpus >= 2:
batch_size = batch_size * num_gpus
model = train_utils.MultiGpuModel(model, num_gpus)
print('Training on {} GPUs'.format(num_gpus))
model.compile(loss=loss_function, optimizer=optimizer, metrics=['accuracy'])
if train_dict['X'].ndim == 4:
DataGenerator = image_generators.SampleDataGenerator
window_size = window_size if window_size else (30, 30)
elif train_dict['X'].ndim == 5:
DataGenerator = image_generators.SampleMovieDataGenerator
window_size = window_size if window_size else (30, 30, 3)
else:
raise ValueError('Expected `X` to have ndim 4 or 5. Got',
train_dict['X'].ndim)
# this will do preprocessing and realtime data augmentation
datagen = DataGenerator(
rotation_range=rotation_range,
shear_range=shear,
zoom_range=zoom_range,
horizontal_flip=flip,
vertical_flip=flip)
# no validation augmentation
datagen_val = DataGenerator(
rotation_range=0,
shear_range=0,
zoom_range=0,
horizontal_flip=0,
vertical_flip=0)
train_data = datagen.flow(
train_dict,
seed=seed,
batch_size=batch_size,
transform=transform,
transform_kwargs=kwargs,
window_size=window_size,
balance_classes=balance_classes,
max_class_samples=max_class_samples)
val_data = datagen_val.flow(
test_dict,
seed=seed,
batch_size=batch_size,
transform=transform,
transform_kwargs=kwargs,
window_size=window_size,
balance_classes=False,
max_class_samples=max_class_samples)
# fit the model on the batches generated by datagen.flow()
loss_history = model.fit_generator(
train_data,
steps_per_epoch=train_data.y.shape[0] // batch_size,
epochs=n_epoch,
validation_data=val_data,
validation_steps=val_data.y.shape[0] // batch_size,
callbacks=[
callbacks.LearningRateScheduler(lr_sched),
callbacks.ModelCheckpoint(
model_path, monitor='val_loss', verbose=1,
save_best_only=True, save_weights_only=num_gpus >= 2),
callbacks.TensorBoard(log_dir=os.path.join(log_dir, model_name))
])
np.savez(loss_path, loss_history=loss_history.history)
return model
train_generator = train_datagen.flow_from_directory(
# This is the target directory
train_dir,
# All images will be resized to 150x150
target_size=(150, 150),
batch_size=20,
# Since we use binary_crossentropy loss, we need binary labels
class_mode='binary')
validation_generator = test_datagen.flow_from_directory(validation_dir,
target_size=(150, 150),
batch_size=20,
class_mode='binary')
tbCallBack = callbacks.TensorBoard(log_dir='./log',
histogram_freq=0,
write_graph=True,
write_images=True)
history = model.fit_generator(train_generator,
steps_per_epoch=100,
epochs=20,
validation_data=validation_generator,
validation_steps=50,
callbacks=[])
model.save('cats_and_dogs_small_1.h5')
acc = history.history['acc']
val_acc = history.history['val_acc']
loss = history.history['loss']
val_loss = history.history['val_loss']
def train_model_siamese_daughter(model,
dataset,
expt='',
test_size=.1,
n_epoch=100,
batch_size=1,
num_gpus=None,
crop_dim=32,
min_track_length=1,
neighborhood_scale_size=10,
features=None,
optimizer=SGD(lr=0.01, decay=1e-6, momentum=0.9, nesterov=True),
log_dir='/data/tensorboard_logs',
model_dir='/data/models',
model_name=None,
focal=False,
gamma=0.5,
lr_sched=rate_scheduler(lr=0.01, decay=0.95),
rotation_range=0,
flip=True,
shear=0,
zoom_range=0,
seed=None,
**kwargs):
is_channels_first = K.image_data_format() == 'channels_first'
if model_name is None:
todays_date = datetime.datetime.now().strftime('%Y-%m-%d')
data_name = os.path.splitext(os.path.basename(dataset))[0]
model_name = '{}_{}_[{}]_neighs={}_epochs={}_seed={}_{}'.format(
todays_date, data_name, ','.join(f[0] for f in sorted(features)),
neighborhood_scale_size, n_epoch, seed, expt)
model_path = os.path.join(model_dir, '{}.h5'.format(model_name))
loss_path = os.path.join(model_dir, '{}.npz'.format(model_name))
print('training on dataset:', dataset)
print('saving model at:', model_path)
print('saving loss at:', loss_path)
train_dict, val_dict = get_data(dataset, mode='siamese_daughters',
seed=seed, test_size=test_size)
# the data, shuffled and split between train and test sets
print('X_train shape:', train_dict['X'].shape)
print('y_train shape:', train_dict['y'].shape)
print('X_test shape:', val_dict['X'].shape)
print('y_test shape:', val_dict['y'].shape)
print('Output Shape:', model.layers[-1].output_shape)
n_classes = model.layers[-1].output_shape[1 if is_channels_first else -1]
def loss_function(y_true, y_pred):
if focal:
return losses.weighted_focal_loss(y_true, y_pred,
gamma=gamma,
n_classes=n_classes,
from_logits=False)
return losses.weighted_categorical_crossentropy(y_true, y_pred,
n_classes=n_classes,
from_logits=False)
if num_gpus is None:
num_gpus = train_utils.count_gpus()
if num_gpus >= 2:
batch_size = batch_size * num_gpus
model = train_utils.MultiGpuModel(model, num_gpus)
print('Training on {} GPUs'.format(num_gpus))
model.compile(loss=loss_function, optimizer=optimizer, metrics=['accuracy'])
print('Using real-time data augmentation.')
# this will do preprocessing and realtime data augmentation
datagen = image_generators.SiameseDataGenerator(
rotation_range=rotation_range,
shear_range=shear,
zoom_range=zoom_range,
horizontal_flip=flip,
vertical_flip=flip)
datagen_val = image_generators.SiameseDataGenerator(
rotation_range=0,
zoom_range=0,
shear_range=0,
horizontal_flip=0,
vertical_flip=0)
total_train_pairs = tracking_utils.count_pairs(train_dict['y'], same_probability=5.0)
total_test_pairs = tracking_utils.count_pairs(val_dict['y'], same_probability=5.0)
train_data = datagen.flow(
train_dict,
seed=seed,
crop_dim=crop_dim,
batch_size=batch_size,
min_track_length=min_track_length,
neighborhood_scale_size=neighborhood_scale_size,
features=features)
val_data = datagen_val.flow(
val_dict,
seed=seed,
crop_dim=crop_dim,
batch_size=batch_size,
min_track_length=min_track_length,
neighborhood_scale_size=neighborhood_scale_size,
features=features)
print('total_train_pairs:', total_train_pairs)
print('total_test_pairs:', total_test_pairs)
print('batch size:', batch_size)
print('validation_steps: ', total_test_pairs // batch_size)
# fit the model on the batches generated by datagen.flow()
loss_history = model.fit_generator(
train_data,
steps_per_epoch=total_train_pairs // batch_size,
epochs=n_epoch,
validation_data=val_data,
validation_steps=total_test_pairs // batch_size,
callbacks=[
callbacks.LearningRateScheduler(lr_sched),
callbacks.ModelCheckpoint(
model_path, monitor='val_loss', verbose=1,
save_best_only=True, save_weights_only=num_gpus >= 2),
callbacks.TensorBoard(log_dir=os.path.join(log_dir, model_name))
])
model.save_weights(model_path)
np.savez(loss_path, loss_history=loss_history.history)
return model
font=dict(
family="Courier New, monospace",
size=18,
color="#7f7f7f"
)
)
)
)
fig.show()
if __name__ == '__main__':
userNet = DQSAVersion2(input_size=config.input_size_user, usernet=True)
userNet.load_weights(path="/home/dorliv/Desktop/DQSAKeras/successful_experiements/three_users/network_central_best_three_users/checkpoint")
logger = get_logger(os.path.join(config.evaluate_log_dir, "evaluate_log"))
Tensorcallback = callbacks.TensorBoard(config.evaluate_log_dir,
write_graph=True, write_images=False)
Tensorcallback.set_model(userNet.model)
env = OneTimeStepEnv()
beta = 10
alpha = 0 # e_greedy
draw_heatmap_flag = False
channelThroughPutPerTstep = initCTP(config.TimeSlotsEvaluate) # init the data structure to view the mean reward at each t
for iteration in range(5):
channelThroughPutMean = 0
loss_value = []
collisonsMean = 0
idle_timesMean = 0
for episode in range(config.Episodes):
heatmap = []
collisons = 0
idle_times = 0
model_name = "SimpleBaseline_Small_FromScratch_RMSProp_Default_L1_1E-1_e_25_is_48_48" ## ENSURE CORRECT
# Images Directory
dir_images = "./data/processed/resized-48-48/" ## ENSURE CORRECT
### INFO FOR TENSORBOARD
# Note: put the tensorboard log into a subdirectory of the main logs folder, i.e. /logs/run_1, /logs/run_2
# This lets tensorboard display their output as separate runs properly. For now we'll just automatically increment run number
dir_tensorboard_logs = "./tensorboard_logs/"
dir_tensorboard_logs = os.path.abspath(dir_tensorboard_logs)
num_tensorboard_runs = len(os.listdir(dir_tensorboard_logs))
dir_tensorboard_logs = dir_tensorboard_logs + "/" + model_name
# Note: make the log directory later, in case the code fails before the training step and the new directory is left empty
callback_tensorboard = callbacks.TensorBoard(log_dir=dir_tensorboard_logs,
write_grads=True,
write_images=True,
histogram_freq=1)
### DATA PREP
# Get images paths & split training/validation
images_summary = pd.read_csv("./results/images_summary.csv")
images_summary_train = images_summary[images_summary.DataRole == "train"]
images_summary_valid = images_summary[images_summary.DataRole == "valid"]
filenames_relative_train = images_summary_train.FileName_Relative.values
filenames_train = dir_images + filenames_relative_train
filenames_relative_valid = images_summary_valid.FileName_Relative.values
filenames_valid = dir_images + filenames_relative_valid
help='pre trained model path.')
FLAGS, unparsed = parser.parse_known_args()
Net_OOP = TilesUnetMirrored()
optimizer = tf.compat.v2.optimizers.Adam(beta_1=0.99)
loader = Loader(batch_size=FLAGS.batch_size)
Net_OOP.compile(optimizer=optimizer,
loss=loss_fn,
metrics=['acc', 'loss', 'val_acc', 'val_loss'])
if not os.path.exists(FLAGS.log_dir):
os.makedirs(FLAGS.log_dir)
if not os.path.exists(FLAGS.model_path):
os.makedirs(FLAGS.model_path)
logger = get_logger(os.path.join(FLAGS.log_dir, "train_log"))
Tensorcallback = callbacks.TensorBoard(FLAGS.log_dir,
write_graph=False,
write_images=False)
Checkpoint = callbacks.ModelCheckpoint(filepath=FLAGS.model_path +
"/checkpoint.hdf5",
monitor='val_acc',
mode='max',
save_best_only=True)
Checkpoint.set_model(Net_OOP)
Tensorcallback.set_model(Net_OOP)
callbacks = {'tensorboard': Tensorcallback, 'checkpoint': Checkpoint}
Net_OOP.fit(logger=logger,
callbacks=callbacks,
epochs=FLAGS.epochs,
steps_per_epoch=config.steps_per_epoch,
val_freq=config.val_freq,
val_steps=config.validation_steps,
model.add(layers.MaxPooling2D(pool_size=(2, 2), strides=(2, 2)))
model.add(layers.Dropout(rate=0.1 + 0.025 * cnn_depth))
model.add(layers.Flatten())
# model.add(layers.Dense(256, activation='relu'))
# model.add(layers.Dense(128, activation='relu'))
model.add(layers.Dense(num_classes, activation='softmax'))
opt_rms = optimizers.RMSprop(lr=0.0005, decay=1e-6)
model.compile(loss=keras.losses.categorical_crossentropy,
optimizer=opt_rms,
metrics=['accuracy'])
# Tensorboard
tensorboard = callbacks.TensorBoard(log_dir="./K_CIFAR_model/{}".format(
datetime.today().strftime('%m-%d__%H-%M-%S')),
histogram_freq=0,
write_graph=True)
tensorboard.set_model(model)
##############################
# Train model
##############################
datagen = preprocessing.image.ImageDataGenerator(
featurewise_center=False,
samplewise_center=False,
featurewise_std_normalization=False,
samplewise_std_normalization=False,
zca_whitening=False,
rotation_range=15,
width_shift_range=0.1,
height_shift_range=0.1,
def train_model_retinanet(model,
dataset,
backbone,
expt='',
test_size=.1,
n_epoch=10,
batch_size=1,
num_gpus=None,
include_masks=False,
mask_size=(28, 28),
optimizer=SGD(lr=0.01,
decay=1e-6,
momentum=0.9,
nesterov=True),
log_dir='/data/tensorboard_logs',
model_dir='/data/models',
model_name=None,
sigma=3.0,
alpha=0.25,
gamma=2.0,
score_threshold=0.01,
iou_threshold=0.5,
max_detections=100,
weighted_average=True,
lr_sched=rate_scheduler(lr=0.01, decay=0.95),
rotation_range=0,
flip=True,
shear=0,
zoom_range=0,
**kwargs):
"""Train a RetinaNet model from the given backbone
Adapted from:
https://github.com/fizyr/keras-retinanet &
https://github.com/fizyr/keras-maskrcnn
"""
is_channels_first = K.image_data_format() == 'channels_first'
if model_name is None:
todays_date = datetime.datetime.now().strftime('%Y-%m-%d')
data_name = os.path.splitext(os.path.basename(dataset))[0]
model_name = '{}_{}_{}'.format(todays_date, data_name, expt)
model_path = os.path.join(model_dir, '{}.h5'.format(model_name))
loss_path = os.path.join(model_dir, '{}.npz'.format(model_name))
train_dict, test_dict = get_data(dataset, mode='conv', test_size=test_size)
n_classes = model.layers[-1].output_shape[1 if is_channels_first else -1]
# the data, shuffled and split between train and test sets
print('X_train shape:', train_dict['X'].shape)
print('y_train shape:', train_dict['y'].shape)
print('X_test shape:', test_dict['X'].shape)
print('y_test shape:', test_dict['y'].shape)
print('Output Shape:', model.layers[-1].output_shape)
print('Number of Classes:', n_classes)
if num_gpus is None:
num_gpus = train_utils.count_gpus()
if num_gpus >= 1e6:
batch_size = batch_size * num_gpus
model = train_utils.MultiGpuModel(model, num_gpus)
print('Training on {} GPUs'.format(num_gpus))
def regress_loss(y_true, y_pred):
# separate target and state
regression = y_pred
regression_target = y_true[..., :-1]
anchor_state = y_true[..., -1]
# filter out "ignore" anchors
indices = tf.where(K.equal(anchor_state, 1))
regression = tf.gather_nd(regression, indices)
regression_target = tf.gather_nd(regression_target, indices)
# compute the loss
loss = losses.smooth_l1(regression_target, regression, sigma=sigma)
# compute the normalizer: the number of positive anchors
normalizer = K.maximum(1, K.shape(indices)[0])
normalizer = K.cast(normalizer, dtype=K.floatx())
return K.sum(loss) / normalizer
def classification_loss(y_true, y_pred):
# TODO: try weighted_categorical_crossentropy
labels = y_true[..., :-1]
# -1 for ignore, 0 for background, 1 for object
anchor_state = y_true[..., -1]
classification = y_pred
# filter out "ignore" anchors
indices = tf.where(K.not_equal(anchor_state, -1))
labels = tf.gather_nd(labels, indices)
classification = tf.gather_nd(classification, indices)
# compute the loss
loss = losses.focal(labels, classification, alpha=alpha, gamma=gamma)
# compute the normalizer: the number of positive anchors
normalizer = tf.where(K.equal(anchor_state, 1))
normalizer = K.cast(K.shape(normalizer)[0], K.floatx())
normalizer = K.maximum(K.cast_to_floatx(1.0), normalizer)
return K.sum(loss) / normalizer
def mask_loss(y_true, y_pred):
def _mask(y_true, y_pred, iou_threshold=0.5, mask_size=(28, 28)):
# split up the different predicted blobs
boxes = y_pred[:, :, :4]
masks = y_pred[:, :, 4:]
# split up the different blobs
annotations = y_true[:, :, :5]
width = K.cast(y_true[0, 0, 5], dtype='int32')
height = K.cast(y_true[0, 0, 6], dtype='int32')
masks_target = y_true[:, :, 7:]
# reshape the masks back to their original size
masks_target = K.reshape(masks_target,
(K.shape(masks_target)[0] *
K.shape(masks_target)[1], height, width))
masks = K.reshape(masks, (K.shape(masks)[0] * K.shape(masks)[1],
mask_size[0], mask_size[1], -1))
# batch size > 1 fix
boxes = K.reshape(boxes, (-1, K.shape(boxes)[2]))
annotations = K.reshape(annotations, (-1, K.shape(annotations)[2]))
# compute overlap of boxes with annotations
iou = overlap(boxes, annotations)
argmax_overlaps_inds = K.argmax(iou, axis=1)
max_iou = K.max(iou, axis=1)
# filter those with IoU > 0.5
indices = tf.where(K.greater_equal(max_iou, iou_threshold))
boxes = tf.gather_nd(boxes, indices)
masks = tf.gather_nd(masks, indices)
argmax_overlaps_inds = tf.gather_nd(argmax_overlaps_inds, indices)
argmax_overlaps_inds = K.cast(argmax_overlaps_inds, 'int32')
labels = K.gather(annotations[:, 4], argmax_overlaps_inds)
labels = K.cast(labels, 'int32')
# make normalized boxes
x1 = boxes[:, 0]
y1 = boxes[:, 1]
x2 = boxes[:, 2]
y2 = boxes[:, 3]
boxes = K.stack([
y1 / (K.cast(height, dtype=K.floatx()) - 1),
x1 / (K.cast(width, dtype=K.floatx()) - 1),
(y2 - 1) / (K.cast(height, dtype=K.floatx()) - 1),
(x2 - 1) / (K.cast(width, dtype=K.floatx()) - 1),
],
axis=1)
# crop and resize masks_target
# append a fake channel dimension
masks_target = K.expand_dims(masks_target, axis=3)
masks_target = tf.image.crop_and_resize(masks_target, boxes,
argmax_overlaps_inds,
mask_size)
# remove fake channel dimension
masks_target = masks_target[:, :, :, 0]
# gather the predicted masks using the annotation label
masks = tf.transpose(masks, (0, 3, 1, 2))
label_indices = K.stack([tf.range(K.shape(labels)[0]), labels],
axis=1)
masks = tf.gather_nd(masks, label_indices)
# compute mask loss
mask_loss = K.binary_crossentropy(masks_target, masks)
normalizer = K.shape(masks)[0] * K.shape(masks)[1] * K.shape(
masks)[2]
normalizer = K.maximum(K.cast(normalizer, K.floatx()), 1)
mask_loss = K.sum(mask_loss) / normalizer
return mask_loss
# if there are no masks annotations, return 0; else, compute the masks loss
return tf.cond(
K.any(K.equal(K.shape(y_true), 0)), lambda: K.cast_to_floatx(0.0),
lambda: _mask(y_true,
y_pred,
iou_threshold=iou_threshold,
mask_size=mask_size))
# evaluation of model is done on `retinanet_bbox`
if include_masks:
prediction_model = model
else:
prediction_model = retinanet_bbox(model,
nms=True,
class_specific_filter=False)
loss = {'regression': regress_loss, 'classification': classification_loss}
if include_masks:
loss['masks'] = mask_loss
model.compile(loss=loss, optimizer=optimizer)
if num_gpus >= 2:
# Each GPU must have at least one validation example
if test_dict['y'].shape[0] < num_gpus:
raise ValueError('Not enough validation data for {} GPUs. '
'Received {} validation sample.'.format(
test_dict['y'].shape[0], num_gpus))
# When using multiple GPUs and skip_connections,
# the training data must be evenly distributed across all GPUs
num_train = train_dict['y'].shape[0]
nb_samples = num_train - num_train % batch_size
if nb_samples:
train_dict['y'] = train_dict['y'][:nb_samples]
train_dict['X'] = train_dict['X'][:nb_samples]
# this will do preprocessing and realtime data augmentation
datagen = image_generators.RetinaNetGenerator(
# fill_mode='constant', # for rotations
rotation_range=rotation_range,
shear_range=shear,
zoom_range=zoom_range,
horizontal_flip=flip,
vertical_flip=flip)
datagen_val = image_generators.RetinaNetGenerator(
# fill_mode='constant', # for rotations
rotation_range=0,
shear_range=0,
zoom_range=0,
horizontal_flip=0,
vertical_flip=0)
if 'vgg' in backbone or 'densenet' in backbone:
compute_shapes = make_shapes_callback(model)
else:
compute_shapes = guess_shapes
train_data = datagen.flow(train_dict,
include_masks=include_masks,
compute_shapes=compute_shapes,
batch_size=batch_size)
val_data = datagen_val.flow(test_dict,
include_masks=include_masks,
compute_shapes=compute_shapes,
batch_size=batch_size)
tensorboard_callback = callbacks.TensorBoard(
log_dir=os.path.join(log_dir, model_name))
# fit the model on the batches generated by datagen.flow()
loss_history = model.fit_generator(
train_data,
steps_per_epoch=train_data.y.shape[0] // batch_size,
epochs=n_epoch,
validation_data=val_data,
validation_steps=val_data.y.shape[0] // batch_size,
callbacks=[
callbacks.LearningRateScheduler(lr_sched),
callbacks.ModelCheckpoint(model_path,
monitor='val_loss',
verbose=1,
save_best_only=True,
save_weights_only=num_gpus >= 2),
tensorboard_callback,
callbacks.ReduceLROnPlateau(monitor='loss',
factor=0.1,
patience=10,
verbose=1,
mode='auto',
min_delta=0.0001,
cooldown=0,
min_lr=0),
RedirectModel(
Evaluate(val_data,
iou_threshold=iou_threshold,
score_threshold=score_threshold,
max_detections=max_detections,
tensorboard=tensorboard_callback,
weighted_average=weighted_average), prediction_model),
])
model.save_weights(model_path)
np.savez(loss_path, loss_history=loss_history.history)
average_precisions = evaluate(
val_data,
prediction_model,
iou_threshold=iou_threshold,
score_threshold=score_threshold,
max_detections=max_detections,
)
# print evaluation
total_instances = []
precisions = []
for label, (average_precision,
num_annotations) in average_precisions.items():
print('{:.0f} instances of class'.format(num_annotations), label,
'with average precision: {:.4f}'.format(average_precision))
total_instances.append(num_annotations)
precisions.append(average_precision)
if sum(total_instances) == 0:
print('No test instances found.')
else:
print(
'mAP using the weighted average of precisions among classes: {:.4f}'
.format(
sum([a * b for a, b in zip(total_instances, precisions)]) /
sum(total_instances)))
print('mAP: {:.4f}'.format(
sum(precisions) / sum(x > 0 for x in total_instances)))
return model
def run(epochs,
num_batches,
batch_size=1,
learning_rate=0.001,
beta1=0.9,
beta2=0.999,
epsilon=1e-08,
save_every=10,
earlyStopping=True,
patience=5,
resume=False):
# Destroy old graph
K.clear_session()
# Initialize batch generators
batch_train = build_features.get_train_batches(batch_size=batch_size)
batch_valid = build_features.get_valid_batches(batch_size=batch_size)
# Create TensorFlow Iterator object
itr_train = build_features.make_iterator(batch_train)
itr_valid = build_features.make_iterator(batch_valid)
# Init callbacks
cbs = list()
# History callback: saves all losses
cbs.append(callbacks.History())
# EarlyStopping callback: stops whenever loss doesn't imporve
if earlyStopping:
cbs.append(
early_stopping.EarlyStopping(monitor='val_binary_accuracy',
baseline=0.8,
patience=patience,
verbose=1))
# ModelCheckpoint callback: saves model every SAVE_EVERY
save_path = paths.checkpoints.multimodal() # ./checkpoints/regnet/train
save_path.parent.mkdir(exist_ok=True, parents=True)
if save_path.exists() and not resume:
save_path.unlink() # deletes file before training
cbs.append(
callbacks.ModelCheckpoint(str(save_path),
save_best_only=True,
period=save_every))
# TensorBoard callback: saves logs for tensorboard
log_path = str(paths.logs.multimodal()) # ./logs/regnet/train
cbs.append(
callbacks.TensorBoard(log_dir=log_path,
batch_size=batch_size,
write_graph=True))
# Create the network
net = multimodal.Multimodal(learning_rate, beta1, beta2, epsilon)
# Configures the model for training
net.model.compile(optimizer=net.train_opt,
loss=net.model_loss,
metrics=net.metrics)
# Load the pretrained imagenet weights
load_weights.regnet_weights(net.model)
if resume:
net.model = keras.models.load_model(save_path,
custom_objects=CUSTOM_LAYERS,
compile=True)
# Train network
log = net.model.fit_generator(generator=itr_train,
validation_data=itr_valid,
validation_steps=batch_size,
epochs=epochs,
steps_per_epoch=num_batches,
callbacks=cbs,
verbose=1,
workers=0)
# save log file
save_path = save_path.with_suffix('.pkl')
with open(save_path, 'wb') as handle:
pickle.dump(log.history, handle, protocol=pickle.HIGHEST_PROTOCOL)
optimizer=optimizers.Adam(lr=learning_rate),
loss='categorical_crossentropy',
metrics=['accuracy'])
util.save_model_summary(model)
# Train
callbacks = [
callbacks.EarlyStopping(
monitor='val_acc',
patience=2),
callbacks.TensorBoard(
log_dir='logs',
histogram_freq=0,
batch_size=32,
write_graph=True,
write_grads=False,
write_images=False),
callbacks.ModelCheckpoint(
filepath='weights.{epoch:02d}-{val_loss:.2f}.hdf5'),
]
model.fit(
x=data.train.images,
y=data.train.labels,
epochs=epochs,
batch_size=128,
validation_data=validation_data,
callbacks=callbacks)
