from tensorflow.python.keras.datasets import mnist from tensorflow.python.keras import backend from tensorflow.python.keras import callbacks from util import * from modelConfig import * from time import time conf = tf.compat.v1.ConfigProto(intra_op_parallelism_threads=1, inter_op_parallelism_threads=1) sess = tf.compat.v1.Session(config=conf) #callbacks epoch = getEpoch() tensorboard = callbacks.TensorBoard(log_dir=f'../Logs/{name}') checkpoint = callbacks.ModelCheckpoint(f"../Models/{name}-{epoch}.h5", monitor='val_loss', mode='auto', period=10) # Decoder - Sequential # decoder = Sequential([ Dense(intermediate_dim, input_dim=latent_dim, activation='relu'), Dense(original_dim, activation='sigmoid') ]) x_pred = decoder(z) vae = Model(inputs=[x, eps], outputs=x_pred) vae.compile(optimizer='rmsprop', loss=nll)
def train_model_retinanet(model, dataset, backbone, expt='', test_size=.1, n_epoch=10, batch_size=1, num_gpus=None, include_masks=False, panoptic=False, panoptic_weight=0.1, transforms=['watershed'], transforms_kwargs={}, anchor_params=None, pyramid_levels=['P3', 'P4', 'P5', 'P6', 'P7'], mask_size=(28, 28), optimizer=SGD(lr=0.01, decay=1e-6, momentum=0.9, nesterov=True), log_dir='/data/tensorboard_logs', model_dir='/data/models', model_name=None, sigma=3.0, alpha=0.25, gamma=2.0, score_threshold=0.01, iou_threshold=0.5, max_detections=100, weighted_average=True, lr_sched=rate_scheduler(lr=0.01, decay=0.95), rotation_range=0, flip=True, shear=0, zoom_range=0, compute_map=True, seed=None, **kwargs): """Train a RetinaNet model from the given backbone Adapted from: https://github.com/fizyr/keras-retinanet & https://github.com/fizyr/keras-maskrcnn """ is_channels_first = K.image_data_format() == 'channels_first' if model_name is None: todays_date = datetime.datetime.now().strftime('%Y-%m-%d') data_name = os.path.splitext(os.path.basename(dataset))[0] model_name = '{}_{}_{}'.format(todays_date, data_name, expt) model_path = os.path.join(model_dir, '{}.h5'.format(model_name)) loss_path = os.path.join(model_dir, '{}.npz'.format(model_name)) train_dict, test_dict = get_data(dataset, seed=seed, test_size=test_size) channel_axis = 1 if is_channels_first else -1 n_classes = model.layers[-1].output_shape[channel_axis] if panoptic: n_semantic_classes = [layer.output_shape[channel_axis] for layer in model.layers if 'semantic' in layer.name] # the data, shuffled and split between train and test sets print('X_train shape:', train_dict['X'].shape) print('y_train shape:', train_dict['y'].shape) print('X_test shape:', test_dict['X'].shape) print('y_test shape:', test_dict['y'].shape) print('Output Shape:', model.layers[-1].output_shape) print('Number of Classes:', n_classes) if num_gpus is None: num_gpus = train_utils.count_gpus() if num_gpus >= 1e6: batch_size = batch_size * num_gpus model = train_utils.MultiGpuModel(model, num_gpus) print('Training on {} GPUs'.format(num_gpus)) # evaluation of model is done on `retinanet_bbox` if include_masks: prediction_model = model else: prediction_model = retinanet_bbox( model, nms=True, anchor_params=anchor_params, num_semantic_heads=len(n_semantic_classes), panoptic=panoptic, class_specific_filter=False) retinanet_losses = losses.RetinaNetLosses(sigma=sigma, alpha=alpha, gamma=gamma, iou_threshold=iou_threshold, mask_size=mask_size) def semantic_loss(n_classes): def _semantic_loss(y_pred, y_true): return panoptic_weight * losses.weighted_categorical_crossentropy( y_pred, y_true, n_classes=n_classes) return _semantic_loss loss = { 'regression': retinanet_losses.regress_loss, 'classification': retinanet_losses.classification_loss } if include_masks: loss['masks'] = retinanet_losses.mask_loss if panoptic: # Give losses for all of the semantic heads for layer in model.layers: if 'semantic' in layer.name: n_classes = layer.output_shape[channel_axis] loss[layer.name] = semantic_loss(n_classes) model.compile(loss=loss, optimizer=optimizer) if num_gpus >= 2: # Each GPU must have at least one validation example if test_dict['y'].shape[0] < num_gpus: raise ValueError('Not enough validation data for {} GPUs. ' 'Received {} validation sample.'.format( test_dict['y'].shape[0], num_gpus)) # When using multiple GPUs and skip_connections, # the training data must be evenly distributed across all GPUs num_train = train_dict['y'].shape[0] nb_samples = num_train - num_train % batch_size if nb_samples: train_dict['y'] = train_dict['y'][:nb_samples] train_dict['X'] = train_dict['X'][:nb_samples] # this will do preprocessing and realtime data augmentation datagen = image_generators.RetinaNetGenerator( # fill_mode='constant', # for rotations rotation_range=rotation_range, shear_range=shear, zoom_range=zoom_range, horizontal_flip=flip, vertical_flip=flip) datagen_val = image_generators.RetinaNetGenerator( # fill_mode='constant', # for rotations rotation_range=0, shear_range=0, zoom_range=0, horizontal_flip=0, vertical_flip=0) # if 'vgg' in backbone or 'densenet' in backbone: # compute_shapes = make_shapes_callback(model) # else: # compute_shapes = guess_shapes compute_shapes = guess_shapes train_data = datagen.flow( train_dict, seed=seed, include_mask_transforms=len(transforms) > 0, include_masks=include_masks, panoptic=panoptic, transforms=transforms, transforms_kwargs=transforms_kwargs, pyramid_levels=pyramid_levels, anchor_params=anchor_params, compute_shapes=compute_shapes, batch_size=batch_size) val_data = datagen_val.flow( test_dict, seed=seed, include_mask_transforms=len(transforms) > 0, include_masks=include_masks, panoptic=panoptic, transforms=transforms, transforms_kwargs=transforms_kwargs, pyramid_levels=pyramid_levels, anchor_params=anchor_params, compute_shapes=compute_shapes, batch_size=batch_size) tensorboard_callback = callbacks.TensorBoard( log_dir=os.path.join(log_dir, model_name)) # fit the model on the batches generated by datagen.flow() loss_history = model.fit_generator( train_data, steps_per_epoch=train_data.y.shape[0] // batch_size, epochs=n_epoch, validation_data=val_data, validation_steps=val_data.y.shape[0] // batch_size, callbacks=[ callbacks.LearningRateScheduler(lr_sched), callbacks.ModelCheckpoint( model_path, monitor='val_loss', verbose=1, save_best_only=True, save_weights_only=num_gpus >= 2), tensorboard_callback, callbacks.ReduceLROnPlateau( monitor='loss', factor=0.1, patience=10, verbose=1, mode='auto', min_delta=0.0001, cooldown=0, min_lr=0), RedirectModel( Evaluate(val_data, iou_threshold=iou_threshold, score_threshold=score_threshold, max_detections=max_detections, tensorboard=tensorboard_callback, weighted_average=weighted_average), prediction_model), ]) model.save_weights(model_path) np.savez(loss_path, loss_history=loss_history.history) if compute_map: average_precisions = evaluate( val_data, prediction_model, iou_threshold=iou_threshold, score_threshold=score_threshold, max_detections=max_detections, ) # print evaluation total_instances = [] precisions = [] for label, (average_precision, num_annotations) in average_precisions.items(): print('{:.0f} instances of class'.format(num_annotations), label, 'with average precision: {:.4f}'.format(average_precision)) total_instances.append(num_annotations) precisions.append(average_precision) if sum(total_instances) == 0: print('No test instances found.') else: print('mAP using the weighted average of precisions among classes: {:.4f}'.format( sum([a * b for a, b in zip(total_instances, precisions)]) / sum(total_instances))) print('mAP: {:.4f}'.format(sum(precisions) / sum(x > 0 for x in total_instances))) return model
def train_model_sample(model, dataset, expt='', test_size=.1, n_epoch=10, batch_size=32, num_gpus=None, transform=None, window_size=None, balance_classes=True, max_class_samples=None, log_dir='/data/tensorboard_logs', model_dir='/data/models', model_name=None, focal=False, gamma=0.5, optimizer=SGD(lr=0.01, decay=1e-6, momentum=0.9, nesterov=True), lr_sched=rate_scheduler(lr=0.01, decay=0.95), rotation_range=0, flip=False, shear=0, zoom_range=0, seed=None, **kwargs): is_channels_first = K.image_data_format() == 'channels_first' if model_name is None: todays_date = datetime.datetime.now().strftime('%Y-%m-%d') data_name = os.path.splitext(os.path.basename(dataset))[0] model_name = '{}_{}_{}'.format(todays_date, data_name, expt) model_path = os.path.join(model_dir, '{}.h5'.format(model_name)) loss_path = os.path.join(model_dir, '{}.npz'.format(model_name)) train_dict, test_dict = get_data(dataset, test_size=test_size, seed=seed) n_classes = model.layers[-1].output_shape[1 if is_channels_first else -1] # the data, shuffled and split between train and test sets print('X_train shape:', train_dict['X'].shape) print('y_train shape:', train_dict['y'].shape) print('X_test shape:', test_dict['X'].shape) print('y_test shape:', test_dict['y'].shape) print('Output Shape:', model.layers[-1].output_shape) print('Number of Classes:', n_classes) def loss_function(y_true, y_pred): if isinstance(transform, str) and transform.lower() == 'disc': return losses.discriminative_instance_loss(y_true, y_pred) if focal: return losses.weighted_focal_loss( y_true, y_pred, gamma=gamma, n_classes=n_classes) return losses.weighted_categorical_crossentropy( y_true, y_pred, n_classes=n_classes) if num_gpus is None: num_gpus = train_utils.count_gpus() if num_gpus >= 2: batch_size = batch_size * num_gpus model = train_utils.MultiGpuModel(model, num_gpus) print('Training on {} GPUs'.format(num_gpus)) model.compile(loss=loss_function, optimizer=optimizer, metrics=['accuracy']) if train_dict['X'].ndim == 4: DataGenerator = image_generators.SampleDataGenerator window_size = window_size if window_size else (30, 30) elif train_dict['X'].ndim == 5: DataGenerator = image_generators.SampleMovieDataGenerator window_size = window_size if window_size else (30, 30, 3) else: raise ValueError('Expected `X` to have ndim 4 or 5. Got', train_dict['X'].ndim) # this will do preprocessing and realtime data augmentation datagen = DataGenerator( rotation_range=rotation_range, shear_range=shear, zoom_range=zoom_range, horizontal_flip=flip, vertical_flip=flip) # no validation augmentation datagen_val = DataGenerator( rotation_range=0, shear_range=0, zoom_range=0, horizontal_flip=0, vertical_flip=0) train_data = datagen.flow( train_dict, seed=seed, batch_size=batch_size, transform=transform, transform_kwargs=kwargs, window_size=window_size, balance_classes=balance_classes, max_class_samples=max_class_samples) val_data = datagen_val.flow( test_dict, seed=seed, batch_size=batch_size, transform=transform, transform_kwargs=kwargs, window_size=window_size, balance_classes=False, max_class_samples=max_class_samples) # fit the model on the batches generated by datagen.flow() loss_history = model.fit_generator( train_data, steps_per_epoch=train_data.y.shape[0] // batch_size, epochs=n_epoch, validation_data=val_data, validation_steps=val_data.y.shape[0] // batch_size, callbacks=[ callbacks.LearningRateScheduler(lr_sched), callbacks.ModelCheckpoint( model_path, monitor='val_loss', verbose=1, save_best_only=True, save_weights_only=num_gpus >= 2), callbacks.TensorBoard(log_dir=os.path.join(log_dir, model_name)) ]) np.savez(loss_path, loss_history=loss_history.history) return model
train_generator = train_datagen.flow_from_directory( # This is the target directory train_dir, # All images will be resized to 150x150 target_size=(150, 150), batch_size=20, # Since we use binary_crossentropy loss, we need binary labels class_mode='binary') validation_generator = test_datagen.flow_from_directory(validation_dir, target_size=(150, 150), batch_size=20, class_mode='binary') tbCallBack = callbacks.TensorBoard(log_dir='./log', histogram_freq=0, write_graph=True, write_images=True) history = model.fit_generator(train_generator, steps_per_epoch=100, epochs=20, validation_data=validation_generator, validation_steps=50, callbacks=[]) model.save('cats_and_dogs_small_1.h5') acc = history.history['acc'] val_acc = history.history['val_acc'] loss = history.history['loss'] val_loss = history.history['val_loss']
def train_model_siamese_daughter(model, dataset, expt='', test_size=.1, n_epoch=100, batch_size=1, num_gpus=None, crop_dim=32, min_track_length=1, neighborhood_scale_size=10, features=None, optimizer=SGD(lr=0.01, decay=1e-6, momentum=0.9, nesterov=True), log_dir='/data/tensorboard_logs', model_dir='/data/models', model_name=None, focal=False, gamma=0.5, lr_sched=rate_scheduler(lr=0.01, decay=0.95), rotation_range=0, flip=True, shear=0, zoom_range=0, seed=None, **kwargs): is_channels_first = K.image_data_format() == 'channels_first' if model_name is None: todays_date = datetime.datetime.now().strftime('%Y-%m-%d') data_name = os.path.splitext(os.path.basename(dataset))[0] model_name = '{}_{}_[{}]_neighs={}_epochs={}_seed={}_{}'.format( todays_date, data_name, ','.join(f[0] for f in sorted(features)), neighborhood_scale_size, n_epoch, seed, expt) model_path = os.path.join(model_dir, '{}.h5'.format(model_name)) loss_path = os.path.join(model_dir, '{}.npz'.format(model_name)) print('training on dataset:', dataset) print('saving model at:', model_path) print('saving loss at:', loss_path) train_dict, val_dict = get_data(dataset, mode='siamese_daughters', seed=seed, test_size=test_size) # the data, shuffled and split between train and test sets print('X_train shape:', train_dict['X'].shape) print('y_train shape:', train_dict['y'].shape) print('X_test shape:', val_dict['X'].shape) print('y_test shape:', val_dict['y'].shape) print('Output Shape:', model.layers[-1].output_shape) n_classes = model.layers[-1].output_shape[1 if is_channels_first else -1] def loss_function(y_true, y_pred): if focal: return losses.weighted_focal_loss(y_true, y_pred, gamma=gamma, n_classes=n_classes, from_logits=False) return losses.weighted_categorical_crossentropy(y_true, y_pred, n_classes=n_classes, from_logits=False) if num_gpus is None: num_gpus = train_utils.count_gpus() if num_gpus >= 2: batch_size = batch_size * num_gpus model = train_utils.MultiGpuModel(model, num_gpus) print('Training on {} GPUs'.format(num_gpus)) model.compile(loss=loss_function, optimizer=optimizer, metrics=['accuracy']) print('Using real-time data augmentation.') # this will do preprocessing and realtime data augmentation datagen = image_generators.SiameseDataGenerator( rotation_range=rotation_range, shear_range=shear, zoom_range=zoom_range, horizontal_flip=flip, vertical_flip=flip) datagen_val = image_generators.SiameseDataGenerator( rotation_range=0, zoom_range=0, shear_range=0, horizontal_flip=0, vertical_flip=0) total_train_pairs = tracking_utils.count_pairs(train_dict['y'], same_probability=5.0) total_test_pairs = tracking_utils.count_pairs(val_dict['y'], same_probability=5.0) train_data = datagen.flow( train_dict, seed=seed, crop_dim=crop_dim, batch_size=batch_size, min_track_length=min_track_length, neighborhood_scale_size=neighborhood_scale_size, features=features) val_data = datagen_val.flow( val_dict, seed=seed, crop_dim=crop_dim, batch_size=batch_size, min_track_length=min_track_length, neighborhood_scale_size=neighborhood_scale_size, features=features) print('total_train_pairs:', total_train_pairs) print('total_test_pairs:', total_test_pairs) print('batch size:', batch_size) print('validation_steps: ', total_test_pairs // batch_size) # fit the model on the batches generated by datagen.flow() loss_history = model.fit_generator( train_data, steps_per_epoch=total_train_pairs // batch_size, epochs=n_epoch, validation_data=val_data, validation_steps=total_test_pairs // batch_size, callbacks=[ callbacks.LearningRateScheduler(lr_sched), callbacks.ModelCheckpoint( model_path, monitor='val_loss', verbose=1, save_best_only=True, save_weights_only=num_gpus >= 2), callbacks.TensorBoard(log_dir=os.path.join(log_dir, model_name)) ]) model.save_weights(model_path) np.savez(loss_path, loss_history=loss_history.history) return model
font=dict( family="Courier New, monospace", size=18, color="#7f7f7f" ) ) ) ) fig.show() if __name__ == '__main__': userNet = DQSAVersion2(input_size=config.input_size_user, usernet=True) userNet.load_weights(path="/home/dorliv/Desktop/DQSAKeras/successful_experiements/three_users/network_central_best_three_users/checkpoint") logger = get_logger(os.path.join(config.evaluate_log_dir, "evaluate_log")) Tensorcallback = callbacks.TensorBoard(config.evaluate_log_dir, write_graph=True, write_images=False) Tensorcallback.set_model(userNet.model) env = OneTimeStepEnv() beta = 10 alpha = 0 # e_greedy draw_heatmap_flag = False channelThroughPutPerTstep = initCTP(config.TimeSlotsEvaluate) # init the data structure to view the mean reward at each t for iteration in range(5): channelThroughPutMean = 0 loss_value = [] collisonsMean = 0 idle_timesMean = 0 for episode in range(config.Episodes): heatmap = [] collisons = 0 idle_times = 0
model_name = "SimpleBaseline_Small_FromScratch_RMSProp_Default_L1_1E-1_e_25_is_48_48" ## ENSURE CORRECT # Images Directory dir_images = "./data/processed/resized-48-48/" ## ENSURE CORRECT ### INFO FOR TENSORBOARD # Note: put the tensorboard log into a subdirectory of the main logs folder, i.e. /logs/run_1, /logs/run_2 # This lets tensorboard display their output as separate runs properly. For now we'll just automatically increment run number dir_tensorboard_logs = "./tensorboard_logs/" dir_tensorboard_logs = os.path.abspath(dir_tensorboard_logs) num_tensorboard_runs = len(os.listdir(dir_tensorboard_logs)) dir_tensorboard_logs = dir_tensorboard_logs + "/" + model_name # Note: make the log directory later, in case the code fails before the training step and the new directory is left empty callback_tensorboard = callbacks.TensorBoard(log_dir=dir_tensorboard_logs, write_grads=True, write_images=True, histogram_freq=1) ### DATA PREP # Get images paths & split training/validation images_summary = pd.read_csv("./results/images_summary.csv") images_summary_train = images_summary[images_summary.DataRole == "train"] images_summary_valid = images_summary[images_summary.DataRole == "valid"] filenames_relative_train = images_summary_train.FileName_Relative.values filenames_train = dir_images + filenames_relative_train filenames_relative_valid = images_summary_valid.FileName_Relative.values filenames_valid = dir_images + filenames_relative_valid
help='pre trained model path.') FLAGS, unparsed = parser.parse_known_args() Net_OOP = TilesUnetMirrored() optimizer = tf.compat.v2.optimizers.Adam(beta_1=0.99) loader = Loader(batch_size=FLAGS.batch_size) Net_OOP.compile(optimizer=optimizer, loss=loss_fn, metrics=['acc', 'loss', 'val_acc', 'val_loss']) if not os.path.exists(FLAGS.log_dir): os.makedirs(FLAGS.log_dir) if not os.path.exists(FLAGS.model_path): os.makedirs(FLAGS.model_path) logger = get_logger(os.path.join(FLAGS.log_dir, "train_log")) Tensorcallback = callbacks.TensorBoard(FLAGS.log_dir, write_graph=False, write_images=False) Checkpoint = callbacks.ModelCheckpoint(filepath=FLAGS.model_path + "/checkpoint.hdf5", monitor='val_acc', mode='max', save_best_only=True) Checkpoint.set_model(Net_OOP) Tensorcallback.set_model(Net_OOP) callbacks = {'tensorboard': Tensorcallback, 'checkpoint': Checkpoint} Net_OOP.fit(logger=logger, callbacks=callbacks, epochs=FLAGS.epochs, steps_per_epoch=config.steps_per_epoch, val_freq=config.val_freq, val_steps=config.validation_steps,
model.add(layers.MaxPooling2D(pool_size=(2, 2), strides=(2, 2))) model.add(layers.Dropout(rate=0.1 + 0.025 * cnn_depth)) model.add(layers.Flatten()) # model.add(layers.Dense(256, activation='relu')) # model.add(layers.Dense(128, activation='relu')) model.add(layers.Dense(num_classes, activation='softmax')) opt_rms = optimizers.RMSprop(lr=0.0005, decay=1e-6) model.compile(loss=keras.losses.categorical_crossentropy, optimizer=opt_rms, metrics=['accuracy']) # Tensorboard tensorboard = callbacks.TensorBoard(log_dir="./K_CIFAR_model/{}".format( datetime.today().strftime('%m-%d__%H-%M-%S')), histogram_freq=0, write_graph=True) tensorboard.set_model(model) ############################## # Train model ############################## datagen = preprocessing.image.ImageDataGenerator( featurewise_center=False, samplewise_center=False, featurewise_std_normalization=False, samplewise_std_normalization=False, zca_whitening=False, rotation_range=15, width_shift_range=0.1, height_shift_range=0.1,
def train_model_retinanet(model, dataset, backbone, expt='', test_size=.1, n_epoch=10, batch_size=1, num_gpus=None, include_masks=False, mask_size=(28, 28), optimizer=SGD(lr=0.01, decay=1e-6, momentum=0.9, nesterov=True), log_dir='/data/tensorboard_logs', model_dir='/data/models', model_name=None, sigma=3.0, alpha=0.25, gamma=2.0, score_threshold=0.01, iou_threshold=0.5, max_detections=100, weighted_average=True, lr_sched=rate_scheduler(lr=0.01, decay=0.95), rotation_range=0, flip=True, shear=0, zoom_range=0, **kwargs): """Train a RetinaNet model from the given backbone Adapted from: https://github.com/fizyr/keras-retinanet & https://github.com/fizyr/keras-maskrcnn """ is_channels_first = K.image_data_format() == 'channels_first' if model_name is None: todays_date = datetime.datetime.now().strftime('%Y-%m-%d') data_name = os.path.splitext(os.path.basename(dataset))[0] model_name = '{}_{}_{}'.format(todays_date, data_name, expt) model_path = os.path.join(model_dir, '{}.h5'.format(model_name)) loss_path = os.path.join(model_dir, '{}.npz'.format(model_name)) train_dict, test_dict = get_data(dataset, mode='conv', test_size=test_size) n_classes = model.layers[-1].output_shape[1 if is_channels_first else -1] # the data, shuffled and split between train and test sets print('X_train shape:', train_dict['X'].shape) print('y_train shape:', train_dict['y'].shape) print('X_test shape:', test_dict['X'].shape) print('y_test shape:', test_dict['y'].shape) print('Output Shape:', model.layers[-1].output_shape) print('Number of Classes:', n_classes) if num_gpus is None: num_gpus = train_utils.count_gpus() if num_gpus >= 1e6: batch_size = batch_size * num_gpus model = train_utils.MultiGpuModel(model, num_gpus) print('Training on {} GPUs'.format(num_gpus)) def regress_loss(y_true, y_pred): # separate target and state regression = y_pred regression_target = y_true[..., :-1] anchor_state = y_true[..., -1] # filter out "ignore" anchors indices = tf.where(K.equal(anchor_state, 1)) regression = tf.gather_nd(regression, indices) regression_target = tf.gather_nd(regression_target, indices) # compute the loss loss = losses.smooth_l1(regression_target, regression, sigma=sigma) # compute the normalizer: the number of positive anchors normalizer = K.maximum(1, K.shape(indices)[0]) normalizer = K.cast(normalizer, dtype=K.floatx()) return K.sum(loss) / normalizer def classification_loss(y_true, y_pred): # TODO: try weighted_categorical_crossentropy labels = y_true[..., :-1] # -1 for ignore, 0 for background, 1 for object anchor_state = y_true[..., -1] classification = y_pred # filter out "ignore" anchors indices = tf.where(K.not_equal(anchor_state, -1)) labels = tf.gather_nd(labels, indices) classification = tf.gather_nd(classification, indices) # compute the loss loss = losses.focal(labels, classification, alpha=alpha, gamma=gamma) # compute the normalizer: the number of positive anchors normalizer = tf.where(K.equal(anchor_state, 1)) normalizer = K.cast(K.shape(normalizer)[0], K.floatx()) normalizer = K.maximum(K.cast_to_floatx(1.0), normalizer) return K.sum(loss) / normalizer def mask_loss(y_true, y_pred): def _mask(y_true, y_pred, iou_threshold=0.5, mask_size=(28, 28)): # split up the different predicted blobs boxes = y_pred[:, :, :4] masks = y_pred[:, :, 4:] # split up the different blobs annotations = y_true[:, :, :5] width = K.cast(y_true[0, 0, 5], dtype='int32') height = K.cast(y_true[0, 0, 6], dtype='int32') masks_target = y_true[:, :, 7:] # reshape the masks back to their original size masks_target = K.reshape(masks_target, (K.shape(masks_target)[0] * K.shape(masks_target)[1], height, width)) masks = K.reshape(masks, (K.shape(masks)[0] * K.shape(masks)[1], mask_size[0], mask_size[1], -1)) # batch size > 1 fix boxes = K.reshape(boxes, (-1, K.shape(boxes)[2])) annotations = K.reshape(annotations, (-1, K.shape(annotations)[2])) # compute overlap of boxes with annotations iou = overlap(boxes, annotations) argmax_overlaps_inds = K.argmax(iou, axis=1) max_iou = K.max(iou, axis=1) # filter those with IoU > 0.5 indices = tf.where(K.greater_equal(max_iou, iou_threshold)) boxes = tf.gather_nd(boxes, indices) masks = tf.gather_nd(masks, indices) argmax_overlaps_inds = tf.gather_nd(argmax_overlaps_inds, indices) argmax_overlaps_inds = K.cast(argmax_overlaps_inds, 'int32') labels = K.gather(annotations[:, 4], argmax_overlaps_inds) labels = K.cast(labels, 'int32') # make normalized boxes x1 = boxes[:, 0] y1 = boxes[:, 1] x2 = boxes[:, 2] y2 = boxes[:, 3] boxes = K.stack([ y1 / (K.cast(height, dtype=K.floatx()) - 1), x1 / (K.cast(width, dtype=K.floatx()) - 1), (y2 - 1) / (K.cast(height, dtype=K.floatx()) - 1), (x2 - 1) / (K.cast(width, dtype=K.floatx()) - 1), ], axis=1) # crop and resize masks_target # append a fake channel dimension masks_target = K.expand_dims(masks_target, axis=3) masks_target = tf.image.crop_and_resize(masks_target, boxes, argmax_overlaps_inds, mask_size) # remove fake channel dimension masks_target = masks_target[:, :, :, 0] # gather the predicted masks using the annotation label masks = tf.transpose(masks, (0, 3, 1, 2)) label_indices = K.stack([tf.range(K.shape(labels)[0]), labels], axis=1) masks = tf.gather_nd(masks, label_indices) # compute mask loss mask_loss = K.binary_crossentropy(masks_target, masks) normalizer = K.shape(masks)[0] * K.shape(masks)[1] * K.shape( masks)[2] normalizer = K.maximum(K.cast(normalizer, K.floatx()), 1) mask_loss = K.sum(mask_loss) / normalizer return mask_loss # if there are no masks annotations, return 0; else, compute the masks loss return tf.cond( K.any(K.equal(K.shape(y_true), 0)), lambda: K.cast_to_floatx(0.0), lambda: _mask(y_true, y_pred, iou_threshold=iou_threshold, mask_size=mask_size)) # evaluation of model is done on `retinanet_bbox` if include_masks: prediction_model = model else: prediction_model = retinanet_bbox(model, nms=True, class_specific_filter=False) loss = {'regression': regress_loss, 'classification': classification_loss} if include_masks: loss['masks'] = mask_loss model.compile(loss=loss, optimizer=optimizer) if num_gpus >= 2: # Each GPU must have at least one validation example if test_dict['y'].shape[0] < num_gpus: raise ValueError('Not enough validation data for {} GPUs. ' 'Received {} validation sample.'.format( test_dict['y'].shape[0], num_gpus)) # When using multiple GPUs and skip_connections, # the training data must be evenly distributed across all GPUs num_train = train_dict['y'].shape[0] nb_samples = num_train - num_train % batch_size if nb_samples: train_dict['y'] = train_dict['y'][:nb_samples] train_dict['X'] = train_dict['X'][:nb_samples] # this will do preprocessing and realtime data augmentation datagen = image_generators.RetinaNetGenerator( # fill_mode='constant', # for rotations rotation_range=rotation_range, shear_range=shear, zoom_range=zoom_range, horizontal_flip=flip, vertical_flip=flip) datagen_val = image_generators.RetinaNetGenerator( # fill_mode='constant', # for rotations rotation_range=0, shear_range=0, zoom_range=0, horizontal_flip=0, vertical_flip=0) if 'vgg' in backbone or 'densenet' in backbone: compute_shapes = make_shapes_callback(model) else: compute_shapes = guess_shapes train_data = datagen.flow(train_dict, include_masks=include_masks, compute_shapes=compute_shapes, batch_size=batch_size) val_data = datagen_val.flow(test_dict, include_masks=include_masks, compute_shapes=compute_shapes, batch_size=batch_size) tensorboard_callback = callbacks.TensorBoard( log_dir=os.path.join(log_dir, model_name)) # fit the model on the batches generated by datagen.flow() loss_history = model.fit_generator( train_data, steps_per_epoch=train_data.y.shape[0] // batch_size, epochs=n_epoch, validation_data=val_data, validation_steps=val_data.y.shape[0] // batch_size, callbacks=[ callbacks.LearningRateScheduler(lr_sched), callbacks.ModelCheckpoint(model_path, monitor='val_loss', verbose=1, save_best_only=True, save_weights_only=num_gpus >= 2), tensorboard_callback, callbacks.ReduceLROnPlateau(monitor='loss', factor=0.1, patience=10, verbose=1, mode='auto', min_delta=0.0001, cooldown=0, min_lr=0), RedirectModel( Evaluate(val_data, iou_threshold=iou_threshold, score_threshold=score_threshold, max_detections=max_detections, tensorboard=tensorboard_callback, weighted_average=weighted_average), prediction_model), ]) model.save_weights(model_path) np.savez(loss_path, loss_history=loss_history.history) average_precisions = evaluate( val_data, prediction_model, iou_threshold=iou_threshold, score_threshold=score_threshold, max_detections=max_detections, ) # print evaluation total_instances = [] precisions = [] for label, (average_precision, num_annotations) in average_precisions.items(): print('{:.0f} instances of class'.format(num_annotations), label, 'with average precision: {:.4f}'.format(average_precision)) total_instances.append(num_annotations) precisions.append(average_precision) if sum(total_instances) == 0: print('No test instances found.') else: print( 'mAP using the weighted average of precisions among classes: {:.4f}' .format( sum([a * b for a, b in zip(total_instances, precisions)]) / sum(total_instances))) print('mAP: {:.4f}'.format( sum(precisions) / sum(x > 0 for x in total_instances))) return model
def run(epochs, num_batches, batch_size=1, learning_rate=0.001, beta1=0.9, beta2=0.999, epsilon=1e-08, save_every=10, earlyStopping=True, patience=5, resume=False): # Destroy old graph K.clear_session() # Initialize batch generators batch_train = build_features.get_train_batches(batch_size=batch_size) batch_valid = build_features.get_valid_batches(batch_size=batch_size) # Create TensorFlow Iterator object itr_train = build_features.make_iterator(batch_train) itr_valid = build_features.make_iterator(batch_valid) # Init callbacks cbs = list() # History callback: saves all losses cbs.append(callbacks.History()) # EarlyStopping callback: stops whenever loss doesn't imporve if earlyStopping: cbs.append( early_stopping.EarlyStopping(monitor='val_binary_accuracy', baseline=0.8, patience=patience, verbose=1)) # ModelCheckpoint callback: saves model every SAVE_EVERY save_path = paths.checkpoints.multimodal() # ./checkpoints/regnet/train save_path.parent.mkdir(exist_ok=True, parents=True) if save_path.exists() and not resume: save_path.unlink() # deletes file before training cbs.append( callbacks.ModelCheckpoint(str(save_path), save_best_only=True, period=save_every)) # TensorBoard callback: saves logs for tensorboard log_path = str(paths.logs.multimodal()) # ./logs/regnet/train cbs.append( callbacks.TensorBoard(log_dir=log_path, batch_size=batch_size, write_graph=True)) # Create the network net = multimodal.Multimodal(learning_rate, beta1, beta2, epsilon) # Configures the model for training net.model.compile(optimizer=net.train_opt, loss=net.model_loss, metrics=net.metrics) # Load the pretrained imagenet weights load_weights.regnet_weights(net.model) if resume: net.model = keras.models.load_model(save_path, custom_objects=CUSTOM_LAYERS, compile=True) # Train network log = net.model.fit_generator(generator=itr_train, validation_data=itr_valid, validation_steps=batch_size, epochs=epochs, steps_per_epoch=num_batches, callbacks=cbs, verbose=1, workers=0) # save log file save_path = save_path.with_suffix('.pkl') with open(save_path, 'wb') as handle: pickle.dump(log.history, handle, protocol=pickle.HIGHEST_PROTOCOL)
optimizer=optimizers.Adam(lr=learning_rate), loss='categorical_crossentropy', metrics=['accuracy']) util.save_model_summary(model) # Train callbacks = [ callbacks.EarlyStopping( monitor='val_acc', patience=2), callbacks.TensorBoard( log_dir='logs', histogram_freq=0, batch_size=32, write_graph=True, write_grads=False, write_images=False), callbacks.ModelCheckpoint( filepath='weights.{epoch:02d}-{val_loss:.2f}.hdf5'), ] model.fit( x=data.train.images, y=data.train.labels, epochs=epochs, batch_size=128, validation_data=validation_data, callbacks=callbacks)