def load_models(self):
cfg = self.cfg
if self.do_rpn:
print('Loading RPN model...')
loss_cls = losses.rpn_loss_cls(cfg.nb_anchors)
loss_rgr = losses.rpn_loss_regr(cfg.nb_anchors)
get_custom_objects().update({"rpn_loss_cls_fixed_num": loss_cls})
get_custom_objects().update({"rpn_loss_regr_fixed_num": loss_rgr})
assert os.path.exists(cfg.my_shared_weights
), 'invalid path: %s' % cfg.my_shared_weights
self.model_rpn = load_model(cfg.my_shared_weights)
self.model_rpn.name = 'rpn'
assert not self.model_rpn.layers[17].get_config()['trainable']
if self.do_det:
print('Loading DET model...')
loss_cls = losses.class_loss_cls
loss_rgr = losses.class_loss_regr(cfg.nb_object_classes - 1)
get_custom_objects().update({"class_loss_cls": loss_cls})
get_custom_objects().update(
{"class_loss_regr_fixed_num": loss_rgr})
get_custom_objects().update(
{"RoiPoolingConv": layers.RoiPoolingConv})
assert os.path.exists(cfg.my_shared_weights
), 'invalid path: %s' % cfg.my_shared_weights
self.model_det = load_model(cfg.my_shared_weights)
self.model_det.name = 'det'
if self.do_hoi:
print('Loading HOI model...')
loss_cls = losses.hoi_loss_cls
get_custom_objects().update({"hoi_loss_cls_fixed_num": loss_cls})
assert os.path.exists(cfg.my_shared_weights
), 'invalid path: %s' % cfg.my_shared_weights
self.model_hoi = load_model(cfg.my_shared_weights)
self.model_hoi.name = 'hoi'
def compile_models(self):
if self.mode == 'test':
return
cfg = self.cfg
print('Compiling models...')
if cfg.optimizer == 'adam':
print(' Opt.:', 'Adam')
opt = Adam(lr=cfg.init_lr)
else:
print(' Opt.:', 'SGD')
opt = SGD(lr=cfg.init_lr, momentum=0.9, decay=0.0, nesterov=False)
print(' Learning rate:', cfg.init_lr)
if self.do_rpn:
if cfg.rpn_uniform_sampling:
print(' Uniform anchor sampling')
else:
print(' Non-Uniform anchor sampling')
model = self.model_rpn
my_losses = [
losses.rpn_loss_cls(cfg.nb_anchors),
losses.rpn_loss_regr(cfg.nb_anchors)
]
my_metrics = None
if self.do_det:
model = self.model_det
my_losses = [
losses.class_loss_cls,
losses.class_loss_regr(cfg.nb_object_classes - 1)
]
my_metrics = {'det_out_class': 'categorical_accuracy'}
if self.do_hoi:
model = self.model_hoi
loss = 'categorical_crossentropy' if cfg.do_categorical_hoi else losses.hoi_loss_cls(
cfg.wp)
print(' -HOI loss', loss)
my_losses = [loss]
my_metrics = ['categorical_accuracy'
] if cfg.do_categorical_hoi else None
model.compile(optimizer=opt, loss=my_losses, metrics=my_metrics)
VGG_model = keras.applications.vgg16.VGG16(include_top=True,
weights='imagenet',
input_tensor=None,
input_shape=None,
pooling=None,
classes=1000)
SSH = model.getModel()
optimizer = SGD(lr=0.00005, momentum=0.9, decay=0.0005, nesterov=True)
SSH.summary()
# SSH = LoadWeights(SSH)
SSH.load_weights('imagenet_pretrained_2_model_ssh.vgg.hdf5')
# SSH = LoadVGGWeights(SSH, VGG_model)
SSH.compile(optimizer=optimizer,
loss=[losses.rpn_loss_cls(num_anchors), losses.rpn_loss_regr(num_anchors), \
losses.rpn_loss_cls(num_anchors), losses.rpn_loss_regr(num_anchors), \
losses.rpn_loss_cls(num_anchors), losses.rpn_loss_regr(num_anchors)])
checkpoint = ModelCheckpoint('imagenet_pretrained_2_' + C.model_path,
monitor='val_loss',
save_best_only=True,
verbose=1)
earlystop = EarlyStopping(monitor='val_loss', patience=patience, verbose=1)
def LogView(batch, logs={}):
print(logs.keys())
LogViewCallback = LambdaCallback(on_batch_end=LogView)
def train_rpn():
# 读取配置
cfg = config.Config()
# 将图像及VOC格式的数据以Img_Data对象的形式进行保存
all_images, classes_count, class_mapping = get_data(cfg.label_file)
cfg.class_mapping = class_mapping
# for bbox_num, bbox in enumerate(all_images[0].bboxes):
# print(bbox_num, bbox)
# 将配置文件进行保存
with open(cfg.config_save_file, 'wb') as config_f:
pickle.dump(cfg, config_f)
print('2、Config已经被写入到{}, 并且可以在测试的时候加载以确保得到正确的结果'.format(
cfg.config_save_file))
print("3、按照类别数量大小顺序输出")
pprint.pprint(classes_count)
print("4、类别个数（大于训练集+测试集数量，并且包括背景）= {}".format(len(classes_count)))
random.shuffle(all_images)
print("5、对样本进行打乱")
train_imgs = [img_data for img_data in all_images if img_data.imageset == 'trainval']
val_imgs = [img_data for img_data in all_images if img_data.imageset == 'test']
print("6、设置训练集及验证集，其中训练集数量为{}，测试集数量为{}".format(len(train_imgs), len(val_imgs)))
# 对训练数据进行打乱
random.shuffle(train_imgs)
# 类别映射
# 得到每一个锚的训练数据，供RPN网络训练使用
data_gen_train = data_generators.get_anchor_gt(train_imgs, classes_count, cfg, nn.get_img_out_length,
K.image_dim_ordering(), mode='train')
# data_gen_val = data_generators.get_anchor_gt(val_imgs, classes_count, cfg, nn.get_img_output_length,
# K.image_dim_ordering(), mode='val')
if K.image_dim_ordering() == 'th':
input_shape_img = (3, None, None)
else:
input_shape_img = (None, None, 3)
img_input = Input(shape=input_shape_img)
# define the base network (resnet here, can be VGG, Inception, etc)
shared_layers = nn.nn_base(img_input, trainable=True)
# define the RPN, built on the base layers
num_anchors = len(cfg.anchor_box_scales) * len(cfg.anchor_box_ratios)
rpn = nn.rpn(shared_layers, num_anchors, cfg.num_regions)
model_rpn = Model(img_input, rpn[:2])
try:
print('7、从{}加载参数'.format(cfg.base_net_weights))
model_rpn.load_weights(cfg.model_path, by_name=True)
except Exception as e:
print(e)
print('无法加载与训练模型权重 ')
optimizer = Adam(lr=1e-5)
model_rpn.compile(optimizer=optimizer,
loss=[losses_fn.rpn_loss_cls(num_anchors), losses_fn.rpn_loss_regr(num_anchors)])
epoch_length = 500
num_epochs = int(cfg.num_epochs)
iter_num = 0
losses = np.zeros((epoch_length, 2))
rpn_accuracy_rpn_monitor = []
rpn_accuracy_for_epoch = []
start_time = time.time()
best_loss = np.Inf
print('8、开始训练')
for epoch_num in range(num_epochs):
progbar = generic_utils.Progbar(epoch_length)
print('Epoch{}/{}'.format(epoch_num + 1, num_epochs))
while True:
try:
if len(rpn_accuracy_rpn_monitor) == epoch_length and cfg.verbose:
mean_overlapping_bboxes = float(sum(rpn_accuracy_rpn_monitor)) / len(rpn_accuracy_rpn_monitor)
rpn_accuracy_rpn_monitor = []
print('RPN的bounding boxes平均覆盖数量 = {} for {} previous iterations'.format(
mean_overlapping_bboxes, epoch_length))
if mean_overlapping_bboxes == 0:
print('RPN不生产覆盖的边框，检查RPN的设置或者继续训练')
X, Y, img_data = next(data_gen_train)
# Y[0].shape (1, X, Y, 18)
# X是input data，Y是labels
loss_rpn = model_rpn.train_on_batch(X, Y)
p_rpn = model_rpn.predict_on_batch(X)
result = roi_helpers.rpn_to_roi(p_rpn[0], p_rpn[1], cfg, K.image_dim_ordering(), use_regr=True,
overlap_thresh=0.7,
max_boxes=10)
# visual_rpn(img_data, result)
print('-------result------')
# print('-------result--------')
# print(result[250])
# print('-------result--------')
# print('-------p_rpn--------')
# print(p_rpn[0].shape)
# print(p_rpn[1].shape)
# (1, 38, 48, 9)
# (1, 38, 48, 36)
# print('-------p_rpn--------')
losses[iter_num, 0] = loss_rpn[1]
losses[iter_num, 1] = loss_rpn[2]
iter_num += 1
progbar.update(iter_num, [('rpn分类损失', np.mean(losses[:iter_num, 0])), ('rpn回归损失', np.mean(losses[:iter_num, 1]))])
if iter_num == epoch_length:
loss_rpn_cls = np.mean(losses[:, 0])
loss_rpn_regr = np.mean(losses[:, 1])
print(loss_rpn_cls, loss_rpn_regr)
# mean_overlapping_bboxes = float(sum(rpn_accuracy_for_epoch)) // len(rpn_accuracy_for_epoch)
# print(mean_overlapping_bboxes)
# rpn_accuracy_for_epoch = []
if cfg.verbose:
# print('Mean number of bounding boxes from RPN overlapping ground truth boxes: {}'.format(mean_overlapping_bboxes))
print('Loss RPN classifier: {}'.format(loss_rpn_cls))
print('Loss RPN regression: {}'.format(loss_rpn_regr))
print('Elapsed time: {}'.format(time.time() - start_time))
curr_loss = loss_rpn_cls + loss_rpn_regr
iter_num = 0
start_time = time.time()
if curr_loss < best_loss:
if cfg.verbose:
('总损失函数从{}减到{}，保存权重'.format(best_loss, curr_loss))
best_loss = curr_loss
model_rpn.save_weights(cfg.model_path)
break
except Exception as e:
print('错误{}'.format(e))
# 保存模型
model_rpn.save_weights(cfg.model_path)
continue
print('训练完成，退出')
VGG_model = keras.applications.vgg16.VGG16(include_top=True,
weights='imagenet',
input_tensor=None,
input_shape=None,
pooling=None,
classes=1000)
SSH = model.getModel()
optimizer = SGD(lr=0.00005, momentum=0.9, decay=0.0005, nesterov=True)
SSH.summary()
# SSH = LoadWeights(SSH)
# SSH.load_weights('model_ssh.vgg.hdf5')
SSH = LoadVGGWeights(SSH, VGG_model)
SSH.compile(optimizer=optimizer, loss=[losses.rpn_loss_cls(num_anchors), losses.rpn_loss_regr(num_anchors), \
losses.rpn_loss_cls(num_anchors), losses.rpn_loss_regr(num_anchors), \
losses.rpn_loss_cls(num_anchors), losses.rpn_loss_regr(num_anchors)])
checkpoint = ModelCheckpoint('imagenet_pretrained_' + C.model_path,
monitor='val_loss',
save_best_only=True,
verbose=1)
earlystop = EarlyStopping(monitor='val_loss', patience=patience, verbose=1)
csv_logger = CSVLogger('training.log', append=True)
SSH.fit_generator(generator=data_gen_train,
steps_per_epoch=len(train_imgs) // batchSize,
validation_data=data_gen_val,
validation_steps=len(val_imgs) // batchSize,
epochs=num_epochs,
model_all = Model([img_input, roi_input], rpn[:2] + classifier)
try:
print('loading weights from {}'.format(C.base_net_weights))
model_rpn.load_weights(C.base_net_weights, by_name=True)
model_classifier.load_weights(C.base_net_weights, by_name=True)
except:
print(
'Could not load pretrained model weights. Weights can be found in the keras application folder \
https://github.com/fchollet/keras/tree/master/keras/applications')
optimizer = Adam(lr=1e-5)
optimizer_classifier = Adam(lr=1e-5)
model_rpn.compile(
optimizer=optimizer,
loss=[losses.rpn_loss_cls(num_anchors),
losses.rpn_loss_regr(num_anchors)])
model_classifier.compile(
optimizer=optimizer_classifier,
loss=[
losses.class_loss_cls,
losses.class_loss_regr(len(classes_count) - 1)
],
metrics={'dense_class_{}'.format(len(classes_count)): 'accuracy'})
model_all.compile(optimizer='sgd', loss='mae')
epoch_length = 1000
num_epochs = int(options.num_epochs)
iter_num = 0
losses = np.zeros((epoch_length, 5))
print('Load weight from ' + C.model_path)
model_rpn.load_weights(C.model_path, by_name=True)
model_classifier.load_weights(C.model_path, by_name=True)
# this is a model that holds both the RPN and the classifier, used to load/save weights for the models
model_all = Model([img_input, roi_input], rpn[:2] + classifier)
# print("MODEL RPN")
# model_rpn.summary()
# print("MODEL CLASSIFIER")
# model_classifier.summary()
# print("MODEL ALL")
# model_all.summary()
# Define
optimizer_rpn = optimizers.Adam(lr=1e-5)
optimizer_classifier = optimizers.Adam(lr=1e-5)
model_rpn.compile(optimizer=optimizer_rpn, loss=[losses.rpn_loss_cls(num_anchors), losses.rpn_loss_regr(num_anchors)])
model_classifier.compile(optimizer=optimizer_classifier,
loss=[losses.class_loss_cls, losses.class_loss_regr(len(classes_count) - 1)],
metrics={'dense_class_{}'.format(len(classes_count)): 'accuracy'})
model_all.compile(optimizer='sgd', loss='mae')
epoch_length = 1000
num_epochs = int(options.num_epochs)
iter_num = 0
losses = np.zeros((epoch_length, 5))
rpn_accuracy_rpn_monitor = []
rpn_accuracy_for_epoch = []
start_time = time.time()
best_loss = np.Inf
temp_bb_train = np.ones((50000, 10, 36))
temp_bb_test = np.ones((10000, 10, 36))
temp_regr_train = np.ones((50000, 2, 2, 36))
temp_regr_test = np.ones((10000, 2, 2, 36))
temp_class_train = np.ones((50000, 2, 2, 9))
temp_class_test = np.ones((10000, 2, 2, 9))
model = ROIModel(10)
model.compile(
optimizer=optimizers.RMSprop(1e-3),
loss={
"output_1": custom_losses.class_loss_cls(),
"output_2": custom_losses.class_loss_regr(9),
"output_3": custom_losses.rpn_loss_regr(num_anchors),
"output_4": custom_losses.rpn_loss_cls(num_anchors),
},
metrics={
"output_1": 'accuracy',
"output_2": 'accuracy',
"output_3": 'accuracy',
"output_4": 'accuracy',
},
)
logdir = "logs/fit/" + datetime.now().strftime("%Y%m%d-%H%M%S")
tensorboard_callback = tf.keras.callbacks.TensorBoard(log_dir=logdir)
# test_image = np.array(train_images[0]).reshape(1,32,32,3)
# model.predict(test_image,callbacks=[tensorboard_callback])