datasets = ['./data/data_batch_1', './data/data_batch_2', './data/data_batch_3', './data/data_batch_4','./data/data_batch_5']
epoch = input('\nPlease input epoch:')
batch_size = input('Please input batch size:')
temp = 50000/batch_size
t1 = time.time()
start_index = 0
images = np.zeros([50000, 32, 32, 3])
labels = np.zeros([50000, 10])
for j in xrange(5):
images[j*10000:(j+1)*10000] = load_data.load_images(datasets[j])
labels[j*10000:(j+1)*10000] = load_data.load_labels(datasets[j])
for i in xrange(epoch * temp):
batch_x, batch_y, start_index = load_data.next_batch(images, labels, batch_size, start_index)
_, loss_val = sess.run([train_step, cross_entropy], feed_dict={x:batch_x, y_:batch_y, keep_prob: 0.8})
if i%temp == 0:
print 'epoch %d , loss = %s' % (i/temp, loss_val)
del images, labels
print '\nEnd training.\n'
t2 = time.time()
test_x, test_y = load_data.load_data('./data/test_batch')
print 'test accuracy %g' %accuracy.eval(feed_dict={x: test_x, y_: test_y, keep_prob: 1.0})
print 'Spends %f seconds.' % (t2 - t1)
def main():
batch_size = 3
model = MyModel((432, 768), training=True)
opt = tf.keras.optimizers.Adam(lr=0.0002)
# model1 = tf.python.keras.Model()
checkpoint_dir = 'path/to/model_dir'
if not os.path.exists(checkpoint_dir):
os.makedirs(checkpoint_dir)
ckpt = tf.train.Checkpoint(step=tf.Variable(1), optimizer=opt, net=model)
manager = tf.train.CheckpointManager(ckpt, './tf_ckpts', max_to_keep=3)
ckpt.restore(manager.latest_checkpoint)
if manager.latest_checkpoint:
print("Restored from {}".format(manager.latest_checkpoint))
else:
print("Initializing from scratch.")
img_, gtbox_, gtmask_ = next_batch(1, 0)
img0_ = img_['1']
gtbox0_ = gtbox_['1']
gtmask0_ = gtmask_['1']
img1_, gtbox1_, gtmask1_ = preprocess1(img0_, gtbox0_, gtmask0_)
images = image_mean_subtraction(img1_)
pre_ini = model(images)
model.model1.resnet50.load_weights(
"./path/resnet50_weights/resnet50_weights_tf.h5", by_name=True)
pr_file = open("./pr.txt", mode='a', encoding='UTF-8-sig')
epochs = 200
ite = 10
for epoch in range(epochs):
TP = 0.
FP = 0.
FN = 0.
for i in range(ite):
ckpt.step.assign_add(1)
img, gtbox, gtmask = next_batch1(batch_size, i % 400)
img_dims = img.shape[1:3]
img_mask = generate_mask(img_dims, gtmask, batch_size)
img_mask = tf.cast(img_mask, tf.int32)
images = image_mean_subtraction(img)
with tf.GradientTape() as t:
result, rpn_cls_score, rpn_bbox_pred = model(images)
roi_pred = result[4]
mask_pred = result[3]
cls_score = result[2]
cls_pred = result[0]
offset_pred = result[5]
# compute rpn loss
rpn_cls_loss1, rpn_bbox_loss1 = loss_rpn(
rpn_cls_score, rpn_bbox_pred, gtbox, img_dims)
rpn_loss = rpn_cls_loss1 + rpn_bbox_loss1
total_loss = rpn_loss
# the first image
batch_ind = np.reshape(
np.where(np.reshape(roi_pred[:, 0], (-1)) == 0), (-1, ))
roi_pred1 = tf.gather(roi_pred, batch_ind)
mask_pred1 = tf.gather(mask_pred, batch_ind)
cls_score1 = tf.gather(cls_score, batch_ind)
cls_pred1 = tf.gather(cls_pred, batch_ind)
offset_pred1 = tf.gather(offset_pred, batch_ind)
keep = model_part3_1(roi_pred1[:, 1:], cls_score1)
roi_pred_keep = tf.gather(roi_pred1, keep, axis=0)
cls_score_keep = tf.gather(cls_score1, keep, axis=0)
cls_pred_keep = tf.gather(cls_pred1, keep, axis=0)
offset_pred_keep = tf.gather(offset_pred1, keep, axis=0)
rois1, labels1, bbox_targets1, bbox_inside_weights1, \
bbox_outside_weights1, keep_inds, fg_num = proposal_target_layer(
roi_pred_keep, gtbox, 1
)
print("positive_nums:" + str(fg_num))
cls_scores2 = tf.gather(cls_pred_keep, keep_inds, axis=0)
preds2 = tf.gather(offset_pred_keep, keep_inds, axis=0)
for j in range(fg_num):
mask1 = model_part3_2(roi_pred1, keep, keep_inds[j],
mask_pred1, cls_score1)
rfcn_mask_loss = loss_mask(mask1,
roi_pred_keep[keep_inds[j]],
img_mask[0])
total_loss = total_loss + 2 * rfcn_mask_loss
# record the TP FP FN to compute precision and recall
gtbox_ind = np.where(gtbox[:, 0] == 0)
TP1, FP1, FN1 = compute_tp_fp_fn(cls_pred_keep[:, 1],
roi_pred_keep[:, 1:],
gtbox[gtbox_ind], 0.5)
TP += TP1
FP += FP1
FN += FN1
for i1 in range(1, batch_size):
batch_ind = np.reshape(
np.where(np.reshape(roi_pred[:, 0], (-1)) == i1),
(-1, ))
roi_pred1 = tf.gather(roi_pred, batch_ind)
mask_pred1 = tf.gather(mask_pred, batch_ind)
cls_score1 = tf.gather(cls_score, batch_ind)
cls_pred1 = tf.gather(cls_pred, batch_ind)
offset_pred1 = tf.gather(offset_pred, batch_ind)
keep = model_part3_1(roi_pred1[:, 1:], cls_score1)
roi_pred_keep = tf.gather(roi_pred1, keep, axis=0)
cls_score_keep = tf.gather(cls_score1, keep, axis=0)
cls_pred_keep = tf.gather(cls_pred1, keep, axis=0)
offset_pred_keep = tf.gather(offset_pred1, keep, axis=0)
# the keep rois in rois and gt_boxes
rois1_1, labels1_1, bbox_targets1_1, bbox_inside_weights1_1, \
bbox_outside_weights1_1, keep_inds, fg_num = proposal_target_layer(
roi_pred_keep, gtbox, 1
)
print("positive_nums:" + str(fg_num))
cls_scores1_1 = tf.gather(cls_pred_keep, keep_inds, axis=0)
preds1_1 = tf.gather(offset_pred_keep, keep_inds, axis=0)
cls_scores2 = tf.concat((cls_scores2, cls_scores1_1),
axis=0)
preds2 = tf.concat((preds2, preds1_1), axis=0)
labels1 = tf.concat((labels1, labels1_1), 0)
bbox_targets1 = tf.concat((bbox_targets1, bbox_targets1_1),
axis=0)
bbox_inside_weights1 = tf.concat(
(bbox_inside_weights1, bbox_inside_weights1_1), 0)
bbox_outside_weights1 = tf.concat(
(bbox_outside_weights1, bbox_outside_weights1_1), 0)
for j in range(fg_num):
mask1 = model_part3_2(roi_pred1, keep, keep_inds[j],
mask_pred1, cls_score1)
rfcn_mask_loss = loss_mask(mask1,
roi_pred_keep[keep_inds[j]],
img_mask[i1])
total_loss = total_loss + 2 * rfcn_mask_loss
gtbox_ind = np.where(gtbox[:, 0] == i1)
TP1, FP1, FN1 = compute_tp_fp_fn(cls_pred_keep[:, 1],
roi_pred_keep[:, 1:],
gtbox[gtbox_ind], 0.5)
TP += TP1
FP += FP1
FN += FN1
rfcn_cls_loss1, rfcn_bbox_loss1 = loss_cls_bbox(
cls=cls_scores2,
offset=preds2,
labels=labels1,
bbox_targets=bbox_targets1,
bbox_inside_weights=bbox_inside_weights1,
bbox_outside_weights=bbox_outside_weights1)
total_loss = total_loss + rfcn_cls_loss1 + rfcn_bbox_loss1
if np.isnan(total_loss):
exit()
grads = t.gradient(total_loss, model.trainable_variables)
opt.apply_gradients(zip(grads, model.trainable_variables))
print('Training loss (for one batch) at epoch %s step %s: %s' %
(epoch, i, float(total_loss)))
print(
'rpn_cls_loss:{:.5f}---rpn_bbox_loss:{:.5f}---rfcn_cls_loss:{:.5f}---'
'rfcn_bbox_loss:{}'.format(rpn_cls_loss1, rpn_bbox_loss1,
rfcn_cls_loss1, rfcn_bbox_loss1))
print(' ' * 20)
if int(ckpt.step) % 5 == 4:
save_path = manager.save()
print("save model")
precision = TP / np.maximum((TP + FP), np.finfo(np.float64).eps)
recall = TP / float(TP + FN)
print('Precision: %s ------Recall: %s' % (precision, recall))
print(' ' * 20)
pr_file.write("P:" + str(precision) + ' ' + "R" + str(recall))
pr_file.write('\n')
def main():
model = MyModel((720, 1280), training=True)
opt = tf.keras.optimizers.Adam(lr=0.002)
# model1 = tf.python.keras.Model()
batch_size = 5
checkpoint_dir = 'path/to/model_dir'
if not os.path.exists(checkpoint_dir):
os.makedirs(checkpoint_dir)
ckpt = tf.train.Checkpoint(step=tf.Variable(1), optimizer=opt, net=model)
manager = tf.train.CheckpointManager(ckpt, './tf_ckpts', max_to_keep=3)
ckpt.restore(manager.latest_checkpoint)
if manager.latest_checkpoint:
print("Restored from {}".format(manager.latest_checkpoint))
# img_, gtbox_, gtmask_ = next_batch(1, 0)
# img0_ = img_['1']
# gtbox0_ = gtbox_['1']
# gtmask0_ = gtmask_['1']
# img1_, gtbox1_, gtmask1_ = preprocess1(img0_, gtbox0_, gtmask0_)
# images = image_mean_subtraction(img1_)
# pre_ini = model(images)
# model.model1.resnet50.load_weights("./path/resnet50_weights/resnet50_weights_tf.h5", by_name=True)
else:
print("Initializing from scratch.")
img_, gtbox_, gtmask_ = next_batch(1, 0)
img0_ = img_['1']
gtbox0_ = gtbox_['1']
gtmask0_ = gtmask_['1']
img1_, gtbox1_, gtmask1_ = preprocess1(img0_, gtbox0_, gtmask0_)
images = image_mean_subtraction(img1_)
pre_ini = model(images)
model.model1.resnet50.load_weights("./path/resnet50_weights/resnet50_weights_tf.h5", by_name=True)
pr_file = open("./pr.txt", mode='a', encoding='UTF-8-sig')
epochs = 300
img_num = 4
for epoch in range(epochs):
TP = 0.
FP = 0.
FN = 0.
for i in range(img_num):
ckpt.step.assign_add(1)
img, gtbox, gtmask = next_batch(1, i % 400)
img0 = img[str(i+1)]
gtbox0 = gtbox[str(i+1)]
gtmask0 = gtmask[str(i+1)]
img1, gtbox1, gtmask1 = preprocess1(img0, gtbox0, gtmask0)
img_dims = img1.shape[0:2]
img_mask = generate_mask(img_dims, gtmask1)
img_mask = tf.cast(img_mask, tf.int32)
images = image_mean_subtraction(img1)
with tf.GradientTape() as t:
result, rpn_cls_score, rpn_bbox_pred, keep = model(images)
roi_pred = result[4]
mask_pred = result[3]
cls_score = result[2]
cls_pred = result[0]
offset_pred = result[5]
roi_pred_keep = tf.gather(roi_pred, keep, axis=0)
cls_score_keep = tf.gather(cls_score, keep, axis=0)
cls_pred_keep = tf.gather(cls_pred, keep, axis=0)
offset_pred_keep = tf.gather(offset_pred, keep, axis=0)
rpn_cls_loss1, rpn_bbox_loss1 = loss_rpn(rpn_cls_score, rpn_bbox_pred, gtbox1, img_dims)
rpn_loss = rpn_cls_loss1 + rpn_bbox_loss1
# the keep rois in rois and gt_boxes
rois1, labels1, bbox_targets1, bbox_inside_weights1, \
bbox_outside_weights1, keep_inds, fg_num = proposal_target_layer(
roi_pred_keep, gtbox1, 1
)
print("positive_nums:"+str(fg_num))
for j in range(fg_num):
pass
rfcn_cls_loss1, rfcn_bbox_loss1 = loss_cls_bbox(cls=cls_pred_keep, offset=offset_pred_keep, keep_inds=keep_inds,
labels=labels1, bbox_targets=bbox_targets1,
bbox_inside_weights=bbox_inside_weights1,
bbox_outside_weights=bbox_outside_weights1)
loss1 = rpn_loss + rfcn_cls_loss1
total_loss = rpn_cls_loss1 + rpn_bbox_loss1 + rfcn_cls_loss1 + 2*rfcn_bbox_loss1
for j in range(fg_num):
mask1 = model_part3_2(roi_pred, keep, keep_inds[j], mask_pred, cls_score)
rfcn_mask_loss = loss_mask(mask1, roi_pred_keep[keep_inds[j]], img_mask)
total_loss = total_loss + rfcn_mask_loss
if np.isnan(total_loss):
exit()
grads = t.gradient(total_loss, model.trainable_variables)
opt.apply_gradients(zip(grads, model.trainable_variables))
# record the TP FP FN to compute precision and recall
TP1, FP1, FN1 = compute_tp_fp_fn(cls_pred_keep[:, 1], roi_pred_keep[:, 1:], gtbox1, 0.5)
TP += TP1
FP += FP1
FN += FN1
print('Training loss (for one batch) at epoch %s step %s: %s' % (epoch, i, float(total_loss)))
print('rpn_cls_loss:{:.5f}---rpn_bbox_loss:{:.5f}---rfcn_cls_loss:{:.5f}---'
'rfcn_bbox_loss:{}'.format(rpn_cls_loss1, rpn_bbox_loss1,
rfcn_cls_loss1, rfcn_bbox_loss1))
print(' ' * 20)
if int(ckpt.step) % 10 == 9:
save_path = manager.save()
print("save model")
precision = TP / np.maximum((TP + FP), np.finfo(np.float64).eps)
recall = TP / float(TP + FN)
print('Precision: %s ------Recall: %s' % (precision, recall))
print(' '*20)
pr_file.write("P:"+str(precision) + ' '+"R"+str(recall))
pr_file.write('\n')
def main():
input_images = tf.placeholder(tf.float32, shape=[1,224,224,3], name='input_image')
img_mask = tf.placeholder(tf.int32, shape=[None,8], name='mask')
gt_boxes = tf.placeholder(tf.int32, shape=[None,5], name='groundtruth_boxes')
global_step = tf.get_variable('global_step', [], initializer=tf.constant_initializer(0), trainable=False)
learning_rate = tf.train.exponential_decay(learning_rate=0.0001, global_step=global_step, decay_steps=10000,
decay_rate=0.94, staircase=True)
tf.summary.scalar('learning_rate',learning_rate)
opt = tf.train.AdamOptimizer(learning_rate)
init_op = tf.global_variables_initializer()
with tf.variable_scope('model'):
x1 = np.zeros((1,224,224,3))
x2 = [[100,100,160,160,1]]
roi, ps_score, bbox_shift, cls_loss_rpn, bbox_loss_rpn = model_part1(images=input_images,is_training=True,gt_boxes=gt_boxes)
result = model_part2(imdims=(224,225), rois=roi, ps_score_map=ps_score, bbox_shift=bbox_shift)
with tf.Session() as sess:
sess.run(init_op)
roi, ps_score, bbox_shift, cls_loss_rpn, \
bbox_loss_rpn = sess.run((roi,ps_score,bbox_shift,cls_loss_rpn,bbox_loss_rpn),
feed_dict={input_images:x1,gt_boxes:x2})
result = sess.run(result)
result_keep = model_part3(results=result)
# compute the postive boxes and positive boxes
# compute the boxes' inside_weights and out_side weights
roi_item = result[str(result_keep[0])]['roi'] #(4,)
offset_item = result[str(result_keep[0])]['shift'] #(4,)
cls_item = result[str(result_keep[0])]['cls']
rois = tf.reshape(roi_item, (1,-1))
offset = tf.reshape(offset_item,(1,-1))
cls = tf.reshape(cls_item,(1,-1))
roi_mask = {}
roi_mask['0'] = result[str(result_keep[0])]['mask']
for i in range(1, len(result_keep)):
roi_item = result[str(result_keep[i])]['roi']
roi_item = tf.reshape(roi_item, (1,-1))
rois = tf.concat((rois,roi_item), axis=0) #(n,4)
offset_item = result[str(result_keep[i])]['shift']
offset = tf.reshape(offset_item, (1, -1))
offset = tf.concat((offset, offset_item), axis=0) #(n,4)
cls_item = result[str(result_keep[i])]['cls']
cls = tf.reshape(cls_item, (1, -1))
cls = tf.concat((cls, cls_item), axis=0) #(n,k+1)
roi_mask[str(i)] = result[str(result_keep[i])]['mask']
# get the rois which need to compute gradient
rois1, labels1, bbox_targets1, bbox_inside_weights1, bbox_outside_weights1, keep_inds, fg_num = proposal_target_layer(
rois, gt_boxes, cfg.dataset.NUM_CLASSES
)
cls_scores1 = cls[keep_inds]
preds1 = offset[keep_inds]
# get image's mask
im_shape = tf.shape(input_images)
im_dims = im_shape[1:3]
mask = tf.py_func(generate_mask,[im_dims,img_mask],tf.int32)
# compute every box's loss
# compute rpn_cls_loss and rpn_bbox_loss
loss1 = cls_loss_rpn
loss2 = bbox_loss_rpn
# compute all boxes' cls_loss
loss3 = rfcn_cls_loss(rfcn_cls_score=cls_scores1,labels=labels1)
# compute positive boxes' bbox_loss
loss4 = rfcn_bbox_loss(rfcn_bbox_pred=preds1, bbox_targets=bbox_targets1,
roi_inside_weights=bbox_inside_weights1, roi_outside_weights=bbox_outside_weights1)
# compute positive boxes' mask_loss
mask_j = roi_mask[str(keep_inds[0])] # (h,w,2)
mask_j = upsample_image(mask_j, rois[keep_inds[0]][2] - rois[keep_inds[0]][0], rois[keep_inds[0]][3] - rois[keep_inds[0]][1])
mask_label = box_mask(rois[keep_inds[0]], mask)
loss5 = mask_loss(rfcn_mask_score=mask_j, labels=mask_label)
for j in keep_inds[1:fg_num]:
mask_j = roi_mask[str(j)] #(h,w,2)
mask_j = upsample_image(mask_j,rois[j][2]-rois[j][0],rois[j][3]-rois[j][1])
mask_label = box_mask(rois[j],mask)
loss5 += mask_loss(rfcn_mask_score=mask_j,labels=mask_label)
total_loss = loss1 + loss2 + loss3 + loss4 + loss5
training_step = opt.minimize(total_loss,global_step=global_step)
with sess.as_default():
start = time.time()
for step in range(1000):
x,y_gtbox,y_mask = next_batch(batch_size=1,pos=step)
sess.run(training_step,feed_dict={input_images:x[str(step + 1)],
img_mask:y_mask[str(step + 1)],
gt_boxes:y_gtbox[str(step + 1)]
})
if step % 50 == 0:
print('step {},rpn_cls_loss{},rpn_bbox_loss{},cls_loss{},'
'bbox_loss{},mask_loss{},total_loss{}'.format(step,loss1,loss2,
loss3,loss4,loss5,total_loss))
pass
cv.polylines(img, [pts], True, (0, 255, 255))
cv.imshow("Image", image)
cv.waitKey(0)
cv.destroyAllWindows()
pass
def draw1(image, gtboxes):
for i in range(len(gtboxes)):
print(gtboxes[i][0], gtboxes[i][1])
cv.rectangle(image, (gtboxes[i][0], gtboxes[i][1]),
(gtboxes[i][2], gtboxes[i][3]), (0, 255, 0), 1)
cv.imshow("Image", image)
cv.waitKey(0)
cv.destroyAllWindows()
pass
#draw(img,mask1)
# draw1(img,gtbox)
for i in range(1000):
img, gtbox, gtmask = next_batch(1, i % 1000)
img0 = img[str(i + 1)]
gtbox0 = gtbox[str(i + 1)]
gtmask0 = gtmask[str(i + 1)]
gtbox1 = np.array(gtbox0)
gtmask1 = np.array(gtmask0)
draw1(img0, gtbox1)