def train_one_epoch(self, sess, writer, epoch, step): print('sub_train_one_epoch') sum_acc = 0 sum_acc_iou = 0 sum_acc_label = 0 sum_acc_ellip = 0 count = 0 total_loss = 0 t0 = time.time() mean_acc = 0 mean_acc_iou = 0 mean_acc_label = 0 mean_acc_ellip = 0 if_epoch = cfgs.test_accu #if epoch % 5 == 0: # if_epoch = True try: while count < self.per_e_train_batch: step += 1 count += 1 #1. train images_, cur_ims_, ellip_infos_, annos_, labels, filenames = sess.run( [ self.train_images, self.train_cur_ims, self.train_ellip_infos, self.train_annotations, self.train_labels, self.train_filenames ]) #cv2.imwrite('%s_anno.bmp' % filenames[0], annos_[0]*255) #pdb.set_trace() cur_batch_size = images_.shape[0] if cur_batch_size == cfgs.batch_size: coord_map_x_cur, coord_map_y_cur = self.coord_map_x, self.coord_map_y else: coord_map_x_cur, coord_map_y_cur = self.generate_coord_map( cur_batch_size) if cur_batch_size != cfgs.batch_size: #pdb.set_trace() last_idx = cur_batch_size - 1 last_im = np.expand_dims(images_[last_idx], axis=0) last_cur_im = np.expand_dims(cur_ims_[last_idx], axis=0) last_ellip_info = np.expand_dims(ellip_infos_[last_idx], axis=0) last_anno = np.expand_dims(annos_[last_idx], axis=0) last_label = np.expand_dims(labels[last_idx], axis=0) last_fn = np.expand_dims(filenames[last_idx], axis=0) for i in range(cur_batch_size, cfgs.batch_size): images_ = np.append(images_, last_im, axis=0) cur_ims_ = np.append(cur_ims_, last_cur_im, axis=0) ellip_infos_ = np.append(ellip_infos_, last_ellip_info, axis=0) annos_ = np.append(annos_, last_anno, axis=0) labels = np.append(labels, last_label, axis=0) filenames = np.append(filenames, last_fn, axis=0) ellip_info_low = np.min(ellip_infos_[:, 2:], 1) / 2 ellip_info_high = np.max(ellip_infos_[:, 2:], 1) / 2 ellip_info_mean = np.mean(ellip_infos_[:, 2:], 1) / 4 #generate mask ims mask_ims, pred_anno_, pred_seq_pro_, summary_str = sess.run( [self.mask_ims, self.mask_anno, self.pro, self.summary_op], feed_dict={ self.images: images_, self.annotations: annos_, self.lr: self.learning_rate, self.class_labels: labels, self.keep_prob: cfgs.keep_prob, self.input_keep_prob: 1, self.cur_batch_size: cfgs.batch_size, self.coord_x_tensor: coord_map_x_cur, self.coord_y_tensor: coord_map_y_cur, #self.ellip_low: ellip_info_low, #self.ellip_high: ellip_info_high, self.ellip_axis: ellip_info_mean }) #self.im_mask_view(filenames, mask_ims, pred_seq_pro_, images_, step) #self.im_mask_view(filenames, pred_anno_, pred_seq_pro_, images_, step) #cv2.imwrite('%s_anno.bmp' % filenames[0], pred_anno_[0]*127) #pdb.set_trace() #2. calculate accurary self.ellip_acc = 0 self.calculate_acc(mask_ims.copy(), filenames, pred_anno_, pred_seq_pro_, annos_, ellip_infos_, if_epoch=if_epoch) #self.calculate_acc(annos_, filenames, pred_anno_, pred_seq_pro_, annos_, ellip_infos_, if_epoch=if_epoch) #self.calculate_acc(cur_ims_[:, 2:cfgs.ANNO_IMAGE_SIZE[0]+2, :, :], filenames, pred_anno_, pred_seq_pro_, annos_, ellip_infos_, if_epoch=if_epoch) self.accu = 0 self.accu_iou = 0 loss = 0 self.acc_label_ = 0 #self.ellip_acc = 0 sum_acc += self.accu sum_acc_iou += self.accu_iou sum_acc_label += self.acc_label_ sum_acc_ellip += self.ellip_acc mean_acc = sum_acc / count mean_acc_iou = sum_acc_iou / count mean_acc_label = sum_acc_label / count mean_acc_ellip = sum_acc_ellip / count #3. calculate loss total_loss += loss #4. time consume time_consumed = time.time() - t0 time_per_batch = time_consumed / count #5. check if change learning rate if count % 100 == 0: self.try_update_lr() #6. summary writer.add_summary(summary_str, global_step=step) #6. print #print('\r' + 2 * ' ', end='') line = 'Train epoch %2d\t lr = %g\t step = %4d\t count = %4d\t loss = %.4f\t m_loss=%.4f\t acc = %.2f%%\t iou_acc = %.2f%%\t acc_label = %.2f%%\t ellip_acc = %.2f\t time = %.2f' % ( epoch, self.learning_rate, step, count, loss, (total_loss / count), mean_acc, mean_acc_iou, mean_acc_label, mean_acc_ellip, time_per_batch) utils.clear_line(len(line)) print('\r' + line, end='') #print('epoch %5d\t lr = %g\t step = %4d\t count = %4d\t loss = %.4f\t mean_loss=%.4f\t train_acc = %.2f%%\t train_iou_acc = %.2f%%\t train_ellip_acc = %.2f\t time = %.2f' % (epoch, self.learning_rate, step, count, loss, (total_loss/count), mean_acc, mean_acc_iou, mean_acc_ellip, time_per_batch)) #End one epoch #count -= 1 print( '\nepoch %5d\t learning_rate = %g\t mean_loss = %.4f\t train_acc = %.2f%%\t train_iou_acc = %.2f%%\t train_acc_label = %.2f%%\t train_ellip_acc = %.2f' % (epoch, self.learning_rate, (total_loss / count), (sum_acc / count), (sum_acc_iou / count), (sum_acc_label / count), (sum_acc_ellip / count))) print('Take time %3.1f' % (time.time() - t0)) except tf.errors.OutOfRangeError: print('Error!') count -= 1 print( 'epoch %5d\t learning_rate = %g\t mean_loss = %.3f\t train_accuracy = %.2f%%\t train_iou_accuracy = %.2f%%' % (epoch, self.learning_rate, (total_loss / count), (sum_acc / count), (sum_acc_iou / count))) print('Take time %3.1f' % (time.time() - t0)) return step
def valid_once(self, sess, writer, epoch, step): count = 0 sum_acc = 0 sum_acc_iou = 0 sum_acc_label = 0 sum_acc_ellip = 0 t0 = time.time() if_epoch = cfgs.test_accu #if epoch % 5 == 0: # if_epoch = True try: total_loss = 0 #self.per_e_valid_batch = 2 while count<self.per_e_valid_batch: count +=1 images_, cur_ims, ellip_infos_, annos_, labels, filenames = sess.run([self.valid_images, self.valid_cur_ims, self.valid_ellip_infos, self.valid_annotations, self.valid_labels, self.valid_filenames]) cur_batch_size = images_.shape[0] if cur_batch_size == cfgs.batch_size: coord_map_x_cur, coord_map_y_cur = self.coord_map_x, self.coord_map_y else: coord_map_x_cur, coord_map_y_cur = self.generate_coord_map(cur_batch_size) if cur_batch_size != cfgs.batch_size: #pdb.set_trace() last_idx = cur_batch_size - 1 last_im = np.expand_dims(images_[last_idx], axis=0) last_cur_im = np.expand_dims(cur_ims[last_idx], axis=0) last_ellip_info = np.expand_dims(ellip_infos_[last_idx], axis=0) last_anno = np.expand_dims(annos_[last_idx], axis=0) last_label = np.expand_dims(labels[last_idx], axis=0) last_fn = np.expand_dims(filenames[last_idx], axis=0) for i in range(cur_batch_size, cfgs.batch_size): images_ = np.append(images_, last_im, axis=0) cur_ims = np.append(cur_ims, last_cur_im, axis=0) ellip_infos_ = np.append(ellip_infos_, last_ellip_info, axis=0) annos_ = np.append(annos_, last_anno, axis=0) labels = np.append(labels, last_label, axis=0) filenames = np.append(filenames, last_fn, axis=0) ellip_info_low = np.min(ellip_infos_[:, 2:], 1) / 2 ellip_info_high = np.max(ellip_infos_[:, 2:], 1) / 2 ellip_info_mean = np.mean(ellip_infos_[:, 2:], 1) / 4 pred_anno, pred_seq_pro, summary_str, loss, self.accu, self.accu_iou = sess.run( fetches=[self.pred_annotation, self.pro, self.summary_op, self.loss, self.accu_tensor, self.accu_iou_tensor], #fetches=[self.pred_anno_lower, self.pro, self.summary_op, self.loss, self.accu_tensor_lower, self.accu_iou_tensor_lower, self.acc_label], feed_dict={self.images: images_, self.annotations: annos_, self.lr: self.learning_rate, self.class_labels: labels, self.keep_prob: 1, self.input_keep_prob: 1, self.cur_batch_size: cfgs.batch_size, self.coord_x_tensor: coord_map_x_cur, self.coord_y_tensor: coord_map_y_cur, self.ellip_axis: ellip_info_mean}) #View result self.view_valid(filenames, pred_anno, pred_seq_pro, images_, step) writer.add_summary(summary_str, global_step=step) self.calculate_acc(cur_ims[:, 2:cfgs.ANNO_IMAGE_SIZE[0]+2, :, :].copy(), filenames, pred_anno, pred_seq_pro, annos_, ellip_infos_, True, if_epoch) sum_acc += self.accu sum_acc_iou += self.accu_iou sum_acc_label += 0 sum_acc_ellip += self.ellip_acc total_loss += loss line = 'epoch %5d\t learning_rate = %g\t step = %4d\t loss = %.4f\t valid_accuracy = %.2f%%\t valid_iou_accuracy = %.2f%%\t valid_acc_label = %.2f%%\t valid_ellip_acc = %.2f' % (epoch, self.learning_rate, step, (total_loss/count), (sum_acc/count), (sum_acc_iou/count), (sum_acc_label/count), (sum_acc_ellip/count)) utils.clear_line(len(line)) print('\r' + line, end='') #print('\r' + 12 * ' ', end='') #print('epoch %5d\t learning_rate = %g\t step = %4d\t loss = %.4f\t valid_accuracy = %.2f%%\t valid_iou_accuracy = %.2f%%\t valid_ellip_acc = %.2f' % (epoch, self.learning_rate, step, (total_loss/count), (sum_acc/count), (sum_acc_iou/count), (sum_acc_ellip/count))) #End valid data #count -= 1 print('\nepoch %5d\t learning_rate = %g\t loss = %.4f\t valid_accuracy = %.2f%%\t valid_iou_accuracy = %.2f%%\t valid_acc_label = %.2f%%\t valid_ellip_acc = %.2f' % (epoch, self.learning_rate, total_loss/count, sum_acc/count, sum_acc_iou/count, sum_acc_label/count, sum_acc_ellip/count)) print('Take time %3.1f' % (time.time() - t0)) except tf.errors.OutOfRangeError: print('Error!')
def valid_once(self, sess, writer, epoch, step): count = 0 sum_acc = 0 sum_acc_iou = 0 sum_acc_ellip = 0 t0 = time.time() if_epoch = False if epoch % 5 == 0: if_epoch = True try: total_loss = 0 #self.per_e_valid_batch = 2 while count < self.per_e_valid_batch: count += 1 images_, cur_ims, ellip_infos_, annos_, filenames = sess.run([ self.valid_images, self.valid_cur_ims, self.valid_ellip_infos, self.valid_annotations, self.valid_filenames ]) cur_batch_size = images_.shape[0] if cur_batch_size == cfgs.batch_size: coord_map_x_cur, coord_map_y_cur = self.coord_map_x, self.coord_map_y else: coord_map_x_cur, coord_map_y_cur = self.generate_coord_map( cur_batch_size) if cur_batch_size == cfgs.batch_size: ellip_info_low = np.min(ellip_infos_[:, 2:], 1) / 2 ellip_info_high = np.max(ellip_infos_[:, 2:], 1) / 2 ellip_info_mean = np.mean(ellip_infos_[:, 2:], 1) / 4 mask_ims, pred_anno_, pred_seq_pro_, summary_str = sess.run( #fetches=[self.pred_annotation, self.pro, self.summary_op, self.loss, self.accu_tensor, self.accu_iou_tensor], #fetches=[self.pred_anno_lower, self.pro, self.summary_op, self.loss, self.accu_tensor_lower, self.accu_iou_tensor_lower], fetches=[ self.mask_ims, self.mask_anno, self.pro, self.summary_op ], feed_dict={ self.images: images_, self.annotations: annos_, self.lr: self.learning_rate, self.keep_prob: 1, self.input_keep_prob: 1, self.cur_batch_size: cur_batch_size, self.coord_x_tensor: coord_map_x_cur, self.coord_y_tensor: coord_map_y_cur, self.ellip_axis: ellip_info_mean }) #View result #self.view_valid(filenames, pred_anno, pred_seq_pro, images_, step) #self.im_mask_view_valid(filenames, pred_anno, pred_seq_pro, images_, step) #self.im_mask_view_valid(filenames, mask_ims, pred_seq_pro_, images_, step) writer.add_summary(summary_str, global_step=step) self.ellip_acc = 0 #self.calculate_acc(annos_, filenames, pred_anno_, pred_seq_pro_, annos_, ellip_infos_, True, if_epoch) self.calculate_acc(mask_ims.copy(), filenames, pred_anno_, pred_seq_pro_, annos_, ellip_infos_, if_epoch=if_epoch) #self.calculate_acc(cur_ims[:, 2:cfgs.ANNO_IMAGE_SIZE[0]+2, :, :], filenames, pred_anno_, pred_seq_pro_, annos_, ellip_infos_, if_epoch=if_epoch) self.accu = 0 self.accu_iou = 0 loss = 0 sum_acc += self.accu sum_acc_iou += self.accu_iou sum_acc_ellip += self.ellip_acc total_loss += loss #print('\r' + 12 * ' ', end='') line = 'epoch %5d\t learning_rate = %g\t step = %4d\t loss = %.4f\t valid_accuracy = %.2f%%\t valid_iou_accuracy = %.2f%%\t valid_ellip_acc = %.2f' % ( epoch, self.learning_rate, step, (total_loss / count), (sum_acc / count), (sum_acc_iou / count), (sum_acc_ellip / count)) utils.clear_line(len(line)) print('\r' + line, end='') #End valid data #count -= 1 print( 'epoch %5d\t learning_rate = %g\t loss = %.4f\t valid_accuracy = %.2f%%\t valid_iou_accuracy = %.2f%%\t valid_ellip_acc = %.2f' % (epoch, self.learning_rate, total_loss / count, sum_acc / count, sum_acc_iou / count, sum_acc_ellip / count)) print('Take time %3.1f' % (time.time() - t0)) except tf.errors.OutOfRangeError: print('Error!')
def valid_once(self, sess, writer, epoch, step): count = 0 sum_acc = 0 sum_acc_iou = 0 sum_acc_ellip = 0 t0 = time.time() if_epoch = cfgs.test_accu #if epoch % 5 == 0: # if_epoch = True try: total_loss = 0 #self.per_e_valid_batch = 2 while count < self.per_e_valid_batch: count += 1 images_, cur_ims, ellip_infos_, annos_, filenames = sess.run([ self.valid_images, self.valid_cur_ims, self.valid_ellip_infos, self.valid_annotations, self.valid_filenames ]) cur_batch_size = images_.shape[0] if cur_batch_size == cfgs.batch_size: coord_map_x_cur, coord_map_y_cur = self.coord_map_x, self.coord_map_y else: coord_map_x_cur, coord_map_y_cur = self.generate_coord_map( cur_batch_size) if cur_batch_size != cfgs.batch_size: #pdb.set_trace() last_idx = cur_batch_size - 1 last_im = np.expand_dims(images_[last_idx], axis=0) last_cur_im = np.expand_dims(cur_ims[last_idx], axis=0) last_ellip_info = np.expand_dims(ellip_infos_[last_idx], axis=0) last_anno = np.expand_dims(annos_[last_idx], axis=0) last_fn = np.expand_dims(filenames[last_idx], axis=0) for i in range(cur_batch_size, cfgs.batch_size): images_ = np.append(images_, last_im, axis=0) cur_ims = np.append(cur_ims, last_cur_im, axis=0) ellip_infos_ = np.append(ellip_infos_, last_ellip_info, axis=0) annos_ = np.append(annos_, last_anno, axis=0) filenames = np.append(filenames, last_fn, axis=0) ellip_info_low = np.min(ellip_infos_[:, 2:], 1) / 2 ellip_info_high = np.max(ellip_infos_[:, 2:], 1) / 2 ellip_info_mean = np.mean(ellip_infos_[:, 2:], 1) / 4 pred_anno, pred_seq_pro, summary_str, loss, self.accu, self.accu_iou = sess.run( fetches=[ self.pred_annotation, self.pro, self.summary_op, self.loss, self.accu_tensor, self.accu_iou_tensor ], #fetches=[self.pred_anno_lower, self.pro, self.summary_op, self.loss, self.accu_tensor_lower, self.accu_iou_tensor_lower], feed_dict={ self.images: images_, self.annotations: annos_, self.lr: self.learning_rate, self.keep_prob: 1, self.input_keep_prob: 1, self.cur_batch_size: cfgs.batch_size, self.coord_x_tensor: coord_map_x_cur, self.coord_y_tensor: coord_map_y_cur, self.ellip_axis: ellip_info_mean }) #View result self.view_valid(filenames, pred_anno, pred_seq_pro, images_, step) writer.add_summary(summary_str, global_step=step) thresh = cfgs.start_thresh min_ellip_loss = 1000 min_thresh = thresh for i in range(cfgs.test_thresh_num): test_pred_anno = sess.run( fetches=[self.pred_anno_test_thresh], feed_dict={ self.pro_find: pred_seq_pro, self.test_thresh: thresh, self.cur_batch_size: cfgs.batch_size }) test_pred_anno = np.squeeze(np.array(test_pred_anno), axis=0) self.calculate_acc( cur_ims[:, 2:cfgs.ANNO_IMAGE_SIZE[0] + 2, :, :].copy(), filenames, test_pred_anno, pred_seq_pro, annos_, ellip_infos_, True, if_epoch) if self.ellip_acc < min_ellip_loss: min_ellip_loss = self.ellip_acc min_thresh = thresh thresh += cfgs.interval self.find_thresh = self.find_thresh * ( 1 - cfgs.thresh_lr) + min_thresh * cfgs.thresh_lr sum_acc += self.accu sum_acc_iou += self.accu_iou sum_acc_ellip += min_ellip_loss mean_acc = sum_acc / count mean_acc_iou = sum_acc_iou / count mean_acc_ellip = sum_acc_ellip / count #3. calculate loss total_loss += loss #4. time consume time_consumed = time.time() - t0 time_per_batch = time_consumed / count line = 'epoch %5d\t lr = %g\t count = %4d\t loss = %.4f\t mean_loss=%.4f\t train_ellip_acc = %.2f\t choose_thresh = %.2f\t cur_thresh = %.3f\t time = %.2f' % ( epoch, self.learning_rate, count, loss, (total_loss / count), min_ellip_loss, min_thresh, self.find_thresh, time_per_batch) utils.clear_line(len(line)) print('\r' + line, end='') #End valid data #count -= 1 print( '\nepoch %5d\t learning_rate = %g\t loss = %.4f\t valid_accuracy = %.2f%%\t valid_iou_accuracy = %.2f%%\t valid_ellip_acc = %.2f' % (epoch, self.learning_rate, total_loss / count, sum_acc / count, sum_acc_iou / count, sum_acc_ellip / count)) print('Take time %3.1f' % (time.time() - t0)) except tf.errors.OutOfRangeError: print('Error!')