def softmax_cross_entropy_loss(self): self._log_accuracy() wsummary.variable_summaries_v2(self.gt_classes, "gt_classes") wsummary.variable_summaries_v2(self.pred_class_logits, "pred_class_logits") scores = tf.stop_gradient(tf.reshape(self.proposals[ED_SCORES], [-1])) #weights = tf.abs(scores-0.5)*4 weights = tf.minimum(tf.pow(tf.abs(scores - 0.5), 2) * 100, 1.0) weights = tf.stop_gradient(weights) wsummary.histogram_or_scalar(weights, "cls_loss_weights") if self.cfg.MODEL.ROI_HEADS.POS_LABELS_THRESHOLD > 1e-3: with tf.name_scope("modify_gtclasses"): threshold = self.cfg.MODEL.ROI_HEADS.POS_LABELS_THRESHOLD gt_classes = self.gt_classes gt_classes = tf.where(tf.greater(scores, threshold), gt_classes, tf.zeros_like(gt_classes)) classes_loss = tf.losses.sparse_softmax_cross_entropy( logits=self.pred_class_logits, labels=gt_classes, loss_collection=None, reduction=tf.losses.Reduction.NONE) else: classes_loss = tf.losses.sparse_softmax_cross_entropy( logits=self.pred_class_logits, labels=self.gt_classes, loss_collection=None, reduction=tf.losses.Reduction.NONE) classes_loss = weights * classes_loss classes_loss = tf.reduce_mean(classes_loss) wsummary.histogram_or_scalar(classes_loss, "fast_rcnn/classes_loss") return classes_loss * self.cfg.MODEL.ROI_HEADS.BOX_CLS_LOSS_SCALE
def get_pred_iou_lossv5(self, threshold=None): ''' 使用proposals bboxes与gtbboxes的iou作为目标 :return: ''' with tf.name_scope("get_pred_iouv0_loss"): gt_scores = self.proposals[ED_SCORES] gt_scores = tf.stop_gradient(tf.reshape(gt_scores, [-1])) pred_iou_logits = self.pred_iou_logits pred_iou_logits = tf.reshape(pred_iou_logits, [-1]) if threshold is not None: pred_score = pred_iou_logits mask0 = tf.greater(gt_scores, threshold) mask1 = tf.logical_and( tf.logical_not(mask0), tf.greater_equal(pred_score, threshold + 0.05)) mask = tf.logical_or(mask0, mask1) gt_scores = tf.boolean_mask(gt_scores, mask) pred_iou_logits = tf.boolean_mask(pred_iou_logits, mask) wsummary.histogram_or_scalar(gt_scores, "gt_iou_by_matcher") loss = tf.losses.absolute_difference( labels=gt_scores, predictions=pred_iou_logits, loss_collection=None, reduction=tf.losses.Reduction.NONE) return tf.reduce_mean(loss)
def smooth_l1_loss(self): """ Compute the smooth L1 loss for box regression. Returns: scalar Tensor """ #gt_anchor_deltas = self.box2box_transform.get_deltas(self.anchors,self.gt_boxes,gt_objectness_logits_i,indices) with tf.name_scope("box_regression_loss"): gt_proposal_deltas = self.box2box_transform.get_deltas_by_proposals_data( self.proposals) batch_size, box_nr, box_dim = wmlt.combined_static_and_dynamic_shape( gt_proposal_deltas) gt_proposal_deltas = tf.reshape(gt_proposal_deltas, [batch_size * box_nr, box_dim]) cls_agnostic_bbox_reg = self.pred_proposal_deltas.get_shape( ).as_list()[-1] == box_dim num_classes = self.pred_class_logits.get_shape().as_list()[-1] fg_num_classes = num_classes - 1 # Box delta loss is only computed between the prediction for the gt class k # (if 0 <= k < bg_class_ind) and the target; there is no loss defined on predictions # for non-gt classes and background. # Empty fg_inds produces a valid loss of zero as long as the size_average # arg to smooth_l1_loss is False (otherwise it uses mean internally # and would produce a nan loss). fg_inds = tf.greater(self.gt_classes, 0) gt_proposal_deltas = tf.boolean_mask(gt_proposal_deltas, fg_inds) pred_proposal_deltas = tf.boolean_mask(self.pred_proposal_deltas, fg_inds) gt_logits_i = tf.boolean_mask(self.gt_classes, fg_inds) if not cls_agnostic_bbox_reg: pred_proposal_deltas = tf.reshape( pred_proposal_deltas, [-1, fg_num_classes, box_dim]) pred_proposal_deltas = wmlt.select_2thdata_by_index_v2( pred_proposal_deltas, gt_logits_i - 1) loss_box_reg = tf.losses.huber_loss( predictions=pred_proposal_deltas, labels=gt_proposal_deltas, loss_collection=None, reduction=tf.losses.Reduction.SUM, ) num_samples = wmlt.num_elements(self.gt_classes) # The loss is normalized using the total number of regions (R), not the number # of foreground regions even though the box regression loss is only defined on # foreground regions. Why? Because doing so gives equal training influence to # each foreground example. To see how, consider two different minibatches: # (1) Contains a single foreground region # (2) Contains 100 foreground regions # If we normalize by the number of foreground regions, the single example in # minibatch (1) will be given 100 times as much influence as each foreground # example in minibatch (2). Normalizing by the total number of regions, R, # means that the single example in minibatch (1) and each of the 100 examples # in minibatch (2) are given equal influence. loss_box_reg = loss_box_reg / num_samples wsummary.histogram_or_scalar(loss_box_reg, "fast_rcnn/box_reg_loss") return loss_box_reg * self.cfg.MODEL.ROI_HEADS.BOX_REG_LOSS_SCALE
def softmax_cross_entropy_loss(self): """ Compute the softmax cross entropy loss for box classification. Returns: scalar Tensor """ self._log_accuracy() wsummary.variable_summaries_v2(self.gt_classes, "gt_classes") wsummary.variable_summaries_v2(self.pred_class_logits, "pred_class_logits") if self.cfg.MODEL.ROI_HEADS.POS_LABELS_THRESHOLD > 1e-3: with tf.name_scope("modify_gtclasses"): threshold = self.cfg.MODEL.ROI_HEADS.POS_LABELS_THRESHOLD scores = tf.reshape(self.proposals[ED_SCORES], [-1]) gt_classes = self.gt_classes gt_classes = tf.where(tf.greater(scores, threshold), gt_classes, tf.zeros_like(gt_classes)) classes_loss = tf.losses.sparse_softmax_cross_entropy( logits=self.pred_class_logits, labels=gt_classes, loss_collection=None, reduction=tf.losses.Reduction.MEAN) else: classes_loss = tf.losses.sparse_softmax_cross_entropy( logits=self.pred_class_logits, labels=self.gt_classes, loss_collection=None, reduction=tf.losses.Reduction.MEAN) wsummary.histogram_or_scalar(classes_loss, "fast_rcnn/classes_loss") return classes_loss * self.cfg.MODEL.ROI_HEADS.BOX_CLS_LOSS_SCALE
def losses(self): with tf.variable_scope("RPNLoss"): gt_objectness_logits, gt_anchor_deltas = self._get_ground_truth() #In a image, all anchors concated togather and sample, Detectron2 use the same strategy pos_idx, neg_idx = subsample_labels(gt_objectness_logits, self.batch_size_per_image, self.positive_fraction) batch_size = self.pred_objectness_logits[0].get_shape().as_list()[0] num_cell_anchors = self.pred_objectness_logits[0].get_shape().as_list()[-1] #RPN num classes==1 box_dim = self.pred_anchor_deltas[0].get_shape().as_list()[-1]//num_cell_anchors pred_objectness_logits = [tf.reshape(x,[batch_size,-1]) for x in self.pred_objectness_logits] pred_objectness_logits = tf.concat(pred_objectness_logits,axis=1) pred_anchor_deltas = [tf.reshape(x,[batch_size,-1,box_dim]) for x in self.pred_anchor_deltas] pred_anchor_deltas = tf.concat(pred_anchor_deltas,axis=1) #shape=[B,-1,4] pred_objectness_logits = tf.reshape(pred_objectness_logits,[-1]) anchors = tf.tile(self.anchors,[batch_size,1,1]) anchors = tf.reshape(anchors,[-1,box_dim]) pred_anchor_deltas = tf.reshape(pred_anchor_deltas,[-1,box_dim]) if global_cfg.GLOBAL.DEBUG: with tf.device(":/cpu:0"): with tf.name_scope("rpn_sampled_box"): log_anchors = self.anchors*tf.ones([batch_size,1,1]) logmask = tf.reshape(pos_idx,[batch_size,-1]) wsummary.detection_image_summary_by_logmask(images=self.inputs[IMAGE],boxes=log_anchors, logmask=logmask) valid_mask = tf.logical_or(pos_idx,neg_idx) gt_objectness_logits = tf.reshape(gt_objectness_logits,[-1]) gt_objectness_logits = tf.boolean_mask(gt_objectness_logits,valid_mask) pred_objectness_logits = tf.boolean_mask(pred_objectness_logits,valid_mask) gt_anchor_deltas = tf.reshape(gt_anchor_deltas,[-1,box_dim]) gt_anchor_deltas = tf.boolean_mask(gt_anchor_deltas,pos_idx) pred_anchor_deltas = tf.boolean_mask(pred_anchor_deltas,pos_idx) anchors = tf.boolean_mask(anchors,pos_idx) pred_anchor_deltas = self.box2box_transform.apply_deltas(deltas=pred_anchor_deltas,boxes=anchors) objectness_loss, localization_loss = rpn_losses_giou( gt_objectness_logits, gt_anchor_deltas, pred_objectness_logits, pred_anchor_deltas, ) if global_cfg.GLOBAL.SUMMARY_LEVEL<=SummaryLevel.INFO: with tf.name_scope("RPNCorrectRatio"): ratio = wnn.sigmoid_accuracy_ratio(logits=pred_objectness_logits,labels=gt_objectness_logits) tf.summary.scalar("rpn_accuracy_ratio",ratio) normalizer = 1.0 / (batch_size* self.batch_size_per_image) loss_cls = objectness_loss * normalizer # cls: classification loss loss_loc = localization_loss * normalizer # loc: localization loss losses = {"loss_rpn_cls": loss_cls, "loss_rpn_loc": loss_loc} wsummary.histogram_or_scalar(loss_cls,"rpn/cls_loss") wsummary.histogram_or_scalar(loss_loc,"rpn/loc_loss") return losses
def forward(self, x, bboxes,img_size): """ Args: x (list[Tensor]): tensor shape is [batch_size,H,W,C] resolution from high to low bboxes:[batch_size,box_nr,4] img_size:[2],(H,W) Returns: Tensor: A tensor of shape (M, output_size, output_size,C) where M is the total number of boxes aggregated over all batch images and C is the number of channels in `x`. which means M = batch_size*box_nr """ assert isinstance(x, list),"Arguments to pooler must be lists" level_num = len(x) with tf.name_scope("ROIPoolers"): if level_num == 1: return self.level_pooler(x[0], bboxes) level_assignments = assign_boxes_to_levels( bboxes, 0, level_num-1, self.canonical_box_size, self.canonical_level,img_size ) self.level_assignments = level_assignments features = [] for net in x: features.append(self.level_pooler(net,bboxes)) if isinstance(features[0],(list,tuple)): features = [tf.stack(x,axis=1) for x in zip(*features)] else: features = tf.stack(features, axis=1) level_assignments = tf.reshape(level_assignments,[-1]) if global_cfg.GLOBAL.SUMMARY_LEVEL<=SummaryLevel.DEBUG: wsummary.histogram_or_scalar(level_assignments,"level_assignments") if isinstance(features,(list,tuple)): output = [wmlt.batch_gather(x, level_assignments) for x in features] else: output = wmlt.batch_gather(features,level_assignments) return output
def preprocess_image(self, batched_inputs): """ Normalize, pad and batch the input images. """ with tf.name_scope(f"preprocess_image_{self.cfg.MODEL.PREPROCESS}"): b_img = batched_inputs[IMAGE] if self.cfg.MODEL.PREPROCESS == "ton1p1": b_img = (b_img - 127.5) / 127.5 elif self.cfg.MODEL.PREPROCESS == "m0v1": b_img = MetaArch.m0v1(b_img) elif self.cfg.MODEL.PREPROCESS == "subimagenetmean": channel_means = [123.68, 116.779, 103.939] b_img = b_img - [[[channel_means]]] elif self.cfg.MODEL.PREPROCESS == "standardization": b_img = tf.image.per_image_standardization(b_img) elif self.cfg.MODEL.PREPROCESS == "none": pass else: raise ValueError( f"Error preprocess type {self.cfg.MODEL.PREPROCESS}") wsummary.histogram_or_scalar(b_img, "preprocessed_image") batched_inputs[IMAGE] = b_img return batched_inputs
def get_pred_iou_lossv0(self, threshold=None): ''' 使用proposals bboxes与gtbboxes的iou作为目标 :return: ''' with tf.name_scope("get_pred_iouv0_loss"): gt_scores = self.proposals[ED_SCORES] gt_scores = tf.stop_gradient(tf.reshape(gt_scores, [-1])) pred_iou_logits = self.pred_iou_logits pred_iou_logits = tf.reshape(pred_iou_logits, [-1]) if threshold is not None: pred_score = tf.nn.sigmoid(pred_iou_logits) mask0 = tf.greater(gt_scores, threshold) mask1 = tf.logical_and( tf.logical_not(mask0), tf.greater_equal(pred_score, threshold + 0.05)) mask = tf.logical_or(mask0, mask1) gt_scores = tf.boolean_mask(gt_scores, mask) pred_iou_logits = tf.boolean_mask(pred_iou_logits, mask) wsummary.histogram_or_scalar(gt_scores, "gt_iou_by_matcher") loss = wnn.sigmoid_cross_entropy_with_logits_FL( labels=gt_scores, logits=pred_iou_logits) return tf.reduce_mean(loss)
def get_pred_centerness_loss(self): with tf.name_scope("get_pred_centerness_loss"): gt_proposal_deltas = wmlt.batch_gather( self.proposals.gt_boxes, tf.nn.relu(self.proposals.indices)) batch_size, box_nr, box_dim = wmlt.combined_static_and_dynamic_shape( gt_proposal_deltas) gt_proposal_deltas = tf.reshape(gt_proposal_deltas, [batch_size * box_nr, box_dim]) proposal_bboxes = tf.reshape(self.proposals.boxes, [batch_size * box_nr, box_dim]) cls_agnostic_bbox_reg = self.pred_proposal_deltas.get_shape( ).as_list()[-1] == box_dim num_classes = self.pred_class_logits.get_shape().as_list()[-1] fg_num_classes = num_classes - 1 pred_iou_logits = self.pred_iou_logits fg_inds = tf.greater(self.gt_classes, 0) gt_proposal_deltas = tf.boolean_mask(gt_proposal_deltas, fg_inds) pred_proposal_deltas = tf.boolean_mask(self.pred_proposal_deltas, fg_inds) proposal_bboxes = tf.boolean_mask(proposal_bboxes, fg_inds) gt_logits_i = tf.boolean_mask(self.gt_classes, fg_inds) pred_iou_logits_pos = tf.reshape( tf.boolean_mask(pred_iou_logits, fg_inds), [-1]) if not cls_agnostic_bbox_reg: pred_proposal_deltas = tf.reshape( pred_proposal_deltas, [-1, fg_num_classes, box_dim]) pred_proposal_deltas = wmlt.select_2thdata_by_index_v2( pred_proposal_deltas, gt_logits_i - 1) pred_bboxes = self.box2box_transform.apply_deltas( pred_proposal_deltas, boxes=proposal_bboxes) pred_bboxes = odb.to_cxyhw(proposal_bboxes) gt_bboxes = odb.to_cxyhw(gt_proposal_deltas) deltas = tf.abs(gt_bboxes[..., :2] - pred_bboxes[..., :2]) * 2 wsummary.histogram_or_scalar(deltas, "centerness_deltas") centerness = 1 - tf.reduce_max( deltas / (gt_bboxes[..., 2:] + 1e-8), axis=-1, keepdims=False) wsummary.histogram_or_scalar(centerness, "centerness") loss_pos = wnn.sigmoid_cross_entropy_with_logits_FL( labels=centerness, logits=pred_iou_logits_pos) wsummary.histogram_or_scalar(tf.nn.sigmoid(pred_iou_logits_pos), "pred_centerness") loss_pos = tf.reduce_mean(loss_pos) tf.summary.scalar("centerness_loss", loss_pos) loss = loss_pos return loss
def losses(self): """ Args: For `gt_classes` and `gt_anchors_deltas` parameters, see :meth:`FCOSGIou.get_ground_truth`. Their shapes are (N, R) and (N, R, 4), respectively, where R is the total number of anchors across levels, i.e. sum(Hi x Wi x A) For `pred_class_logits` and `pred_anchor_deltas`, see :meth:`FCOSGIouHead.forward`. Returns: dict[str: Tensor]: mapping from a named loss to a scalar tensor storing the loss. Used during training only. The dict keys are: "loss_cls" and "loss_box_reg" """ assert len(self.pred_logits[0].get_shape()) == 4, "error logits dim" gt_results = self._get_ground_truth() loss_cls_list = [] loss_regression_list = [] loss_center_ness_list = [] total_num_foreground = [] img_size = tf.shape(self.batched_inputs[IMAGE])[1:3] for i, gt_results_item in enumerate(gt_results): gt_classes = gt_results_item['g_classes'] gt_boxes = gt_results_item['g_boxes'] g_center_ness = gt_results_item['g_center_ness'] pred_class_logits = self.pred_logits[i] pred_regression = self.pred_regression[i] pred_center_ness = self.pred_center_ness[i] foreground_idxs = (gt_classes > 0) num_foreground = tf.reduce_sum(tf.cast(foreground_idxs, tf.int32)) total_num_foreground.append(num_foreground) gt_classes_target = tf.one_hot(gt_classes, depth=self.num_classes + 1) gt_classes_target = gt_classes_target[..., 1:] # pred_center_ness = tf.expand_dims(pred_center_ness, axis=-1) wsummary.histogram_or_scalar(pred_center_ness, "center_ness") # logits loss loss_cls = tf.reduce_sum( wnn.sigmoid_cross_entropy_with_logits_FL( labels=gt_classes_target, logits=pred_class_logits, alpha=self.focal_loss_alpha, gamma=self.focal_loss_gamma)) # regression loss pred_boxes = self.box2box_transform.apply_deltas( regression=pred_regression, img_size=img_size) if global_cfg.GLOBAL.SUMMARY_LEVEL <= SummaryLevel.DEBUG and gt_classes.get_shape( ).as_list()[0] > 1: log_boxes = self.box2box_transform.apply_deltas( regression=gt_results_item['g_regression'], img_size=img_size) log_boxes = odbox.tfabsolutely_boxes_to_relative_boxes( log_boxes, width=img_size[1], height=img_size[0]) boxes1 = tf.reshape(log_boxes[1:2], [1, -1, 4]) wsummary.detection_image_summary( images=self.batched_inputs[IMAGE][1:2], boxes=boxes1, name="FCOSGIou_decode_test") pred_center_ness = tf.boolean_mask(pred_center_ness, foreground_idxs) g_center_ness = tf.boolean_mask(g_center_ness, foreground_idxs) pred_boxes = tf.boolean_mask(pred_boxes, foreground_idxs) gt_boxes = tf.boolean_mask(gt_boxes, foreground_idxs) wsummary.histogram_or_scalar(pred_center_ness, "center_ness_pos") reg_loss_sum = (1.0 - odl.giou(pred_boxes, gt_boxes)) wmlt.variable_summaries_v2(reg_loss_sum, f"giou_loss{i}") pred_center_ness = tf.squeeze(pred_center_ness, axis=-1) reg_norm = tf.reduce_sum(g_center_ness) + 1e-5 reg_loss_sum = reg_loss_sum * g_center_ness wmlt.variable_summaries_v2(reg_loss_sum, f"loss_sum{i}") loss_box_reg = tf.reduce_sum(reg_loss_sum) * 300 / reg_norm wmlt.variable_summaries_v2(loss_box_reg, f"box_reg_loss_{i}") loss_center_ness = 0.5 * tf.nn.sigmoid_cross_entropy_with_logits( labels=g_center_ness, logits=pred_center_ness) loss_center_ness = tf.reduce_sum(loss_center_ness) * 0.1 wmlt.variable_summaries_v2(loss_center_ness, f"center_ness_loss{i}") loss_cls_list.append(loss_cls) loss_regression_list.append(loss_box_reg) loss_center_ness_list.append(loss_center_ness) total_num_foreground = tf.to_float( tf.maximum(tf.add_n(total_num_foreground), 1)) return { "fcos_loss_cls": tf.add_n(loss_cls_list) / total_num_foreground, "fcos_loss_center_ness": tf.add_n(loss_center_ness_list) / total_num_foreground, "fcos_loss_box_reg": tf.add_n(loss_regression_list) / total_num_foreground }
def build_net_run_on_multi_gpus_nccl(self): if not os.path.exists(self.log_dir): wmlu.create_empty_dir(self.log_dir) if not os.path.exists(self.ckpt_dir): wmlu.create_empty_dir(self.ckpt_dir) '''if self.cfg.GLOBAL.DEBUG: data[IMAGE] = tf.Print(data[IMAGE],[tf.shape(data[IMAGE]),data[ORG_HEIGHT],data[ORG_WIDTH],data[HEIGHT],data[WIDTH]],summarize=100, name="XXXXX")''' all_loss_dict = {} steps = self.cfg.SOLVER.STEPS print("Train steps:", steps) lr = wnn.build_learning_rate( self.cfg.SOLVER.BASE_LR, global_step=self.global_step, lr_decay_type=self.cfg.SOLVER.LR_DECAY_TYPE, steps=steps, decay_factor=self.cfg.SOLVER.LR_DECAY_FACTOR, total_steps=steps[-1], min_lr=1e-6, warmup_steps=self.cfg.SOLVER.WARMUP_ITERS) tf.summary.scalar("lr", lr) self.max_train_step = steps[-1] if self.cfg.SOLVER.OPTIMIZER == "Momentum": opt = wnn.str2optimizer( "Momentum", lr, momentum=self.cfg.SOLVER.OPTIMIZER_momentum) else: opt = wnn.str2optimizer(self.cfg.SOLVER.OPTIMIZER, lr) tower_grads = [] if len(self.gpus) == 0: self.gpus = [0] if len(self.cfg.SOLVER.TRAIN_SCOPES) > 1: train_scopes = self.cfg.SOLVER.TRAIN_SCOPES else: train_scopes = None if len(self.cfg.SOLVER.TRAIN_REPATTERN) > 1: train_repattern = self.cfg.SOLVER.TRAIN_REPATTERN else: train_repattern = None for i in range(len(self.gpus)): scope = tf.get_variable_scope() if i > 0: #scope._reuse = tf.AUTO_REUSE scope.reuse_variables() with tf.device(f"/gpu:{i}"): with tf.device(":/cpu:0"): data = self.data.get_next() self.input_data = data with tf.name_scope(f"GPU{self.gpus[i]}"): with tf.device(":/cpu:0"): DataLoader.detection_image_summary( data, name=f"data_source{i}") self.res_data, loss_dict = self.model.forward(data) loss_values = [] for k, v in loss_dict.items(): all_loss_dict[k + f"_stage{i}"] = v tf.summary.scalar(f"loss/{k}", v) ## #v = tf.Print(v,[k,tf.is_nan(v), tf.is_inf(v)]) ## v = tf.cond(tf.logical_or(tf.is_nan(v), tf.is_inf(v)), lambda: tf.zeros_like(v), lambda: v) loss_values.append(v) scope._reuse = tf.AUTO_REUSE '''if (i==0) and len(tf.get_collection(GRADIENT_DEBUG_COLLECTION))>0: total_loss_sum = tf.add_n(loss_values) xs = tf.get_collection(GRADIENT_DEBUG_COLLECTION) grads = tf.gradients(total_loss_sum,xs) grads = [tf.reduce_sum(tf.abs(x)) for x in grads] loss_values[0] = tf.Print(loss_values[0],grads+["grads"],summarize=100)''' grads, total_loss, variables_to_train = wnn.nget_train_opv3( optimizer=opt, loss=loss_values, scopes=train_scopes, re_pattern=train_repattern) # if self.cfg.SOLVER.FILTER_NAN_AND_INF_GRADS: grads = [list(x) for x in grads] for i, (g, v) in enumerate(grads): try: if g is not None: g = tf.where( tf.logical_or(tf.is_nan(g), tf.is_inf(g)), tf.random_normal( shape=wmlt. combined_static_and_dynamic_shape(g), stddev=1e-5), g) except: print(f"Error {g}/{v}") raise Exception("Error") grads[i][0] = g # tower_grads.append(grads) ######################## '''tower_grads[0] = [list(x) for x in tower_grads[0]] for i,(g,v) in enumerate(tower_grads[0]): tower_grads[0][i][0] = tf.Print(g,["B_"+v.name,tf.reduce_min(g),tf.reduce_mean(g),tf.reduce_max(g)])''' ######################## if self.cfg.SOLVER.CLIP_NORM > 1: avg_grads = wnn.average_grads_nccl( tower_grads, clip_norm=self.cfg.SOLVER.CLIP_NORM) else: avg_grads = wnn.average_grads_nccl(tower_grads, clip_norm=None) '''avg_grads = [list(x) for x in avg_grads] for i,(g,v) in enumerate(avg_grads): avg_grads[i][0] = tf.Print(g,[v.name,tf.reduce_min(g),tf.reduce_mean(g),tf.reduce_max(g)])''' opt0 = wnn.apply_gradientsv3(avg_grads, self.global_step, opt) opt1 = wnn.get_batch_norm_ops() self.train_op = tf.group(opt0, opt1) self.total_loss, self.variables_to_train = total_loss, variables_to_train self.loss_dict = all_loss_dict config = tf.ConfigProto(allow_soft_placement=True) #config.gpu_options.allow_growth = True self.sess = tf.Session(config=config) if self.debug_tf: self.sess = tfdbg.LocalCLIDebugWrapperSession(self.sess) print("variables to train:") wmlu.show_list(self.variables_to_train) for v in self.variables_to_train: wsummary.histogram_or_scalar(v, v.name[:-2]) wnn.log_moving_variable() self.saver = tf.train.Saver(max_to_keep=100) tf.summary.scalar("total_loss", self.total_loss) self.summary = tf.summary.merge_all() self.summary_writer = tf.summary.FileWriter(self.log_dir, self.sess.graph) init = tf.global_variables_initializer() self.sess.run(init) print("batch_norm_ops.") wmlu.show_list( [x.name for x in tf.get_collection(tf.GraphKeys.UPDATE_OPS)])
def build_net(self): if not os.path.exists(self.log_dir): wmlu.create_empty_dir(self.log_dir) if not os.path.exists(self.ckpt_dir): wmlu.create_empty_dir(self.ckpt_dir) with tf.device(":/cpu:0"): data = self.data.get_next() DataLoader.detection_image_summary(data, name="data_source") self.input_data = data '''if self.cfg.GLOBAL.DEBUG: data[IMAGE] = tf.Print(data[IMAGE],[tf.shape(data[IMAGE]),data[ORG_HEIGHT],data[ORG_WIDTH],data[HEIGHT],data[WIDTH]],summarize=100, name="XXXXX")''' self.res_data, loss_dict = self.model.forward(data) if self.model.is_training: for k, v in loss_dict.items(): tf.summary.scalar(f"loss/{k}", v) v = tf.cond(tf.logical_or(tf.is_nan(v), tf.is_inf(v)), lambda: tf.zeros_like(v), lambda: v) tf.losses.add_loss(v) elif self.cfg.GLOBAL.SUMMARY_LEVEL <= SummaryLevel.RESEARCH: research = self.cfg.GLOBAL.RESEARCH if 'result_classes' in research: print("replace labels with gtlabels.") labels = odt.replace_with_gtlabels( bboxes=self.res_data[RD_BOXES], labels=self.res_data[RD_LABELS], length=self.res_data[RD_LENGTH], gtbboxes=data[GT_BOXES], gtlabels=data[GT_LABELS], gtlength=data[GT_LENGTH]) self.res_data[RD_LABELS] = labels if 'result_bboxes' in research: print("replace bboxes with gtbboxes.") bboxes = odt.replace_with_gtbboxes( bboxes=self.res_data[RD_BOXES], labels=self.res_data[RD_LABELS], length=self.res_data[RD_LENGTH], gtbboxes=data[GT_BOXES], gtlabels=data[GT_LABELS], gtlength=data[GT_LENGTH]) self.res_data[RD_BOXES] = bboxes self.loss_dict = loss_dict if not self.model.is_training and self.cfg.GLOBAL.GPU_MEM_FRACTION > 0.1: gpu_options = tf.GPUOptions(per_process_gpu_memory_fraction=self. cfg.GLOBAL.GPU_MEM_FRACTION) config = tf.ConfigProto(allow_soft_placement=True, gpu_options=gpu_options) else: config = tf.ConfigProto(allow_soft_placement=True) if not self.model.is_training and self.cfg.GLOBAL.GPU_MEM_FRACTION <= 0.1: config.gpu_options.allow_growth = True self.sess = tf.Session(config=config) self.top_variable_name_scope = "Model" if self.model.is_training: steps = self.cfg.SOLVER.STEPS print("Train steps:", steps) lr = wnn.build_learning_rate( self.cfg.SOLVER.BASE_LR, global_step=self.global_step, lr_decay_type=self.cfg.SOLVER.LR_DECAY_TYPE, steps=steps, decay_factor=self.cfg.SOLVER.LR_DECAY_FACTOR, total_steps=steps[-1], warmup_steps=self.cfg.SOLVER.WARMUP_ITERS) tf.summary.scalar("lr", lr) opt = wnn.str2optimizer("Momentum", lr, momentum=0.9) self.max_train_step = steps[-1] self.train_op, self.total_loss, self.variables_to_train = wnn.nget_train_op( self.global_step, optimizer=opt, clip_norm=self.cfg.SOLVER.CLIP_NORM) print("variables to train:") wmlu.show_list(self.variables_to_train) for v in self.variables_to_train: wsummary.histogram_or_scalar(v, v.name[:-2]) wnn.log_moving_variable() self.saver = tf.train.Saver(max_to_keep=100) tf.summary.scalar(self.cfg.GLOBAL.PROJ_NAME + "_total_loss", self.total_loss) self.summary = tf.summary.merge_all() self.summary_writer = tf.summary.FileWriter(self.log_dir, self.sess.graph) init = tf.global_variables_initializer() self.sess.run(init) print("batch_norm_ops.") wmlu.show_list( [x.name for x in tf.get_collection(tf.GraphKeys.UPDATE_OPS)])