def on_loss_calculation_end(self, training_context): """Returns mixed inputs, pairs of targets, and lambda""" train_data = training_context['train_data'] x = None y = None x = train_data.value_list[0].copy().detach() # input y = train_data.value_list[1].copy().detach() # label model = training_context['current_model'] lam = builtins.min( builtins.max(np.random.beta(self.alpha, self.alpha), 0.3), 0.7) batch_size = int_shape(x)[0] index = arange(batch_size) index = cast(shuffle(index), 'long') this_loss = None mixed_x = None if get_backend() == 'pytorch': mixed_x = lam * x + (1 - lam) * x[index, :] pred = model(to_tensor(mixed_x, requires_grad=True)) y_a, y_b = y, y[index] this_loss = lam * self.loss_criterion(pred, y_a.long()) + ( 1 - lam) * self.loss_criterion(pred, y_b.long()) elif get_backend() == 'tensorflow': x1 = tf.gather(x, index, axis=0) y1 = tf.gather(y, index, axis=0) mixed_x = lam * x + (1 - lam) * x1 pred = model(to_tensor(mixed_x, requires_grad=True)) y_a, y_b = y, y1 this_loss = lam * self.loss_criterion( pred, y_a) + (1 - lam) * self.loss_criterion(pred, y_b) training_context['current_loss'] = training_context[ 'current_loss'] + this_loss * self.loss_weight if training_context['is_collect_data']: training_context['losses'].collect( 'mixup_loss', training_context['steps'], float(to_numpy(this_loss * self.loss_weight))) if training_context['current_batch'] == 0: for item in mixed_x: if self.save_path is None and not is_in_colab(): item = unnormalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225])(to_numpy(item)) item = unnormalize(0, 255)(item) array2image(item).save('Results/mixup_{0}.jpg'.format( get_time_suffix())) elif self.save_path is not None: item = unnormalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225])(to_numpy(item)) item = unnormalize(0, 255)(item) array2image(item).save( os.path.join(self.save_path, 'mixup_{0}.jpg'.format( get_time_suffix())))
def plot_detection_image(self, training_context): tile_images_list = [] input = None target = None output = None data_feed = training_context['data_feed'] data = training_context['train_data'] model = training_context['current_model'] output = try_map_args_and_call(model, data, data_feed) target = data['bbox'] input = data[data_feed[model.signature.inputs.key_list[0]]] input_image = self.reverse_image_transform(to_numpy(input)) targetmask = (target[:, 4] > 0.9) input_image1 = input_image.copy() target_boxes = to_numpy(xywh2xyxy(target[targetmask, :])) for box in target_boxes: plot_one_box(box, input_image1, (255, 128, 128), self.labels[box[5:]]) # input_arr=np.asarray(input_arr) tile_images_list.append(input_image1) input_image = self.reverse_image_transform(to_numpy(input)) mask = (output[:, :, 4] > 0.7) if len(output[:, :, 4]) > 0: mask2 = (argmax(softmax(output[:, :, 5:], -1), -1) != 0) mask = (mask.float() + mask2.float() == 2) output = output[mask, :] input_image2 = input_image.copy() output_boxes = to_numpy(xywh2xyxy(output[mask, :])) for box in output_boxes: plot_one_box(box, input_image2, (255, 255, 128), self.labels[np.argmax(box[5:])]) tile_images_list.append(input_image2) fig = tile_rgb_images(*tile_images_list, save_path=os.path.join( self.save_path, self.tile_image_name_prefix), imshow=True) if ctx.enable_tensorboard and ctx.summary_writer is not None: ctx.summary_writer.add_figure( training_context['training_name'] + '/plot/detection_plot', fig, global_step=training_context['steps'], close=True, walltime=time.time()) plt.close()
def on_loss_calculation_end(self, training_context): """Returns mixed inputs, pairs of targets, and lambda""" train_data = training_context['train_data'] x = None y = None if get_backend() == 'pytorch': x = train_data.value_list[0].clone() # input y = train_data.value_list[1].clone() # label elif get_backend() == 'tensorflow': x = copy.deepcopy(train_data.value_list[0]) # input y = copy.deepcopy(train_data.value_list[1]) # label model = training_context['current_model'] if self.alpha > 0: lam = np.random.beta(self.alpha, self.alpha) else: lam = 1 batch_size = int_shape(x)[0] index = cast(arange(batch_size),'int64') index=shuffle(index) this_loss=None if get_backend()=='pytorch': y_a, y_b = y, y[index] bbx1, bby1, bbx2, bby2 = self.rand_bbox(x.shape[3], x.shape[2], lam) x[:, :, bbx1:bbx2, bby1:bby2] = x[index, :, bbx1:bbx2, bby1:bby2] # adjust lambda to exactly match pixel ratio lam = 1 - ((bbx2 - bbx1) * (bby2 - bby1) / (x.shape[3] * x.shape[2])) pred = model(to_tensor(x, requires_grad=True)) this_loss = lam * self.loss_criterion(pred, y_a.long()) + (1 - lam) * self.loss_criterion(pred, y_b.long()) elif get_backend() == 'tensorflow': y1 = tf.gather(y,index,axis=0) x1= tf.gather(x,index,axis=0) y_a, y_b = y, y1 bbx1, bby1, bbx2, bby2 = self.rand_bbox(x.shape[2], x.shape[1], lam) filter=np.zeros(int_shape(x)) filter[:, bbx1:bbx2, bby1:bby2, :] =1 filter=to_tensor(x) x=x*(1-filter)+x1*filter #x[:, bbx1:bbx2, bby1:bby2, :] = x1[:, bbx1:bbx2, bby1:bby2,:] # adjust lambda to exactly match pixel ratio lam = 1 - ((bbx2 - bbx1) * (bby2 - bby1) / (x.shape[2] * x.shape[1])) pred = model(to_tensor(x, requires_grad=True)) loss1=self.loss_criterion(pred, y_a) loss2=self.loss_criterion(pred, y_b) this_loss = lam *loss1 + (1 - lam) * loss2 if training_context['current_batch'] == 0: for item in x: item = unnormalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225])(to_numpy(item)) item = unnormalize(0, 255)(item) array2image(item).save('Results/cutmix_{0}.jpg'.format(get_time_suffix())) training_context['current_loss'] = training_context['current_loss'] + this_loss *self.loss_weight if training_context['is_collect_data']: training_context['losses'].collect('cutmix_loss', training_context['steps'], float(to_numpy(this_loss * self.loss_weight)))
def on_batch_end(self, training_context): if self.unit_base is None: if training_context['total_batch'] > _session.epoch_equivalent: self.unit_base = 'batch' print( 'one epoch have {0} batches, use {1} as epoch equivalent in long epoch. ' .format(training_context['total_batch'], _session.epoch_equivalent)) else: self.unit_base = 'epoch' print('ReduceLROnPlateau reseted.') num_batches = training_context['steps'] if self.unit_base == 'batch' and training_context[ 'steps'] > 0 and training_context[ 'steps'] % _session.epoch_equivalent == 0: training_context['current_lr'] = training_context['optimizer'].lr history = training_context['losses'].get( self.monitor, training_context['metrics'].get( self.monitor, training_context['losses']['total_losses'])) steps, values = zip(*history) current = to_numpy(values[-min(5, len(values)):]).mean() if current is None: warnings.warn( 'Reduce LR on plateau conditioned on metric `%s` ' 'which is not available. Available metrics are: %s' % (self.monitor, ','.join( training_context['metrics'].keys_list)), RuntimeWarning) else: if self.in_cooldown(): self.cooldown_counter -= 1 self.wait = 0 if self.monitor_op(current, self.best): self.best = current self.wait = 0 elif not self.in_cooldown(): self.wait += 1 if self.wait >= self.patience: old_lr = float(training_context['optimizer'].lr) if old_lr > self.min_lr: new_lr = old_lr * self.factor new_lr = max(new_lr, self.min_lr) training_context['optimizer'].adjust_learning_rate( new_lr, True) if self.verbose > 0: print( '\nEpoch %05d: ReduceLROnPlateau reducing ' 'learning rate to %s.' % (training_context['current_epoch'], new_lr)) self.cooldown_counter = self.cooldown self.wait = 0
def plot_tile_image(self, training_context): output = None legend = [] data_feed = training_context['data_feed'] data = training_context['train_data'] model = training_context['current_model'].eval() dataprovider = enforce_singleton(ctx.get_data_provider()) output = to_numpy(model(data[ data_feed['input']])) if data_feed['input'] in data else to_numpy( data['output']) model.train() reverse_image_transform = dataprovider.reverse_image_transform if reverse_image_transform is not None: for i in range(len(output)): self.output_arr.append(reverse_image_transform(output[i])) if len(self.output_arr) == self.row: self.tile_images_list.append(self.output_arr) if len(self.tile_images_list) == self.row: self.sample_enough = True break self.output_arr = [] legend.append('output') if self.sample_enough: fig = tile_rgb_images(*self.tile_images_list, row=self.row, save_path=os.path.join( self.save_path, self.tile_image_name_prefix), imshow=True, legend=None) if ctx.enable_tensorboard and ctx.summary_writer is not None: ctx.summary_writer.add_figure( training_context['training_name'] + '/plot/tile_image', fig, global_step=training_context['steps'], close=True, walltime=time.time()) plt.close()
def on_epoch_end(self, training_context): training_context['current_lr'] = training_context['optimizer'].lr if self.unit_base == 'epoch': steps = None history = None if self.monitor in training_context['losses']: steps, history = training_context['losses'].get_series( self.monitor) elif self.monitor in training_context['metrics']: steps, history = training_context['metrics'].get_series( self.monitor) else: steps, history = training_context['losses'].get_series( 'total_losses') current = to_numpy(history[-min(5, len(history)):]).mean() if current is None: warnings.warn( 'Reduce LR on plateau conditioned on metric `%s` ' 'which is not available. Available metrics are: %s' % (self.monitor, ','.join( training_context['metrics'].keys_list)), RuntimeWarning) else: if self.in_cooldown(): self.cooldown_counter -= 1 self.wait = 0 if self.monitor_op(current, self.best): self.best = current self.wait = 0 elif not self.in_cooldown(): self.wait += 1 if self.wait >= self.patience: old_lr = float(training_context['optimizer'].lr) if old_lr > self.min_lr: new_lr = old_lr * self.factor new_lr = max(new_lr, self.min_lr) training_context['optimizer'].adjust_learning_rate( new_lr, True) if self.verbose > 0: print( '\nEpoch %05d: ReduceLROnPlateau reducing ' 'learning rate to %s.' % (training_context['current_epoch'] + 1, new_lr)) self.cooldown_counter = self.cooldown self.wait = 0
def __init__(self, epoch_inteval=-1, batch_inteval=-1, save_path: str = 'results', reverse_image_transform=None, palette=None, background=(120, 120, 120), name_prefix: str = 'segtile_image_{0}.png', imshow=False): super(SegTileImageCallback, self).__init__(epoch_inteval, batch_inteval, save_path, imshow) self.is_in_ipython = is_in_ipython() self.is_in_colab = is_in_colab() self.palette = palette self.tile_image_name_prefix = name_prefix self.reverse_image_transform = reverse_image_transform self.background = np.expand_dims( np.expand_dims(to_numpy(background), 0), 0)
def __init__(self, frequency=-1, unit='batch', save_path: str = 'results', reverse_image_transform=None, labels=None, palette=None, background=(120, 120, 120), name_prefix: str = 'detection_plot_image_{0}.png', imshow=False): super(DetectionPlotImageCallback, self).__init__(frequency, unit, save_path, imshow) self.is_in_ipython = is_in_ipython() self.is_in_colab = is_in_colab() self.labels = labels self.palette = palette self.tile_image_name_prefix = name_prefix self.reverse_image_transform = reverse_image_transform self.background = np.expand_dims( np.expand_dims(to_numpy(background), 0), 0)
def __init__(self, frequency=-1, unit='batch', save_path: str = 'results', reverse_image_transform=None, is_label_mask=False, palette=None, background=(120, 120, 120), name_prefix: str = 'segtile_image_{0}.png', imshow=False): super(SegTileImageCallback, self).__init__(frequency, unit, save_path, imshow) self.is_in_ipython = is_in_ipython() self.is_in_colab = is_in_colab() self.is_label_mask = is_label_mask self.palette = palette self.tile_image_name_prefix = name_prefix self.reverse_image_transform = reverse_image_transform self.background = to_numpy(background)
def plot_tile_image(self, training_context): tile_images_list = [] input = None target = None output = None data_feed = training_context['data_feed'] data = training_context['train_data'] model = training_context['current_model'] if len(data_feed) == 3 and len(data) <= 3: input = data[data_feed.value_list[0]] output = data[data_feed.value_list[1]] target = data[data_feed.value_list[2]] #loss function signature loss(output,target) else: for data_key in data.key_list: if self.include_input and data_key == model.signature.inputs.key_list[ 0]: input = data[data_key] elif self.include_output and data_key == model.signature.outputs.key_list[ 0]: output = data[data_key] elif self.include_input and model.signature.inputs.key_list[ 0] in data_feed and data_feed[ model.signature.inputs.key_list[0]] == data_key: input = data[data_key] elif self.include_output and model.signature.outputs.key_list[ 0] in data_feed and data_feed[ model.signature.outputs.key_list[0]] == data_key: output = data[data_key] elif self.include_target and data_key == 'target': output = data['target'] elif 'output' in data_key or 'pred' in data_key: output = data[data_key] elif ('target' in data_key or 'label' in data_key or 'mask' in data_key): target = data[data_key] if output is None and 'tensor' in model.__class__.__name__.lower(): output = model.clone() elif output is None and input is not None: output = model(input) if self.include_input and input is not None: if self.reverse_image_transform is not None: input_arr = [] for i in range(len(input)): input_arr.append( self.reverse_image_transform(to_numpy(input[i]))) tile_images_list.append(input_arr) else: input_arr = to_numpy(input).transpose([ 0, 2, 3, 1 ]) if get_backend() != 'tensorflow' else to_numpy(input) tile_images_list.append(input_arr * 127.5 + 127.5) if self.include_target and target is not None: if self.reverse_image_transform is not None: target_arr = [] for i in range(len(target)): target_arr.append( self.reverse_image_transform(to_numpy(target[i]))) tile_images_list.append(target_arr) else: target_arr = to_numpy(target).transpose([ 0, 2, 3, 1 ]) if get_backend() != 'tensorflow' else to_numpy(target) tile_images_list.append(target_arr * 127.5 + 127.5) if self.include_output and output is not None: if self.reverse_image_transform is not None: output_arr = [] for i in range(len(output)): output_arr.append( self.reverse_image_transform(to_numpy(output[i]))) tile_images_list.append(output_arr) else: output_arr = to_numpy(output).transpose([ 0, 2, 3, 1 ]) if get_backend() != 'tensorflow' else to_numpy(output) tile_images_list.append(output_arr * 127.5 + 127.5) # if self.tile_image_include_mask: # tile_images_list.append(input*127.5+127.5) tile_rgb_images(*tile_images_list, row=self.row, save_path=os.path.join(self.save_path, self.tile_image_name_prefix), imshow=True)
def on_optimization_step_start(self, training_context): if get_backend() == 'pytorch': if (training_context['current_epoch'] * training_context['total_batch'] + training_context['current_batch'] ) % self.batch_inteval == 0: grad_dict = {} if 'grads_state' not in training_context: training_context['grads_state'] = OrderedDict() training_context['grads_state']['first_layer'] = [] training_context['grads_state']['last_layer'] = [] if training_context['current_batch'] == 0 and training_context[ 'current_epoch'] > 0: #relocate the first/ last layers self.first_layer = '' self.last_layer = '' if self.first_layer != '' and self.last_layer != '': for i, (k, v) in enumerate(training_context['current_model']. named_parameters()): if v is not None and v.requires_grad == True: if k == self.first_layer: training_context['grads_state'][ 'first_layer'].append( np.abs( to_numpy(0 if v.grad is None else v .grad)).mean()) elif k == self.last_layer: training_context['grads_state'][ 'last_layer'].append( np.abs( to_numpy(0 if v.grad is None else v .grad)).mean()) else: for i, (k, v) in enumerate(training_context['current_model']. named_parameters()): if v is not None and v.requires_grad == True: grad_dict[k] = np.abs( to_numpy(0 if v.grad is None else v.grad)) if grad_dict[k].ndim > 1: if self.first_layer == '': self.first_layer = k self.last_layer = k if self.first_layer != '' and self.first_layer in grad_dict: training_context['grads_state']['first_layer'].append( grad_dict[self.first_layer].mean()) if self.last_layer != '' and self.last_layer in grad_dict: training_context['grads_state']['last_layer'].append( grad_dict[self.last_layer].mean()) if len(training_context['grads_state'] ['first_layer']) > 0 and len( training_context['grads_state']['last_layer']) > 0: self.lines.append( '{0:<16s} first_layer gradients: {1:<8.3e}| last_layer gradients: {2:<8.3e}' .format( training_context['current_model'].name, training_context['grads_state']['first_layer'][-1], training_context['grads_state']['last_layer'][-1])) elif get_backend() == 'tensorflow': if (training_context['current_epoch'] * training_context['total_batch'] + training_context['current_batch'] ) % self.batch_inteval == 0: if 'grads_state' not in training_context: training_context['grads_state'] = OrderedDict() training_context['grads_state']['first_layer'] = [] training_context['grads_state']['last_layer'] = [] grads_and_vars = list( training_context['optimizer'].grads_and_vars) grads_and_vars = [ gv for gv in grads_and_vars if gv[1].trainable and reduce_any(not_equal(gv[0], 0)) ] training_context['grads_state']['first_layer'].append( np.abs(to_numpy(grads_and_vars[0][0])).mean()) training_context['grads_state']['last_layer'].append( np.abs(to_numpy(grads_and_vars[-1][0])).mean()) if len(training_context['grads_state'] ['first_layer']) > 0 and len( training_context['grads_state']['last_layer']) > 0: self.lines.append( '{0:<16s} first_layer gradients: {1:<8.3e}| last_layer gradients: {2:<8.3e}' .format( training_context['current_model'].name, training_context['grads_state']['first_layer'][-1], training_context['grads_state']['last_layer'][-1]))
def plot_tile_image(self, training_context): tile_images_list = [] input = None target = None output = None is_label_mask = False data_feed = training_context['data_feed'] data = training_context['train_data'] model = training_context['current_model'] if model.output_shape[0] > 2: is_label_mask = True # if len(data) >= 3: for data_key in data.key_list: if data_key == data_feed[model.signature.inputs.key_list[0]]: input = data[data_feed[model.signature.inputs.key_list[0]]] training_context['current_model'].eval() output = model(input) training_context['current_model'].train() elif ( 'target' in data_key or 'label' in data_key or 'mask' in data_key ) and not 'output' in data_key and data_key in data_feed.value_list: target = to_numpy(data[data_key]) if 'alpha' not in data: output = np.argmax(to_numpy(output), 1) if is_label_mask: target = label2color(target, self.palette) output = label2color(output, self.palette) else: output = to_numpy(output[:, 1, :, :] * argmax(output, 1)) target = to_numpy(data['alpha']) input_arr = [] input = to_numpy(input) for i in range(len(input)): input_arr.append(self.reverse_image_transform(input[i])) # input_arr=np.asarray(input_arr) tile_images_list.append(input_arr) if is_label_mask: tile_images_list.append(target) tile_images_list.append(output) else: target_arr = np.expand_dims(target, -1) output_arr = np.expand_dims(output, -1) if 'alpha' not in data: target_arr[target_arr > 0] = 1 background = np.ones_like(target_arr) * self.background tile_images_list.append(target_arr * input_arr + (1 - target_arr) * background) output_arr = np.expand_dims(output, -1) if 'alpha' not in data: output_arr[output_arr > 0] = 1 tile_images_list.append(output_arr * input_arr + (1 - output_arr) * background) # if self.tile_image_include_mask: # tile_images_list.append(input*127.5+127.5) tile_rgb_images(*tile_images_list, save_path=os.path.join(self.save_path, self.tile_image_name_prefix), imshow=True)
def plot_tile_image(self, training_context): tile_images_list = [] input = None target = None output = None mask = None legend = [] data_feed = training_context['data_feed'] data = training_context['train_data'] model = training_context['current_model'] dataprovider = enforce_singleton(ctx.get_data_provider()) input = to_numpy(data[data_feed['input']]) if 'tensor' in model.__class__.__name__.lower(): output = to_numpy(model.clone()) elif isinstance(model, Layer): output = to_numpy(data[data_feed['output']].copy()) if self.include_target: target = to_numpy(data[data_feed['target']].copy()) if self.include_mask: if isinstance(dataprovider.traindata.label, MaskDataset): mask = to_numpy(data[dataprovider.traindata.label.symbol]) elif isinstance(dataprovider.traindata.label, ZipDataset): for ds in dataprovider.traindata.label._datasets: if isinstance(ds, MaskDataset): mask = to_numpy(data[ds.symbol]) reverse_image_transform = dataprovider.reverse_image_transform reverse_image_transform_funcs = dataprovider.reverse_image_transform_funcs if self.include_input and input is not None: if len(reverse_image_transform_funcs) > 1: input_arr = [] for i in range(len(input)): input_arr.append(reverse_image_transform(input[i])) tile_images_list.append(input_arr) else: input_arr = to_numpy(input).transpose([ 0, 2, 3, 1 ]) if get_backend() != 'tensorflow' else to_numpy(input) max_value = input_arr.max() min_value = input_arr.min() if max_value <= 1.1 and min_value >= 0: tile_images_list.append(input_arr * 255.0) elif max_value <= 1.1 and min_value >= -1.1: tile_images_list.append(input_arr * 128 + 127) else: print(max_value, min_value) tile_images_list.append(input_arr) legend.append('input') if self.include_target and target is not None: if len(reverse_image_transform_funcs) > 1: target_arr = [] for i in range(len(target)): target_arr.append(reverse_image_transform(target[i])) tile_images_list.append(target_arr) else: target_arr = to_numpy(target).transpose([ 0, 2, 3, 1 ]) if get_backend() != 'tensorflow' else to_numpy(target) if target_arr.max() <= 1.1 and target_arr.min() >= 0: tile_images_list.append(target_arr * 255) elif target_arr.max() <= 1.1 and target_arr.min() >= -1.1: tile_images_list.append(target_arr * 128 + 127) else: tile_images_list.append(target_arr) legend.append('target') if self.include_output and output is not None: if len(reverse_image_transform_funcs) > 1: output_arr = [] for i in range(len(output)): output_arr.append(reverse_image_transform(output[i])) tile_images_list.append(output_arr) else: output_arr = to_numpy(output).transpose([ 0, 2, 3, 1 ]) if get_backend() != 'tensorflow' else to_numpy(output) if output_arr.max() <= 1.2 and output_arr.min() >= 0: tile_images_list.append(output_arr * 255) elif output_arr.max() <= 1.2 and output_arr.min() >= -1.2: tile_images_list.append(output_arr * 128 + 127) else: tile_images_list.append(output_arr) legend.append('output') # if self.tile_image_include_mask: # tile_images_list.append(input*127.5+127.5) fig = tile_rgb_images(*tile_images_list, row=self.row, save_path=os.path.join( self.save_path, self.tile_image_name_prefix), imshow=True, legend=legend) if ctx.enable_tensorboard and ctx.summary_writer is not None: ctx.summary_writer.add_figure( training_context['training_name'] + '/plot/tile_image', fig, global_step=training_context['steps'], close=True, walltime=time.time()) plt.close()
def plot_tile_image(self, training_context): axis = 1 if get_backend() == 'tensorflow': axis = -1 tile_images_list = [] input = None target = None output = None is_label_mask = False data_feed = training_context['data_feed'] data = training_context['train_data'] model = training_context['current_model'] if model.output_shape[model.filter_index] > 2: is_label_mask = True # if len(data) >= 3: for data_key in data.key_list: if data_key == data_feed[model.signature.inputs.key_list[0]]: input = data[data_feed[model.signature.inputs.key_list[0]]] model.eval() if is_label_mask: output = to_numpy(argmax(model(input), axis=axis)) else: output = to_numpy( expand_dims(cast(argmax(model(input), axis=axis), input.dtype), axis=-1)) model.train() # elif data_key == data_feed[model.signature.outputs.key_list[0]]: # output = data[data_feed[model.signature.outputs.key_list[0]]] # if output.max() < 0: # output = exp(output) elif ( 'target' in data_key or 'label' in data_key or 'mask' in data_key ) and not 'output' in data_key and data_key in data_feed.value_list: target = to_numpy(data[data_key]) output_arr = None if 'alpha' not in data: output_arr = output.copy() if is_label_mask: target = label2color(target, self.palette) output = label2color(output, self.palette) else: if get_backend() == 'tensorflow': output = output[:, :, :, 1:2] * argmax(output, axis) else: output = (output[:, 1:2, :, :] * argmax(output, axis)).transpose(0, 2, 3, 1) target = to_numpy(data['alpha']) input_arr = [] input = to_numpy(input) for i in range(len(input)): input_arr.append(self.reverse_image_transform(input[i])) # input_arr=np.asarray(input_arr) tile_images_list.append(input_arr) if is_label_mask: tile_images_list.append(target) tile_images_list.append(output) else: target_arr = target if len(target.shape) < len(int_shape(input)): if get_backend() == 'tensorflow': target_arr = np.expand_dims(target, -1) else: target_arr = np.expand_dims(target, 1) if 'alpha' not in data: target_arr[target_arr > 0] = 1 background = np.ones_like(target_arr) * self.background tile_images_list.append(target_arr * input_arr + (1 - target_arr) * background) tile_images_list.append(output_arr * input_arr + (1 - output_arr) * background) # if self.tile_image_include_mask: # tile_images_list.append(input*127.5+127.5) fig = tile_rgb_images(*tile_images_list, save_path=os.path.join( self.save_path, self.tile_image_name_prefix), imshow=True) if ctx.enable_tensorboard and ctx.summary_writer is not None: ctx.summary_writer.add_figure( training_context['training_name'] + '/plot/segtile_image', fig, global_step=training_context['steps'], close=True, walltime=time.time()) plt.close()
def on_optimization_step_start(self, training_context): if get_backend() == 'pytorch': with torch.no_grad(): if self.frequency > 0 and ( (self.unit == 'batch' and (training_context['current_batch'] + 1) % self.frequency == 0) or (self.unit == 'step' and (training_context['steps'] + 1) % self.frequency == 0) or (self.unit == 'epoch' and (training_context['current_epoch'] + 1) % self.frequency == 0)): grad_dict = OrderedDict() if 'grads_state' not in training_context: training_context['grads_state'] = OrderedDict() if training_context[ 'current_batch'] == 0 and training_context[ 'current_epoch'] > 0: # relocate the first/ last layers self.first_layer = OrderedDict() self.last_layer = OrderedDict() if len(self.first_layer) == 0 and len( self.last_layer) == 0: if training_context['current_model'][ -1].__class__.__name__ == 'ModuleDict': self.is_modulefict = True for k, v in training_context['current_model'][ -1].items(): last_layer_name = '' for name, module in v.named_modules(): if len([ pk for pk, pv in module._parameters. items() if 'bias' not in pk and pv.requires_grad ]) > 0: last_layer_name = module.relative_name if last_layer_name != '': self.last_layer[last_layer_name] = k first_layer_name = '' last_layer_name = '' for k, v in training_context[ 'current_model'].named_modules(): if len([ pk for pk, pv in v._parameters.items() if 'bias' not in pk and pv is not None and pv.requires_grad ]) > 0: if first_layer_name == '': first_layer_name = v.relative_name self.first_layer[ first_layer_name] = 'first_layer' if not self.is_modulefict: last_layer_name = v.relative_name if last_layer_name != '' and not self.is_modulefict: self.last_layer[last_layer_name] = 'last_layer' for name, module in training_context[ 'current_model'].named_modules(): if module.relative_name in self.first_layer or module.relative_name in self.last_layer: grads_data = [ np.abs(np.reshape(to_numpy(pv.grad.data), -1)) for pk, pv in module._parameters.items() if 'bias' not in pk and pv is not None and pv.requires_grad and pv.grad is not None ] weights_data = [ np.abs(np.reshape(to_numpy(pv.data), -1)) for pk, pv in module._parameters.items() if 'bias' not in pk and pv is not None and pv.requires_grad ] if ctx.enable_tensorboard and ctx.summary_writer is not None: ctx.summary_writer.add_histogram( training_context['training_name'] + '/gradients/' + self.first_layer[module.relative_name] if module.relative_name in self.first_layer else self.last_layer[module.relative_name], np.concatenate(grads_data, axis=0), training_context['steps']) ctx.summary_writer.add_histogram( training_context['training_name'] + '/weights/' + self.first_layer[module.relative_name] if module.relative_name in self.first_layer else self.last_layer[module.relative_name], np.concatenate(weights_data, axis=0), training_context['steps']) if len(grads_data) > 0: grads_data = np.concatenate(grads_data, axis=0).mean() else: grads_data = None if module.relative_name in self.first_layer: training_context['grads_state'][ 'first_layer'] = grads_data elif module.relative_name in self.last_layer: training_context['grads_state'][ self.last_layer[ module.relative_name]] = grads_data if len(training_context['grads_state']) > 0: grads_str = yellow_color( '{0:<16s}'.format( training_context['current_model'].name) + '|'.join([ '{0} gradients: {1:<8.3e} '.format(k, v) for k, v in training_context['grads_state']. items() if isinstance(v, numbers.Number) ])) self.lines.append(grads_str + '\n') elif get_backend() == 'tensorflow': if self.frequency > 0 and ( (self.unit == 'batch' and (training_context['current_batch'] + 1) % self.frequency == 0) or (self.unit == 'step' and (training_context['steps'] + 1) % self.frequency == 0) or (self.unit == 'epoch' and training_context['current_batch'] == 0 and (training_context['current_epoch'] + 1) % self.frequency == 0)): grad_dict = OrderedDict() if 'grads_state' not in training_context: training_context['grads_state'] = OrderedDict() if training_context['current_batch'] == 0 and training_context[ 'current_epoch'] > 0: # relocate the first/ last layers self.first_layer = OrderedDict() self.last_layer = OrderedDict() if len(self.first_layer) == 0 and len(self.last_layer) == 0: if training_context['current_model'][ -1].__class__.__name__ == 'ModuleDict': self.is_modulefict = True for k, v in training_context['current_model'][ -1].items(): last_layer_name = '' for name, module in v.named_modules(): if len([ pk for pk, pv in module._parameters.items( ) if 'bias' not in pk and pv is not None and pv.trainable ]) > 0: last_layer_name = module.relative_name if last_layer_name != '': self.last_layer[last_layer_name] = k first_layer_name = '' last_layer_name = '' for k, v in training_context[ 'current_model'].named_modules(): if len([ pk for pk, pv in v._parameters.items() if 'bias' not in pk and pv is not None and pv.trainable ]) > 0: if first_layer_name == '': first_layer_name = v.relative_name self.first_layer[ first_layer_name] = 'first_layer' if not self.is_modulefict: last_layer_name = v.relative_name if last_layer_name != '' and not self.is_modulefict: self.last_layer[last_layer_name] = 'last_layer' grads_and_vars = list(training_context['grads_and_vars']) grads_dict = OrderedDict() for grad, var in grads_and_vars: grads_dict[var.ref()] = to_numpy(grad) for name, module in training_context[ 'current_model'].named_modules(): if module.relative_name in self.first_layer or module.relative_name in self.last_layer: grads_data = [ np.abs(np.reshape(grads_dict[pv.ref()], -1)) for pk, pv in module._parameters.items() if 'bias' not in pk and pv.trainable and grads_dict[pv.ref()] is not None ] weights_data = [ np.abs(np.reshape(to_numpy(pv.value()), -1)) for pk, pv in module._parameters.items() if 'bias' not in pk and pv.trainable ] if ctx.enable_tensorboard and ctx.summary_writer is not None: ctx.summary_writer.add_histogram( training_context['training_name'] + '/gradients/' + self.first_layer[module.relative_name] if module.relative_name in self.first_layer else self.last_layer[module.relative_name], np.concatenate(grads_data, axis=0), training_context['steps']) ctx.summary_writer.add_histogram( training_context['training_name'] + '/weights/' + self.first_layer[module.relative_name] if module.relative_name in self.first_layer else self.last_layer[module.relative_name], np.concatenate(weights_data, axis=0), training_context['steps']) if len(grads_data) > 0: grads_data = np.concatenate(grads_data, axis=0).mean() else: grads_data = None if module.relative_name in self.first_layer: training_context['grads_state'][ 'first_layer'] = grads_data elif module.relative_name in self.last_layer: training_context['grads_state'][self.last_layer[ module.relative_name]] = grads_data if len(training_context['grads_state']) > 0: grad_str = yellow_color('{0:<16s}'.format( training_context['current_model'].name) + '|'.join([ '{0} gradients: {1:<8.3e} '.format(k, v) for k, v in training_context['grads_state'].items( ) if isinstance(v, numbers.Number) ])) self.lines.append(grad_str + '\n')