def on_optimization_step_start(self, training_context): if get_backend() == 'pytorch': if self.clip_value is not None: nn.utils.clip_grad_value_( training_context['optimizer'].param_groups['params'], self.clip_value) else: nn.utils.clip_grad_norm_( training_context['optimizer'].param_groups['params'], max_norm=self.max_norm, norm_type=2 if self.norm_type == 'l2' else 1) elif get_backend() == 'tensorflow': # training_context['grads'] pass elif get_backend() == 'cntk': pass
def collect(self, data_name: str, step: int, value: (float,np.ndarray, Tensor)): if data_name not in self: self.regist(data_name) if value is not None: if isinstance(value,(list,tuple)): value=to_tensor(value) elif isinstance(value,np.ndarray): value = to_tensor(value) if get_backend() == 'pytorch': if isinstance(value,numbers.Number): self[data_name].append((step, value)) elif is_tensor(value): value=value.copy().cpu().detach().mean().item() self[data_name].append((step, value)) elif isinstance(value, np.ndarray): value = value.mean()[0] self[data_name].append((step, value)) elif get_backend() == 'tensorflow': with tf.device('/cpu:0'): value = tf.identity(value).numpy().mean() self[data_name].append((step, value)) # if is_tensor(value): # if get_backend()=='pytorch': # value=to_numpy(value.copy().cpu().detach()).mean() # self[data_name].append((step, value)) # elif get_backend()=='tensorflow': # with tf.device('/cpu:0'): # value = to_numpy(tf.identity(value)).mean() # self[data_name].append((step, value)) # # # else: # self[data_name].append((step, value)) if ctx.enable_tensorboard: if self.training_name is None: ctx.summary_writer.add_scalar( self.name+"/"+data_name, value, global_step=step, walltime=time.time()) else: ctx.summary_writer.add_scalar(self.training_name+ "/"+self.name + "/" + data_name, value, global_step=step, walltime=time.time())
def __init__(self, alpha=1, loss_criterion=None, loss_weight=1, save_path=None, **kwargs): super(MixupCallback, self).__init__() self.alpha = alpha if loss_criterion is None: loss_criterion = get_class( 'CrossEntropyLoss', 'trident.optims.pytorch_losses' if get_backend() == 'pytorch' else 'trident.optims.tensorflow_losses') self.loss_criterion = loss_criterion() self.loss_weight = loss_weight if save_path is None: self.save_path = os.path.join(working_directory, 'Results') else: self.save_path = save_path make_dir_if_need(self.save_path)
def _make_recovery_model_include_top(recovery_model:Layer,default_shape=None,input_shape=None, include_top=True, classes=1000, freeze_features=False): size_change=False if default_shape is None: if recovery_model.built: default_shape=tuple(recovery_model._input_shape.dims[1:] if isinstance(recovery_model._input_shape,TensorShape) else recovery_model._input_shape) else: default_shape=(3,224,224) if get_backend() == 'pytorch' else (224,224,3) if input_shape is not None and input_shape !=default_shape: size_change=True dims = list(input_shape) dims.insert(0, None) if isinstance(recovery_model.signature, Signature): recovery_model._input_shape = TensorShape(dims) recovery_model.signature.inputs.value_list[0].shape = TensorShape(dims) recovery_model.signature.inputs.value_list[0].object_type=ObjectType.rgb if freeze_features: recovery_model.trainable=False idx=-1 while (len(recovery_model[idx]._parameters) == 0 or isinstance(recovery_model[idx], Dense))and len(recovery_model[idx].output_shape) >= 2: layer=recovery_model[idx] if layer.output_shape.rank>2: break if len(recovery_model[idx]._parameters) >0: recovery_model[idx].trainable=True idx-=1 if not include_top: while len(recovery_model[-1]._parameters)==0 or isinstance(recovery_model[-1],Dense) and len(recovery_model[-1].output_shape)>=2: layer = recovery_model[-1] if layer.output_shape.rank > 2: break recovery_model.remove_at(-1) recovery_model.class_names = [] elif size_change: new_layers=[] dims = list(input_shape) dims.insert(0, None) shp=TensorShape(dims) while len(recovery_model[-1]._parameters) == 0 or isinstance(recovery_model[-1], Dense) and len(recovery_model[-1].output_shape) >= 2: layer = recovery_model[-1] if layer.output_shape.rank > 2: break new_layer=copy.deepcopy(layer) if isinstance(layer,Dense) : if layer.num_filters==1000 and classes != 1000: new_layer=Dense((classes)) recovery_model.class_names = [] else: num_filters=new_layer.num_filters new_layer=Dense((num_filters)) new_layers.insert(0,new_layer) recovery_model.remove_at(-1) out=recovery_model(to_tensor(shp.get_dummy_tensor())) recovery_model[-1].output_shape=tensor_to_shape(out,need_exclude_batch_axis=True) fc_seq=0 for ly in new_layers: if isinstance(ly, Dense): recovery_model.add_module('fc' if fc_seq==0 else 'fc{0}'.format(fc_seq),ly) fc_seq += 1 else: recovery_model.add_module(ly.name, ly) if isinstance(recovery_model.signature, Signature): recovery_model.output_shape = TensorShape([None, classes]) recovery_model.signature.outputs.value_list[0].shape = TensorShape([None, classes]) recovery_model.signature.outputs.value_list[0].object_type = ObjectType.classification_label else: #include_top=True if classes != 1000: while len(recovery_model[-1]._parameters)==0 or isinstance(recovery_model[-1],Dense) and len(recovery_model[-1].output_shape)>=2: m=recovery_model[-1] if isinstance(m,Dense): recovery_model[-1]=Dense((classes)) recovery_model.add_module('softmax',SoftMax()) break else: recovery_model.remove_at(-1) if isinstance(recovery_model.signature, Signature): recovery_model.output_shape= TensorShape([None,classes]) recovery_model.signature.outputs.value_list[0].shape = TensorShape([None,classes]) recovery_model.signature.outputs.value_list[0].object_type = ObjectType.classification_label recovery_model.class_names = [] return recovery_model
snake2camel, PrintException, unpack_singleton, enforce_singleton, OrderedDict, split_path, sanitize_path, make_dir_if_need, Signature, TensorShape from trident.backend.model import ModelBase, progress_bar from trident.callbacks.visualization_callbacks import * from trident.data.data_provider import * from trident.misc.ipython_utils import * from trident.misc.visualization_utils import tile_rgb_images, loss_metric_curve from trident.backend.tensorspec import TensorSpec, assert_spec_compatibility, ObjectType from trident.loggers.history import HistoryBase _session = get_session() working_directory=_session.working_directory if get_backend() == 'pytorch': import torch import torch.nn as nn from trident.backend.pytorch_backend import * from trident.backend.pytorch_ops import * from trident.optims.pytorch_optimizers import * from trident.layers.pytorch_layers import * elif get_backend() == 'tensorflow': import tensorflow as tf from trident.backend.tensorflow_backend import * from trident.backend.tensorflow_ops import * from trident.optims.tensorflow_optimizers import * from trident.layers.tensorflow_layers import * def _make_recovery_model_include_top(recovery_model:Layer,default_shape=None,input_shape=None, include_top=True, classes=1000, freeze_features=False):
import os import sys import builtins import numbers import time import math import numpy as np from trident import context from trident.backend.common import get_backend,to_list, addindent, get_time_suffix, format_time, get_terminal_size, get_session,get_backend, \ snake2camel, PrintException, unpack_singleton, enforce_singleton, OrderedDict, split_path, sanitize_path,make_dir_if_need,Signature ctx = context._context() _backend = get_backend() working_directory = ctx.working_directory if _backend == 'pytorch': import torch import torch.nn as nn from trident.backend.pytorch_backend import Tensor from trident.backend.pytorch_ops import * elif _backend == 'tensorflow': import tensorflow as tf from trident.backend.tensorflow_backend import Tensor from trident.backend.tensorflow_ops import * class HistoryBase(OrderedDict): def __init__(self, prevent_redundant=True, name='', **kwargs): super().__init__(**kwargs)
def on_loss_calculation_end(self, training_context): """Returns mixed inputs, pairs of targets, and lambda""" model = training_context['current_model'] train_data = training_context['train_data'] x = None y = None x = train_data.value_list[0].copy().detach().to(model.device) # input y = train_data.value_list[1].copy().detach().to(model.device) # label lam = builtins.min( builtins.max(np.random.beta(self.alpha, self.alpha), 0.3), 0.7) batch_size = int_shape(x)[0] index = arange(batch_size) index = cast(shuffle(index), 'long') this_loss = None mixed_x = None if get_backend() == 'pytorch': mixed_x = lam * x + (1 - lam) * x[index, :] pred = model( to_tensor(mixed_x, requires_grad=True, device=model.device)) y_a, y_b = y, y[index] this_loss = lam * self.loss_criterion(pred, y_a.long()) + ( 1 - lam) * self.loss_criterion(pred, y_b.long()) training_context['current_loss'] = training_context[ 'current_loss'] + this_loss * self.loss_weight if training_context['is_collect_data']: training_context['losses'].collect( 'mixup_loss', training_context['steps'], float(to_numpy(this_loss * self.loss_weight))) elif get_backend() == 'tensorflow': with tf.device(get_device()): x1 = tf.gather(x, index, axis=0) y1 = tf.gather(y, index, axis=0) mixed_x = lam * x + (1 - lam) * x1 pred = model(to_tensor(mixed_x, requires_grad=True)) y_a, y_b = y, y1 this_loss = lam * self.loss_criterion( pred, y_a) + (1 - lam) * self.loss_criterion(pred, y_b) training_context['current_loss'] = training_context[ 'current_loss'] + this_loss * self.loss_weight if training_context['is_collect_data']: training_context['losses'].collect( 'mixup_loss', training_context['steps'], float(to_numpy(this_loss * self.loss_weight))) if training_context['current_batch'] == 0: for item in mixed_x: if self.save_path is None and not is_in_colab(): item = Unnormalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225])(to_numpy(item)) item = Unnormalize(0, 255)(item) array2image(item).save( os.path.join(self.save_path, 'mixup_{0}.jpg'.format( get_time_suffix()))) elif self.save_path is not None: item = Unnormalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225])(to_numpy(item)) item = Unnormalize(0, 255)(item) array2image(item).save( os.path.join(self.save_path, 'mixup_{0}.jpg'.format( get_time_suffix()))) mixed_x = None x = None y = None
def on_loss_calculation_end(self, training_context): """Returns mixed inputs, pairs of targets, and lambda""" model = training_context['current_model'] train_data = training_context['train_data'] x = None y = None x = train_data.value_list[0].copy().detach().to(model.device) # input y = train_data.value_list[1].copy().detach().to(model.device) # label lam = builtins.min( builtins.max(np.random.beta(self.alpha, self.alpha), 0.1), 0.4) batch_size = int_shape(x)[0] index = cast(arange(batch_size), 'int64') index = shuffle(index) this_loss = None if get_backend() == 'pytorch': y_a, y_b = y, y[index] bbx1, bby1, bbx2, bby2 = self.rand_bbox(x.shape[3], x.shape[2], lam) x[:, :, bbx1:bbx2, bby1:bby2] = x[index, :, bbx1:bbx2, bby1:bby2] # adjust lambda to exactly match pixel ratio lam = 1 - ((bbx2 - bbx1) * (bby2 - bby1) / (x.shape[3] * x.shape[2])) pred = model(to_tensor(x, requires_grad=True, device=model.device)) this_loss = lam * self.loss_criterion(pred, y_a.long()) + ( 1 - lam) * self.loss_criterion(pred, y_b.long()) training_context['current_loss'] = training_context[ 'current_loss'] + this_loss * self.loss_weight if training_context['is_collect_data']: training_context['losses'].collect( 'cutmix_loss', training_context['steps'], float(to_numpy(this_loss * self.loss_weight))) elif get_backend() == 'tensorflow': with tf.device(get_device()): y1 = tf.gather(y, index, axis=0) x1 = tf.gather(x, index, axis=0) y_a, y_b = y, y1 bbx1, bby1, bbx2, bby2 = self.rand_bbox( x.shape[2], x.shape[1], lam) filter = np.zeros(int_shape(x)) filter[:, bbx1:bbx2, bby1:bby2, :] = 1 filter = to_tensor(x) x = x * (1 - filter) + x1 * filter # x[:, bbx1:bbx2, bby1:bby2, :] = x1[:, bbx1:bbx2, bby1:bby2,:] # adjust lambda to exactly match pixel ratio lam = 1 - ((bbx2 - bbx1) * (bby2 - bby1) / (x.shape[2] * x.shape[1])) pred = model(to_tensor(x, requires_grad=True)) loss1 = self.loss_criterion(pred, y_a) loss2 = self.loss_criterion(pred, y_b) this_loss = lam * loss1 + (1 - lam) * loss2 training_context['current_loss'] = training_context[ 'current_loss'] + this_loss * self.loss_weight if training_context['is_collect_data']: training_context['losses'].collect( 'cutmix_loss', training_context['steps'], float(to_numpy(this_loss * self.loss_weight))) if training_context['current_batch'] == 0: if self.save_path is None and not is_in_colab(): for item in x: item = Unnormalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225])(to_numpy(item)) item = Unnormalize(0, 255)(item) array2image(item).save( os.path.join( self.save_path, 'cutmix_{0}.jpg'.format(get_time_suffix()))) elif self.save_path is not None: for item in x: item = Unnormalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225])(to_numpy(item)) item = Unnormalize(0, 255)(item) array2image(item).save( os.path.join( self.save_path, 'cutmix_{0}.jpg'.format(get_time_suffix()))) x = None y = None
def mask_backend_adaptive(mask, label_mapping=None, object_type=None): if get_backend() == 'pytorch': if mask is None: return None elif isinstance(mask, np.ndarray): # binary mask if object_type == ObjectType.binary_mask: if mask.ndim == 2: mask[mask > 0] = 1 return mask.astype(np.int64) elif mask.ndim == 3 and mask.shape[-1] in [1, 2]: if mask.shape[-1] == 2: mask = mask[:, :, 1] elif mask.shape[-1] == 1: mask = mask[:, :, 0] mask[mask > 0] = 1 return mask.astype(np.int64) elif object_type == ObjectType.label_mask or object_type == ObjectType.color_mask: if mask.ndim == 2: return mask.astype(np.int64) if mask.ndim == 3 and mask.shape[-1] > 2: if check_is_onehot(mask): mask = np.argmax(mask, -1).astype(np.int64) return mask return mask.astype(np.int64) elif object_type == ObjectType.alpha_mask: if mask.ndim == 2: mask = mask / 255.0 return mask.astype(np.float32) if mask.ndim == 3: mask = color.rgb2gray(mask.astype(np.float32)) if mask.max() > 1: mask = mask / mask.max() return mask.astype(np.float32) else: return mask elif get_backend() == 'tensorflow': if mask is None: return None elif isinstance(mask, np.ndarray): # binary mask if object_type == ObjectType.binary_mask: if mask.ndim == 2: mask[mask > 0] = 1 return mask.astype(np.int64) elif mask.ndim == 3 and mask.shape[-1] in [1, 2]: if mask.shape[-1] == 2: mask = mask[:, :, 1] elif mask.shape[-1] == 1: mask = mask[:, :, 0] mask[mask > 0] = 1 return mask.astype(np.int64) elif object_type == ObjectType.label_mask or object_type == ObjectType.color_mask: if mask.ndim == 2 and label_mapping is not None and len( label_mapping) > 0: return to_onehot(mask, len(label_mapping)) if mask.ndim == 3 and mask.shape[-1] > 2: return mask return mask elif object_type == ObjectType.alpha_mask: if mask.ndim == 2: mask = mask / 255.0 return mask.astype(np.float32) if mask.ndim == 3: mask = color.rgb2gray(mask.astype(np.float32)) if mask.max() > 1: mask = mask / mask.max() return mask.astype(np.float32) else: return mask
def _make_recovery_model_include_top(recovery_model: Layer, default_shape=None, input_shape=None, include_top=True, classes=1000, freeze_features=True): size_change = False if default_shape is None: if recovery_model.built: default_shape = tuple( recovery_model._input_shape. dims[1:] if isinstance(recovery_model._input_shape, TensorShape ) else recovery_model._input_shape) else: default_shape = (3, 224, 224) if get_backend() == 'pytorch' else (224, 224, 3) if input_shape is not None and input_shape != default_shape: size_change = True if freeze_features: recovery_model.trainable = False idx = -1 is_last_dense = True while (len(recovery_model[idx]._parameters) == 0 or isinstance(recovery_model[idx], Dense)) and len( recovery_model[idx].output_shape) >= 2: layer = recovery_model[idx] if layer.output_shape.rank > 2: break elif len(recovery_model[idx]._parameters) > 0: if not include_top: recovery_model.remove_at(idx) idx += 1 elif size_change or (is_last_dense and classes != 1000 and isinstance(recovery_model[idx], Dense)): if hasattr( recovery_model[idx], 'num_filters' ) and recovery_model[idx].num_filters != classes: recovery_model[idx].num_filters = classes recovery_model[idx]._built = False recovery_model[idx]._parameters.clear() else: recovery_model[idx].trainable = True else: if not include_top: recovery_model.remove_at(idx) idx += 1 idx -= 1 dims = list(default_shape) dims.insert(0, None) new_tensorshape = TensorShape(dims) if size_change: dims = list(input_shape) dims.insert(0, None) new_tensorshape = TensorShape(dims) for module in recovery_model.modules(): module._input_shape = None module._output_shape = None recovery_model.to(get_device()) out = recovery_model( to_tensor(new_tensorshape.get_dummy_tensor(), device=get_device())) if isinstance(recovery_model.signature, Signature): recovery_model.signature.inputs.value_list[0].shape = TensorShape(dims) recovery_model.signature.inputs.value_list[ 0].object_type = ObjectType.rgb recovery_model.to(get_device()) return recovery_model