def on_optimization_step_start(self, training_context):
if get_backend() == 'pytorch':
if self.clip_value is not None:
nn.utils.clip_grad_value_(
training_context['optimizer'].param_groups['params'],
self.clip_value)
else:
nn.utils.clip_grad_norm_(
training_context['optimizer'].param_groups['params'],
max_norm=self.max_norm,
norm_type=2 if self.norm_type == 'l2' else 1)
elif get_backend() == 'tensorflow':
# training_context['grads']
pass
elif get_backend() == 'cntk':
pass
def collect(self, data_name: str, step: int, value: (float,np.ndarray, Tensor)):
if data_name not in self:
self.regist(data_name)
if value is not None:
if isinstance(value,(list,tuple)):
value=to_tensor(value)
elif isinstance(value,np.ndarray):
value = to_tensor(value)
if get_backend() == 'pytorch':
if isinstance(value,numbers.Number):
self[data_name].append((step, value))
elif is_tensor(value):
value=value.copy().cpu().detach().mean().item()
self[data_name].append((step, value))
elif isinstance(value, np.ndarray):
value = value.mean()[0]
self[data_name].append((step, value))
elif get_backend() == 'tensorflow':
with tf.device('/cpu:0'):
value = tf.identity(value).numpy().mean()
self[data_name].append((step, value))
# if is_tensor(value):
# if get_backend()=='pytorch':
# value=to_numpy(value.copy().cpu().detach()).mean()
# self[data_name].append((step, value))
# elif get_backend()=='tensorflow':
# with tf.device('/cpu:0'):
# value = to_numpy(tf.identity(value)).mean()
# self[data_name].append((step, value))
#
#
# else:
# self[data_name].append((step, value))
if ctx.enable_tensorboard:
if self.training_name is None:
ctx.summary_writer.add_scalar( self.name+"/"+data_name, value, global_step=step, walltime=time.time())
else:
ctx.summary_writer.add_scalar(self.training_name+ "/"+self.name + "/" + data_name, value, global_step=step, walltime=time.time())
def __init__(self,
alpha=1,
loss_criterion=None,
loss_weight=1,
save_path=None,
**kwargs):
super(MixupCallback, self).__init__()
self.alpha = alpha
if loss_criterion is None:
loss_criterion = get_class(
'CrossEntropyLoss',
'trident.optims.pytorch_losses' if get_backend() == 'pytorch'
else 'trident.optims.tensorflow_losses')
self.loss_criterion = loss_criterion()
self.loss_weight = loss_weight
if save_path is None:
self.save_path = os.path.join(working_directory, 'Results')
else:
self.save_path = save_path
make_dir_if_need(self.save_path)
def _make_recovery_model_include_top(recovery_model:Layer,default_shape=None,input_shape=None, include_top=True, classes=1000, freeze_features=False):
size_change=False
if default_shape is None:
if recovery_model.built:
default_shape=tuple(recovery_model._input_shape.dims[1:] if isinstance(recovery_model._input_shape,TensorShape) else recovery_model._input_shape)
else:
default_shape=(3,224,224) if get_backend() == 'pytorch' else (224,224,3)
if input_shape is not None and input_shape !=default_shape:
size_change=True
dims = list(input_shape)
dims.insert(0, None)
if isinstance(recovery_model.signature, Signature):
recovery_model._input_shape = TensorShape(dims)
recovery_model.signature.inputs.value_list[0].shape = TensorShape(dims)
recovery_model.signature.inputs.value_list[0].object_type=ObjectType.rgb
if freeze_features:
recovery_model.trainable=False
idx=-1
while (len(recovery_model[idx]._parameters) == 0 or isinstance(recovery_model[idx], Dense))and len(recovery_model[idx].output_shape) >= 2:
layer=recovery_model[idx]
if layer.output_shape.rank>2:
break
if len(recovery_model[idx]._parameters) >0:
recovery_model[idx].trainable=True
idx-=1
if not include_top:
while len(recovery_model[-1]._parameters)==0 or isinstance(recovery_model[-1],Dense) and len(recovery_model[-1].output_shape)>=2:
layer = recovery_model[-1]
if layer.output_shape.rank > 2:
break
recovery_model.remove_at(-1)
recovery_model.class_names = []
elif size_change:
new_layers=[]
dims = list(input_shape)
dims.insert(0, None)
shp=TensorShape(dims)
while len(recovery_model[-1]._parameters) == 0 or isinstance(recovery_model[-1], Dense) and len(recovery_model[-1].output_shape) >= 2:
layer = recovery_model[-1]
if layer.output_shape.rank > 2:
break
new_layer=copy.deepcopy(layer)
if isinstance(layer,Dense) :
if layer.num_filters==1000 and classes != 1000:
new_layer=Dense((classes))
recovery_model.class_names = []
else:
num_filters=new_layer.num_filters
new_layer=Dense((num_filters))
new_layers.insert(0,new_layer)
recovery_model.remove_at(-1)
out=recovery_model(to_tensor(shp.get_dummy_tensor()))
recovery_model[-1].output_shape=tensor_to_shape(out,need_exclude_batch_axis=True)
fc_seq=0
for ly in new_layers:
if isinstance(ly, Dense):
recovery_model.add_module('fc' if fc_seq==0 else 'fc{0}'.format(fc_seq),ly)
fc_seq += 1
else:
recovery_model.add_module(ly.name, ly)
if isinstance(recovery_model.signature, Signature):
recovery_model.output_shape = TensorShape([None, classes])
recovery_model.signature.outputs.value_list[0].shape = TensorShape([None, classes])
recovery_model.signature.outputs.value_list[0].object_type = ObjectType.classification_label
else:
#include_top=True
if classes != 1000:
while len(recovery_model[-1]._parameters)==0 or isinstance(recovery_model[-1],Dense) and len(recovery_model[-1].output_shape)>=2:
m=recovery_model[-1]
if isinstance(m,Dense):
recovery_model[-1]=Dense((classes))
recovery_model.add_module('softmax',SoftMax())
break
else:
recovery_model.remove_at(-1)
if isinstance(recovery_model.signature, Signature):
recovery_model.output_shape= TensorShape([None,classes])
recovery_model.signature.outputs.value_list[0].shape = TensorShape([None,classes])
recovery_model.signature.outputs.value_list[0].object_type = ObjectType.classification_label
recovery_model.class_names = []
return recovery_model
snake2camel, PrintException, unpack_singleton, enforce_singleton, OrderedDict, split_path, sanitize_path, make_dir_if_need, Signature, TensorShape
from trident.backend.model import ModelBase, progress_bar
from trident.callbacks.visualization_callbacks import *
from trident.data.data_provider import *
from trident.misc.ipython_utils import *
from trident.misc.visualization_utils import tile_rgb_images, loss_metric_curve
from trident.backend.tensorspec import TensorSpec, assert_spec_compatibility, ObjectType
from trident.loggers.history import HistoryBase
_session = get_session()
working_directory=_session.working_directory
if get_backend() == 'pytorch':
import torch
import torch.nn as nn
from trident.backend.pytorch_backend import *
from trident.backend.pytorch_ops import *
from trident.optims.pytorch_optimizers import *
from trident.layers.pytorch_layers import *
elif get_backend() == 'tensorflow':
import tensorflow as tf
from trident.backend.tensorflow_backend import *
from trident.backend.tensorflow_ops import *
from trident.optims.tensorflow_optimizers import *
from trident.layers.tensorflow_layers import *
def _make_recovery_model_include_top(recovery_model:Layer,default_shape=None,input_shape=None, include_top=True, classes=1000, freeze_features=False):
import os
import sys
import builtins
import numbers
import time
import math
import numpy as np
from trident import context
from trident.backend.common import get_backend,to_list, addindent, get_time_suffix, format_time, get_terminal_size, get_session,get_backend, \
snake2camel, PrintException, unpack_singleton, enforce_singleton, OrderedDict, split_path, sanitize_path,make_dir_if_need,Signature
ctx = context._context()
_backend = get_backend()
working_directory = ctx.working_directory
if _backend == 'pytorch':
import torch
import torch.nn as nn
from trident.backend.pytorch_backend import Tensor
from trident.backend.pytorch_ops import *
elif _backend == 'tensorflow':
import tensorflow as tf
from trident.backend.tensorflow_backend import Tensor
from trident.backend.tensorflow_ops import *
class HistoryBase(OrderedDict):
def __init__(self, prevent_redundant=True, name='', **kwargs):
super().__init__(**kwargs)
def on_loss_calculation_end(self, training_context):
"""Returns mixed inputs, pairs of targets, and lambda"""
model = training_context['current_model']
train_data = training_context['train_data']
x = None
y = None
x = train_data.value_list[0].copy().detach().to(model.device) # input
y = train_data.value_list[1].copy().detach().to(model.device) # label
lam = builtins.min(
builtins.max(np.random.beta(self.alpha, self.alpha), 0.3), 0.7)
batch_size = int_shape(x)[0]
index = arange(batch_size)
index = cast(shuffle(index), 'long')
this_loss = None
mixed_x = None
if get_backend() == 'pytorch':
mixed_x = lam * x + (1 - lam) * x[index, :]
pred = model(
to_tensor(mixed_x, requires_grad=True, device=model.device))
y_a, y_b = y, y[index]
this_loss = lam * self.loss_criterion(pred, y_a.long()) + (
1 - lam) * self.loss_criterion(pred, y_b.long())
training_context['current_loss'] = training_context[
'current_loss'] + this_loss * self.loss_weight
if training_context['is_collect_data']:
training_context['losses'].collect(
'mixup_loss', training_context['steps'],
float(to_numpy(this_loss * self.loss_weight)))
elif get_backend() == 'tensorflow':
with tf.device(get_device()):
x1 = tf.gather(x, index, axis=0)
y1 = tf.gather(y, index, axis=0)
mixed_x = lam * x + (1 - lam) * x1
pred = model(to_tensor(mixed_x, requires_grad=True))
y_a, y_b = y, y1
this_loss = lam * self.loss_criterion(
pred, y_a) + (1 - lam) * self.loss_criterion(pred, y_b)
training_context['current_loss'] = training_context[
'current_loss'] + this_loss * self.loss_weight
if training_context['is_collect_data']:
training_context['losses'].collect(
'mixup_loss', training_context['steps'],
float(to_numpy(this_loss * self.loss_weight)))
if training_context['current_batch'] == 0:
for item in mixed_x:
if self.save_path is None and not is_in_colab():
item = Unnormalize([0.485, 0.456, 0.406],
[0.229, 0.224, 0.225])(to_numpy(item))
item = Unnormalize(0, 255)(item)
array2image(item).save(
os.path.join(self.save_path, 'mixup_{0}.jpg'.format(
get_time_suffix())))
elif self.save_path is not None:
item = Unnormalize([0.485, 0.456, 0.406],
[0.229, 0.224, 0.225])(to_numpy(item))
item = Unnormalize(0, 255)(item)
array2image(item).save(
os.path.join(self.save_path, 'mixup_{0}.jpg'.format(
get_time_suffix())))
mixed_x = None
x = None
y = None
def on_loss_calculation_end(self, training_context):
"""Returns mixed inputs, pairs of targets, and lambda"""
model = training_context['current_model']
train_data = training_context['train_data']
x = None
y = None
x = train_data.value_list[0].copy().detach().to(model.device) # input
y = train_data.value_list[1].copy().detach().to(model.device) # label
lam = builtins.min(
builtins.max(np.random.beta(self.alpha, self.alpha), 0.1), 0.4)
batch_size = int_shape(x)[0]
index = cast(arange(batch_size), 'int64')
index = shuffle(index)
this_loss = None
if get_backend() == 'pytorch':
y_a, y_b = y, y[index]
bbx1, bby1, bbx2, bby2 = self.rand_bbox(x.shape[3], x.shape[2],
lam)
x[:, :, bbx1:bbx2, bby1:bby2] = x[index, :, bbx1:bbx2, bby1:bby2]
# adjust lambda to exactly match pixel ratio
lam = 1 - ((bbx2 - bbx1) * (bby2 - bby1) /
(x.shape[3] * x.shape[2]))
pred = model(to_tensor(x, requires_grad=True, device=model.device))
this_loss = lam * self.loss_criterion(pred, y_a.long()) + (
1 - lam) * self.loss_criterion(pred, y_b.long())
training_context['current_loss'] = training_context[
'current_loss'] + this_loss * self.loss_weight
if training_context['is_collect_data']:
training_context['losses'].collect(
'cutmix_loss', training_context['steps'],
float(to_numpy(this_loss * self.loss_weight)))
elif get_backend() == 'tensorflow':
with tf.device(get_device()):
y1 = tf.gather(y, index, axis=0)
x1 = tf.gather(x, index, axis=0)
y_a, y_b = y, y1
bbx1, bby1, bbx2, bby2 = self.rand_bbox(
x.shape[2], x.shape[1], lam)
filter = np.zeros(int_shape(x))
filter[:, bbx1:bbx2, bby1:bby2, :] = 1
filter = to_tensor(x)
x = x * (1 - filter) + x1 * filter
# x[:, bbx1:bbx2, bby1:bby2, :] = x1[:, bbx1:bbx2, bby1:bby2,:]
# adjust lambda to exactly match pixel ratio
lam = 1 - ((bbx2 - bbx1) * (bby2 - bby1) /
(x.shape[2] * x.shape[1]))
pred = model(to_tensor(x, requires_grad=True))
loss1 = self.loss_criterion(pred, y_a)
loss2 = self.loss_criterion(pred, y_b)
this_loss = lam * loss1 + (1 - lam) * loss2
training_context['current_loss'] = training_context[
'current_loss'] + this_loss * self.loss_weight
if training_context['is_collect_data']:
training_context['losses'].collect(
'cutmix_loss', training_context['steps'],
float(to_numpy(this_loss * self.loss_weight)))
if training_context['current_batch'] == 0:
if self.save_path is None and not is_in_colab():
for item in x:
item = Unnormalize([0.485, 0.456, 0.406],
[0.229, 0.224, 0.225])(to_numpy(item))
item = Unnormalize(0, 255)(item)
array2image(item).save(
os.path.join(
self.save_path,
'cutmix_{0}.jpg'.format(get_time_suffix())))
elif self.save_path is not None:
for item in x:
item = Unnormalize([0.485, 0.456, 0.406],
[0.229, 0.224, 0.225])(to_numpy(item))
item = Unnormalize(0, 255)(item)
array2image(item).save(
os.path.join(
self.save_path,
'cutmix_{0}.jpg'.format(get_time_suffix())))
x = None
y = None
def mask_backend_adaptive(mask, label_mapping=None, object_type=None):
if get_backend() == 'pytorch':
if mask is None:
return None
elif isinstance(mask, np.ndarray):
# binary mask
if object_type == ObjectType.binary_mask:
if mask.ndim == 2:
mask[mask > 0] = 1
return mask.astype(np.int64)
elif mask.ndim == 3 and mask.shape[-1] in [1, 2]:
if mask.shape[-1] == 2:
mask = mask[:, :, 1]
elif mask.shape[-1] == 1:
mask = mask[:, :, 0]
mask[mask > 0] = 1
return mask.astype(np.int64)
elif object_type == ObjectType.label_mask or object_type == ObjectType.color_mask:
if mask.ndim == 2:
return mask.astype(np.int64)
if mask.ndim == 3 and mask.shape[-1] > 2:
if check_is_onehot(mask):
mask = np.argmax(mask, -1).astype(np.int64)
return mask
return mask.astype(np.int64)
elif object_type == ObjectType.alpha_mask:
if mask.ndim == 2:
mask = mask / 255.0
return mask.astype(np.float32)
if mask.ndim == 3:
mask = color.rgb2gray(mask.astype(np.float32))
if mask.max() > 1:
mask = mask / mask.max()
return mask.astype(np.float32)
else:
return mask
elif get_backend() == 'tensorflow':
if mask is None:
return None
elif isinstance(mask, np.ndarray):
# binary mask
if object_type == ObjectType.binary_mask:
if mask.ndim == 2:
mask[mask > 0] = 1
return mask.astype(np.int64)
elif mask.ndim == 3 and mask.shape[-1] in [1, 2]:
if mask.shape[-1] == 2:
mask = mask[:, :, 1]
elif mask.shape[-1] == 1:
mask = mask[:, :, 0]
mask[mask > 0] = 1
return mask.astype(np.int64)
elif object_type == ObjectType.label_mask or object_type == ObjectType.color_mask:
if mask.ndim == 2 and label_mapping is not None and len(
label_mapping) > 0:
return to_onehot(mask, len(label_mapping))
if mask.ndim == 3 and mask.shape[-1] > 2:
return mask
return mask
elif object_type == ObjectType.alpha_mask:
if mask.ndim == 2:
mask = mask / 255.0
return mask.astype(np.float32)
if mask.ndim == 3:
mask = color.rgb2gray(mask.astype(np.float32))
if mask.max() > 1:
mask = mask / mask.max()
return mask.astype(np.float32)
else:
return mask
def _make_recovery_model_include_top(recovery_model: Layer,
default_shape=None,
input_shape=None,
include_top=True,
classes=1000,
freeze_features=True):
size_change = False
if default_shape is None:
if recovery_model.built:
default_shape = tuple(
recovery_model._input_shape.
dims[1:] if isinstance(recovery_model._input_shape, TensorShape
) else recovery_model._input_shape)
else:
default_shape = (3, 224,
224) if get_backend() == 'pytorch' else (224, 224,
3)
if input_shape is not None and input_shape != default_shape:
size_change = True
if freeze_features:
recovery_model.trainable = False
idx = -1
is_last_dense = True
while (len(recovery_model[idx]._parameters) == 0
or isinstance(recovery_model[idx], Dense)) and len(
recovery_model[idx].output_shape) >= 2:
layer = recovery_model[idx]
if layer.output_shape.rank > 2:
break
elif len(recovery_model[idx]._parameters) > 0:
if not include_top:
recovery_model.remove_at(idx)
idx += 1
elif size_change or (is_last_dense and classes != 1000 and
isinstance(recovery_model[idx], Dense)):
if hasattr(
recovery_model[idx], 'num_filters'
) and recovery_model[idx].num_filters != classes:
recovery_model[idx].num_filters = classes
recovery_model[idx]._built = False
recovery_model[idx]._parameters.clear()
else:
recovery_model[idx].trainable = True
else:
if not include_top:
recovery_model.remove_at(idx)
idx += 1
idx -= 1
dims = list(default_shape)
dims.insert(0, None)
new_tensorshape = TensorShape(dims)
if size_change:
dims = list(input_shape)
dims.insert(0, None)
new_tensorshape = TensorShape(dims)
for module in recovery_model.modules():
module._input_shape = None
module._output_shape = None
recovery_model.to(get_device())
out = recovery_model(
to_tensor(new_tensorshape.get_dummy_tensor(), device=get_device()))
if isinstance(recovery_model.signature, Signature):
recovery_model.signature.inputs.value_list[0].shape = TensorShape(dims)
recovery_model.signature.inputs.value_list[
0].object_type = ObjectType.rgb
recovery_model.to(get_device())
return recovery_model