def on_loss_calculation_end(self, training_context):
"""Returns mixed inputs, pairs of targets, and lambda"""
train_data = training_context['train_data']
x = None
y = None
x = train_data.value_list[0].copy().detach() # input
y = train_data.value_list[1].copy().detach() # label
model = training_context['current_model']
lam = builtins.min(
builtins.max(np.random.beta(self.alpha, self.alpha), 0.3), 0.7)
batch_size = int_shape(x)[0]
index = arange(batch_size)
index = cast(shuffle(index), 'long')
this_loss = None
mixed_x = None
if get_backend() == 'pytorch':
mixed_x = lam * x + (1 - lam) * x[index, :]
pred = model(to_tensor(mixed_x, requires_grad=True))
y_a, y_b = y, y[index]
this_loss = lam * self.loss_criterion(pred, y_a.long()) + (
1 - lam) * self.loss_criterion(pred, y_b.long())
elif get_backend() == 'tensorflow':
x1 = tf.gather(x, index, axis=0)
y1 = tf.gather(y, index, axis=0)
mixed_x = lam * x + (1 - lam) * x1
pred = model(to_tensor(mixed_x, requires_grad=True))
y_a, y_b = y, y1
this_loss = lam * self.loss_criterion(
pred, y_a) + (1 - lam) * self.loss_criterion(pred, y_b)
training_context['current_loss'] = training_context[
'current_loss'] + this_loss * self.loss_weight
if training_context['is_collect_data']:
training_context['losses'].collect(
'mixup_loss', training_context['steps'],
float(to_numpy(this_loss * self.loss_weight)))
if training_context['current_batch'] == 0:
for item in mixed_x:
if self.save_path is None and not is_in_colab():
item = unnormalize([0.485, 0.456, 0.406],
[0.229, 0.224, 0.225])(to_numpy(item))
item = unnormalize(0, 255)(item)
array2image(item).save('Results/mixup_{0}.jpg'.format(
get_time_suffix()))
elif self.save_path is not None:
item = unnormalize([0.485, 0.456, 0.406],
[0.229, 0.224, 0.225])(to_numpy(item))
item = unnormalize(0, 255)(item)
array2image(item).save(
os.path.join(self.save_path, 'mixup_{0}.jpg'.format(
get_time_suffix())))
def plot_detection_image(self, training_context):
tile_images_list = []
input = None
target = None
output = None
data_feed = training_context['data_feed']
data = training_context['train_data']
model = training_context['current_model']
output = try_map_args_and_call(model, data, data_feed)
target = data['bbox']
input = data[data_feed[model.signature.inputs.key_list[0]]]
input_image = self.reverse_image_transform(to_numpy(input))
targetmask = (target[:, 4] > 0.9)
input_image1 = input_image.copy()
target_boxes = to_numpy(xywh2xyxy(target[targetmask, :]))
for box in target_boxes:
plot_one_box(box, input_image1, (255, 128, 128),
self.labels[box[5:]])
# input_arr=np.asarray(input_arr)
tile_images_list.append(input_image1)
input_image = self.reverse_image_transform(to_numpy(input))
mask = (output[:, :, 4] > 0.7)
if len(output[:, :, 4]) > 0:
mask2 = (argmax(softmax(output[:, :, 5:], -1), -1) != 0)
mask = (mask.float() + mask2.float() == 2)
output = output[mask, :]
input_image2 = input_image.copy()
output_boxes = to_numpy(xywh2xyxy(output[mask, :]))
for box in output_boxes:
plot_one_box(box, input_image2, (255, 255, 128),
self.labels[np.argmax(box[5:])])
tile_images_list.append(input_image2)
fig = tile_rgb_images(*tile_images_list,
save_path=os.path.join(
self.save_path, self.tile_image_name_prefix),
imshow=True)
if ctx.enable_tensorboard and ctx.summary_writer is not None:
ctx.summary_writer.add_figure(
training_context['training_name'] + '/plot/detection_plot',
fig,
global_step=training_context['steps'],
close=True,
walltime=time.time())
plt.close()
def on_loss_calculation_end(self, training_context):
"""Returns mixed inputs, pairs of targets, and lambda"""
train_data = training_context['train_data']
x = None
y = None
if get_backend() == 'pytorch':
x = train_data.value_list[0].clone() # input
y = train_data.value_list[1].clone() # label
elif get_backend() == 'tensorflow':
x = copy.deepcopy(train_data.value_list[0]) # input
y = copy.deepcopy(train_data.value_list[1]) # label
model = training_context['current_model']
if self.alpha > 0:
lam = np.random.beta(self.alpha, self.alpha)
else:
lam = 1
batch_size = int_shape(x)[0]
index = cast(arange(batch_size),'int64')
index=shuffle(index)
this_loss=None
if get_backend()=='pytorch':
y_a, y_b = y, y[index]
bbx1, bby1, bbx2, bby2 = self.rand_bbox(x.shape[3], x.shape[2], lam)
x[:, :, bbx1:bbx2, bby1:bby2] = x[index, :, bbx1:bbx2, bby1:bby2]
# adjust lambda to exactly match pixel ratio
lam = 1 - ((bbx2 - bbx1) * (bby2 - bby1) / (x.shape[3] * x.shape[2]))
pred = model(to_tensor(x, requires_grad=True))
this_loss = lam * self.loss_criterion(pred, y_a.long()) + (1 - lam) * self.loss_criterion(pred, y_b.long())
elif get_backend() == 'tensorflow':
y1 = tf.gather(y,index,axis=0)
x1= tf.gather(x,index,axis=0)
y_a, y_b = y, y1
bbx1, bby1, bbx2, bby2 = self.rand_bbox(x.shape[2], x.shape[1], lam)
filter=np.zeros(int_shape(x))
filter[:, bbx1:bbx2, bby1:bby2, :] =1
filter=to_tensor(x)
x=x*(1-filter)+x1*filter
#x[:, bbx1:bbx2, bby1:bby2, :] = x1[:, bbx1:bbx2, bby1:bby2,:]
# adjust lambda to exactly match pixel ratio
lam = 1 - ((bbx2 - bbx1) * (bby2 - bby1) / (x.shape[2] * x.shape[1]))
pred = model(to_tensor(x, requires_grad=True))
loss1=self.loss_criterion(pred, y_a)
loss2=self.loss_criterion(pred, y_b)
this_loss = lam *loss1 + (1 - lam) * loss2
if training_context['current_batch'] == 0:
for item in x:
item = unnormalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225])(to_numpy(item))
item = unnormalize(0, 255)(item)
array2image(item).save('Results/cutmix_{0}.jpg'.format(get_time_suffix()))
training_context['current_loss'] = training_context['current_loss'] + this_loss *self.loss_weight
if training_context['is_collect_data']:
training_context['losses'].collect('cutmix_loss', training_context['steps'], float(to_numpy(this_loss * self.loss_weight)))
def on_batch_end(self, training_context):
if self.unit_base is None:
if training_context['total_batch'] > _session.epoch_equivalent:
self.unit_base = 'batch'
print(
'one epoch have {0} batches, use {1} as epoch equivalent in long epoch. '
.format(training_context['total_batch'],
_session.epoch_equivalent))
else:
self.unit_base = 'epoch'
print('ReduceLROnPlateau reseted.')
num_batches = training_context['steps']
if self.unit_base == 'batch' and training_context[
'steps'] > 0 and training_context[
'steps'] % _session.epoch_equivalent == 0:
training_context['current_lr'] = training_context['optimizer'].lr
history = training_context['losses'].get(
self.monitor, training_context['metrics'].get(
self.monitor, training_context['losses']['total_losses']))
steps, values = zip(*history)
current = to_numpy(values[-min(5, len(values)):]).mean()
if current is None:
warnings.warn(
'Reduce LR on plateau conditioned on metric `%s` '
'which is not available. Available metrics are: %s' %
(self.monitor, ','.join(
training_context['metrics'].keys_list)),
RuntimeWarning)
else:
if self.in_cooldown():
self.cooldown_counter -= 1
self.wait = 0
if self.monitor_op(current, self.best):
self.best = current
self.wait = 0
elif not self.in_cooldown():
self.wait += 1
if self.wait >= self.patience:
old_lr = float(training_context['optimizer'].lr)
if old_lr > self.min_lr:
new_lr = old_lr * self.factor
new_lr = max(new_lr, self.min_lr)
training_context['optimizer'].adjust_learning_rate(
new_lr, True)
if self.verbose > 0:
print(
'\nEpoch %05d: ReduceLROnPlateau reducing '
'learning rate to %s.' %
(training_context['current_epoch'],
new_lr))
self.cooldown_counter = self.cooldown
self.wait = 0
def plot_tile_image(self, training_context):
output = None
legend = []
data_feed = training_context['data_feed']
data = training_context['train_data']
model = training_context['current_model'].eval()
dataprovider = enforce_singleton(ctx.get_data_provider())
output = to_numpy(model(data[
data_feed['input']])) if data_feed['input'] in data else to_numpy(
data['output'])
model.train()
reverse_image_transform = dataprovider.reverse_image_transform
if reverse_image_transform is not None:
for i in range(len(output)):
self.output_arr.append(reverse_image_transform(output[i]))
if len(self.output_arr) == self.row:
self.tile_images_list.append(self.output_arr)
if len(self.tile_images_list) == self.row:
self.sample_enough = True
break
self.output_arr = []
legend.append('output')
if self.sample_enough:
fig = tile_rgb_images(*self.tile_images_list,
row=self.row,
save_path=os.path.join(
self.save_path,
self.tile_image_name_prefix),
imshow=True,
legend=None)
if ctx.enable_tensorboard and ctx.summary_writer is not None:
ctx.summary_writer.add_figure(
training_context['training_name'] + '/plot/tile_image',
fig,
global_step=training_context['steps'],
close=True,
walltime=time.time())
plt.close()
def on_epoch_end(self, training_context):
training_context['current_lr'] = training_context['optimizer'].lr
if self.unit_base == 'epoch':
steps = None
history = None
if self.monitor in training_context['losses']:
steps, history = training_context['losses'].get_series(
self.monitor)
elif self.monitor in training_context['metrics']:
steps, history = training_context['metrics'].get_series(
self.monitor)
else:
steps, history = training_context['losses'].get_series(
'total_losses')
current = to_numpy(history[-min(5, len(history)):]).mean()
if current is None:
warnings.warn(
'Reduce LR on plateau conditioned on metric `%s` '
'which is not available. Available metrics are: %s' %
(self.monitor, ','.join(
training_context['metrics'].keys_list)),
RuntimeWarning)
else:
if self.in_cooldown():
self.cooldown_counter -= 1
self.wait = 0
if self.monitor_op(current, self.best):
self.best = current
self.wait = 0
elif not self.in_cooldown():
self.wait += 1
if self.wait >= self.patience:
old_lr = float(training_context['optimizer'].lr)
if old_lr > self.min_lr:
new_lr = old_lr * self.factor
new_lr = max(new_lr, self.min_lr)
training_context['optimizer'].adjust_learning_rate(
new_lr, True)
if self.verbose > 0:
print(
'\nEpoch %05d: ReduceLROnPlateau reducing '
'learning rate to %s.' %
(training_context['current_epoch'] + 1,
new_lr))
self.cooldown_counter = self.cooldown
self.wait = 0
def __init__(self,
epoch_inteval=-1,
batch_inteval=-1,
save_path: str = 'results',
reverse_image_transform=None,
palette=None,
background=(120, 120, 120),
name_prefix: str = 'segtile_image_{0}.png',
imshow=False):
super(SegTileImageCallback,
self).__init__(epoch_inteval, batch_inteval, save_path, imshow)
self.is_in_ipython = is_in_ipython()
self.is_in_colab = is_in_colab()
self.palette = palette
self.tile_image_name_prefix = name_prefix
self.reverse_image_transform = reverse_image_transform
self.background = np.expand_dims(
np.expand_dims(to_numpy(background), 0), 0)
def __init__(self,
frequency=-1,
unit='batch',
save_path: str = 'results',
reverse_image_transform=None,
labels=None,
palette=None,
background=(120, 120, 120),
name_prefix: str = 'detection_plot_image_{0}.png',
imshow=False):
super(DetectionPlotImageCallback,
self).__init__(frequency, unit, save_path, imshow)
self.is_in_ipython = is_in_ipython()
self.is_in_colab = is_in_colab()
self.labels = labels
self.palette = palette
self.tile_image_name_prefix = name_prefix
self.reverse_image_transform = reverse_image_transform
self.background = np.expand_dims(
np.expand_dims(to_numpy(background), 0), 0)
def __init__(self,
frequency=-1,
unit='batch',
save_path: str = 'results',
reverse_image_transform=None,
is_label_mask=False,
palette=None,
background=(120, 120, 120),
name_prefix: str = 'segtile_image_{0}.png',
imshow=False):
super(SegTileImageCallback, self).__init__(frequency, unit, save_path,
imshow)
self.is_in_ipython = is_in_ipython()
self.is_in_colab = is_in_colab()
self.is_label_mask = is_label_mask
self.palette = palette
self.tile_image_name_prefix = name_prefix
self.reverse_image_transform = reverse_image_transform
self.background = to_numpy(background)
def plot_tile_image(self, training_context):
tile_images_list = []
input = None
target = None
output = None
data_feed = training_context['data_feed']
data = training_context['train_data']
model = training_context['current_model']
if len(data_feed) == 3 and len(data) <= 3:
input = data[data_feed.value_list[0]]
output = data[data_feed.value_list[1]]
target = data[data_feed.value_list[2]]
#loss function signature loss(output,target)
else:
for data_key in data.key_list:
if self.include_input and data_key == model.signature.inputs.key_list[
0]:
input = data[data_key]
elif self.include_output and data_key == model.signature.outputs.key_list[
0]:
output = data[data_key]
elif self.include_input and model.signature.inputs.key_list[
0] in data_feed and data_feed[
model.signature.inputs.key_list[0]] == data_key:
input = data[data_key]
elif self.include_output and model.signature.outputs.key_list[
0] in data_feed and data_feed[
model.signature.outputs.key_list[0]] == data_key:
output = data[data_key]
elif self.include_target and data_key == 'target':
output = data['target']
elif 'output' in data_key or 'pred' in data_key:
output = data[data_key]
elif ('target' in data_key or 'label' in data_key
or 'mask' in data_key):
target = data[data_key]
if output is None and 'tensor' in model.__class__.__name__.lower():
output = model.clone()
elif output is None and input is not None:
output = model(input)
if self.include_input and input is not None:
if self.reverse_image_transform is not None:
input_arr = []
for i in range(len(input)):
input_arr.append(
self.reverse_image_transform(to_numpy(input[i])))
tile_images_list.append(input_arr)
else:
input_arr = to_numpy(input).transpose([
0, 2, 3, 1
]) if get_backend() != 'tensorflow' else to_numpy(input)
tile_images_list.append(input_arr * 127.5 + 127.5)
if self.include_target and target is not None:
if self.reverse_image_transform is not None:
target_arr = []
for i in range(len(target)):
target_arr.append(
self.reverse_image_transform(to_numpy(target[i])))
tile_images_list.append(target_arr)
else:
target_arr = to_numpy(target).transpose([
0, 2, 3, 1
]) if get_backend() != 'tensorflow' else to_numpy(target)
tile_images_list.append(target_arr * 127.5 + 127.5)
if self.include_output and output is not None:
if self.reverse_image_transform is not None:
output_arr = []
for i in range(len(output)):
output_arr.append(
self.reverse_image_transform(to_numpy(output[i])))
tile_images_list.append(output_arr)
else:
output_arr = to_numpy(output).transpose([
0, 2, 3, 1
]) if get_backend() != 'tensorflow' else to_numpy(output)
tile_images_list.append(output_arr * 127.5 + 127.5)
# if self.tile_image_include_mask:
# tile_images_list.append(input*127.5+127.5)
tile_rgb_images(*tile_images_list,
row=self.row,
save_path=os.path.join(self.save_path,
self.tile_image_name_prefix),
imshow=True)
def on_optimization_step_start(self, training_context):
if get_backend() == 'pytorch':
if (training_context['current_epoch'] *
training_context['total_batch'] +
training_context['current_batch']
) % self.batch_inteval == 0:
grad_dict = {}
if 'grads_state' not in training_context:
training_context['grads_state'] = OrderedDict()
training_context['grads_state']['first_layer'] = []
training_context['grads_state']['last_layer'] = []
if training_context['current_batch'] == 0 and training_context[
'current_epoch'] > 0:
#relocate the first/ last layers
self.first_layer = ''
self.last_layer = ''
if self.first_layer != '' and self.last_layer != '':
for i, (k,
v) in enumerate(training_context['current_model'].
named_parameters()):
if v is not None and v.requires_grad == True:
if k == self.first_layer:
training_context['grads_state'][
'first_layer'].append(
np.abs(
to_numpy(0 if v.grad is None else v
.grad)).mean())
elif k == self.last_layer:
training_context['grads_state'][
'last_layer'].append(
np.abs(
to_numpy(0 if v.grad is None else v
.grad)).mean())
else:
for i, (k,
v) in enumerate(training_context['current_model'].
named_parameters()):
if v is not None and v.requires_grad == True:
grad_dict[k] = np.abs(
to_numpy(0 if v.grad is None else v.grad))
if grad_dict[k].ndim > 1:
if self.first_layer == '':
self.first_layer = k
self.last_layer = k
if self.first_layer != '' and self.first_layer in grad_dict:
training_context['grads_state']['first_layer'].append(
grad_dict[self.first_layer].mean())
if self.last_layer != '' and self.last_layer in grad_dict:
training_context['grads_state']['last_layer'].append(
grad_dict[self.last_layer].mean())
if len(training_context['grads_state']
['first_layer']) > 0 and len(
training_context['grads_state']['last_layer']) > 0:
self.lines.append(
'{0:<16s} first_layer gradients: {1:<8.3e}| last_layer gradients: {2:<8.3e}'
.format(
training_context['current_model'].name,
training_context['grads_state']['first_layer'][-1],
training_context['grads_state']['last_layer'][-1]))
elif get_backend() == 'tensorflow':
if (training_context['current_epoch'] *
training_context['total_batch'] +
training_context['current_batch']
) % self.batch_inteval == 0:
if 'grads_state' not in training_context:
training_context['grads_state'] = OrderedDict()
training_context['grads_state']['first_layer'] = []
training_context['grads_state']['last_layer'] = []
grads_and_vars = list(
training_context['optimizer'].grads_and_vars)
grads_and_vars = [
gv for gv in grads_and_vars
if gv[1].trainable and reduce_any(not_equal(gv[0], 0))
]
training_context['grads_state']['first_layer'].append(
np.abs(to_numpy(grads_and_vars[0][0])).mean())
training_context['grads_state']['last_layer'].append(
np.abs(to_numpy(grads_and_vars[-1][0])).mean())
if len(training_context['grads_state']
['first_layer']) > 0 and len(
training_context['grads_state']['last_layer']) > 0:
self.lines.append(
'{0:<16s} first_layer gradients: {1:<8.3e}| last_layer gradients: {2:<8.3e}'
.format(
training_context['current_model'].name,
training_context['grads_state']['first_layer'][-1],
training_context['grads_state']['last_layer'][-1]))
def plot_tile_image(self, training_context):
tile_images_list = []
input = None
target = None
output = None
is_label_mask = False
data_feed = training_context['data_feed']
data = training_context['train_data']
model = training_context['current_model']
if model.output_shape[0] > 2:
is_label_mask = True
# if len(data) >= 3:
for data_key in data.key_list:
if data_key == data_feed[model.signature.inputs.key_list[0]]:
input = data[data_feed[model.signature.inputs.key_list[0]]]
training_context['current_model'].eval()
output = model(input)
training_context['current_model'].train()
elif (
'target' in data_key or 'label' in data_key
or 'mask' in data_key
) and not 'output' in data_key and data_key in data_feed.value_list:
target = to_numpy(data[data_key])
if 'alpha' not in data:
output = np.argmax(to_numpy(output), 1)
if is_label_mask:
target = label2color(target, self.palette)
output = label2color(output, self.palette)
else:
output = to_numpy(output[:, 1, :, :] * argmax(output, 1))
target = to_numpy(data['alpha'])
input_arr = []
input = to_numpy(input)
for i in range(len(input)):
input_arr.append(self.reverse_image_transform(input[i]))
# input_arr=np.asarray(input_arr)
tile_images_list.append(input_arr)
if is_label_mask:
tile_images_list.append(target)
tile_images_list.append(output)
else:
target_arr = np.expand_dims(target, -1)
output_arr = np.expand_dims(output, -1)
if 'alpha' not in data:
target_arr[target_arr > 0] = 1
background = np.ones_like(target_arr) * self.background
tile_images_list.append(target_arr * input_arr +
(1 - target_arr) * background)
output_arr = np.expand_dims(output, -1)
if 'alpha' not in data:
output_arr[output_arr > 0] = 1
tile_images_list.append(output_arr * input_arr +
(1 - output_arr) * background)
# if self.tile_image_include_mask:
# tile_images_list.append(input*127.5+127.5)
tile_rgb_images(*tile_images_list,
save_path=os.path.join(self.save_path,
self.tile_image_name_prefix),
imshow=True)
def plot_tile_image(self, training_context):
tile_images_list = []
input = None
target = None
output = None
mask = None
legend = []
data_feed = training_context['data_feed']
data = training_context['train_data']
model = training_context['current_model']
dataprovider = enforce_singleton(ctx.get_data_provider())
input = to_numpy(data[data_feed['input']])
if 'tensor' in model.__class__.__name__.lower():
output = to_numpy(model.clone())
elif isinstance(model, Layer):
output = to_numpy(data[data_feed['output']].copy())
if self.include_target:
target = to_numpy(data[data_feed['target']].copy())
if self.include_mask:
if isinstance(dataprovider.traindata.label, MaskDataset):
mask = to_numpy(data[dataprovider.traindata.label.symbol])
elif isinstance(dataprovider.traindata.label, ZipDataset):
for ds in dataprovider.traindata.label._datasets:
if isinstance(ds, MaskDataset):
mask = to_numpy(data[ds.symbol])
reverse_image_transform = dataprovider.reverse_image_transform
reverse_image_transform_funcs = dataprovider.reverse_image_transform_funcs
if self.include_input and input is not None:
if len(reverse_image_transform_funcs) > 1:
input_arr = []
for i in range(len(input)):
input_arr.append(reverse_image_transform(input[i]))
tile_images_list.append(input_arr)
else:
input_arr = to_numpy(input).transpose([
0, 2, 3, 1
]) if get_backend() != 'tensorflow' else to_numpy(input)
max_value = input_arr.max()
min_value = input_arr.min()
if max_value <= 1.1 and min_value >= 0:
tile_images_list.append(input_arr * 255.0)
elif max_value <= 1.1 and min_value >= -1.1:
tile_images_list.append(input_arr * 128 + 127)
else:
print(max_value, min_value)
tile_images_list.append(input_arr)
legend.append('input')
if self.include_target and target is not None:
if len(reverse_image_transform_funcs) > 1:
target_arr = []
for i in range(len(target)):
target_arr.append(reverse_image_transform(target[i]))
tile_images_list.append(target_arr)
else:
target_arr = to_numpy(target).transpose([
0, 2, 3, 1
]) if get_backend() != 'tensorflow' else to_numpy(target)
if target_arr.max() <= 1.1 and target_arr.min() >= 0:
tile_images_list.append(target_arr * 255)
elif target_arr.max() <= 1.1 and target_arr.min() >= -1.1:
tile_images_list.append(target_arr * 128 + 127)
else:
tile_images_list.append(target_arr)
legend.append('target')
if self.include_output and output is not None:
if len(reverse_image_transform_funcs) > 1:
output_arr = []
for i in range(len(output)):
output_arr.append(reverse_image_transform(output[i]))
tile_images_list.append(output_arr)
else:
output_arr = to_numpy(output).transpose([
0, 2, 3, 1
]) if get_backend() != 'tensorflow' else to_numpy(output)
if output_arr.max() <= 1.2 and output_arr.min() >= 0:
tile_images_list.append(output_arr * 255)
elif output_arr.max() <= 1.2 and output_arr.min() >= -1.2:
tile_images_list.append(output_arr * 128 + 127)
else:
tile_images_list.append(output_arr)
legend.append('output')
# if self.tile_image_include_mask:
# tile_images_list.append(input*127.5+127.5)
fig = tile_rgb_images(*tile_images_list,
row=self.row,
save_path=os.path.join(
self.save_path, self.tile_image_name_prefix),
imshow=True,
legend=legend)
if ctx.enable_tensorboard and ctx.summary_writer is not None:
ctx.summary_writer.add_figure(
training_context['training_name'] + '/plot/tile_image',
fig,
global_step=training_context['steps'],
close=True,
walltime=time.time())
plt.close()
def plot_tile_image(self, training_context):
axis = 1
if get_backend() == 'tensorflow':
axis = -1
tile_images_list = []
input = None
target = None
output = None
is_label_mask = False
data_feed = training_context['data_feed']
data = training_context['train_data']
model = training_context['current_model']
if model.output_shape[model.filter_index] > 2:
is_label_mask = True
# if len(data) >= 3:
for data_key in data.key_list:
if data_key == data_feed[model.signature.inputs.key_list[0]]:
input = data[data_feed[model.signature.inputs.key_list[0]]]
model.eval()
if is_label_mask:
output = to_numpy(argmax(model(input), axis=axis))
else:
output = to_numpy(
expand_dims(cast(argmax(model(input), axis=axis),
input.dtype),
axis=-1))
model.train()
# elif data_key == data_feed[model.signature.outputs.key_list[0]]:
# output = data[data_feed[model.signature.outputs.key_list[0]]]
# if output.max() < 0:
# output = exp(output)
elif (
'target' in data_key or 'label' in data_key
or 'mask' in data_key
) and not 'output' in data_key and data_key in data_feed.value_list:
target = to_numpy(data[data_key])
output_arr = None
if 'alpha' not in data:
output_arr = output.copy()
if is_label_mask:
target = label2color(target, self.palette)
output = label2color(output, self.palette)
else:
if get_backend() == 'tensorflow':
output = output[:, :, :, 1:2] * argmax(output, axis)
else:
output = (output[:, 1:2, :, :] *
argmax(output, axis)).transpose(0, 2, 3, 1)
target = to_numpy(data['alpha'])
input_arr = []
input = to_numpy(input)
for i in range(len(input)):
input_arr.append(self.reverse_image_transform(input[i]))
# input_arr=np.asarray(input_arr)
tile_images_list.append(input_arr)
if is_label_mask:
tile_images_list.append(target)
tile_images_list.append(output)
else:
target_arr = target
if len(target.shape) < len(int_shape(input)):
if get_backend() == 'tensorflow':
target_arr = np.expand_dims(target, -1)
else:
target_arr = np.expand_dims(target, 1)
if 'alpha' not in data:
target_arr[target_arr > 0] = 1
background = np.ones_like(target_arr) * self.background
tile_images_list.append(target_arr * input_arr +
(1 - target_arr) * background)
tile_images_list.append(output_arr * input_arr +
(1 - output_arr) * background)
# if self.tile_image_include_mask:
# tile_images_list.append(input*127.5+127.5)
fig = tile_rgb_images(*tile_images_list,
save_path=os.path.join(
self.save_path, self.tile_image_name_prefix),
imshow=True)
if ctx.enable_tensorboard and ctx.summary_writer is not None:
ctx.summary_writer.add_figure(
training_context['training_name'] + '/plot/segtile_image',
fig,
global_step=training_context['steps'],
close=True,
walltime=time.time())
plt.close()
def on_optimization_step_start(self, training_context):
if get_backend() == 'pytorch':
with torch.no_grad():
if self.frequency > 0 and (
(self.unit == 'batch' and
(training_context['current_batch'] + 1) % self.frequency
== 0) or
(self.unit == 'step' and
(training_context['steps'] + 1) % self.frequency == 0) or
(self.unit == 'epoch' and
(training_context['current_epoch'] + 1) % self.frequency
== 0)):
grad_dict = OrderedDict()
if 'grads_state' not in training_context:
training_context['grads_state'] = OrderedDict()
if training_context[
'current_batch'] == 0 and training_context[
'current_epoch'] > 0:
# relocate the first/ last layers
self.first_layer = OrderedDict()
self.last_layer = OrderedDict()
if len(self.first_layer) == 0 and len(
self.last_layer) == 0:
if training_context['current_model'][
-1].__class__.__name__ == 'ModuleDict':
self.is_modulefict = True
for k, v in training_context['current_model'][
-1].items():
last_layer_name = ''
for name, module in v.named_modules():
if len([
pk
for pk, pv in module._parameters.
items() if 'bias' not in pk
and pv.requires_grad
]) > 0:
last_layer_name = module.relative_name
if last_layer_name != '':
self.last_layer[last_layer_name] = k
first_layer_name = ''
last_layer_name = ''
for k, v in training_context[
'current_model'].named_modules():
if len([
pk for pk, pv in v._parameters.items()
if 'bias' not in pk and pv is not None
and pv.requires_grad
]) > 0:
if first_layer_name == '':
first_layer_name = v.relative_name
self.first_layer[
first_layer_name] = 'first_layer'
if not self.is_modulefict:
last_layer_name = v.relative_name
if last_layer_name != '' and not self.is_modulefict:
self.last_layer[last_layer_name] = 'last_layer'
for name, module in training_context[
'current_model'].named_modules():
if module.relative_name in self.first_layer or module.relative_name in self.last_layer:
grads_data = [
np.abs(np.reshape(to_numpy(pv.grad.data), -1))
for pk, pv in module._parameters.items()
if 'bias' not in pk and pv is not None
and pv.requires_grad and pv.grad is not None
]
weights_data = [
np.abs(np.reshape(to_numpy(pv.data), -1))
for pk, pv in module._parameters.items()
if 'bias' not in pk and pv is not None
and pv.requires_grad
]
if ctx.enable_tensorboard and ctx.summary_writer is not None:
ctx.summary_writer.add_histogram(
training_context['training_name'] +
'/gradients/' +
self.first_layer[module.relative_name]
if module.relative_name in self.first_layer
else self.last_layer[module.relative_name],
np.concatenate(grads_data, axis=0),
training_context['steps'])
ctx.summary_writer.add_histogram(
training_context['training_name'] +
'/weights/' +
self.first_layer[module.relative_name]
if module.relative_name in self.first_layer
else self.last_layer[module.relative_name],
np.concatenate(weights_data, axis=0),
training_context['steps'])
if len(grads_data) > 0:
grads_data = np.concatenate(grads_data,
axis=0).mean()
else:
grads_data = None
if module.relative_name in self.first_layer:
training_context['grads_state'][
'first_layer'] = grads_data
elif module.relative_name in self.last_layer:
training_context['grads_state'][
self.last_layer[
module.relative_name]] = grads_data
if len(training_context['grads_state']) > 0:
grads_str = yellow_color(
'{0:<16s}'.format(
training_context['current_model'].name) +
'|'.join([
'{0} gradients: {1:<8.3e} '.format(k, v)
for k, v in training_context['grads_state'].
items() if isinstance(v, numbers.Number)
]))
self.lines.append(grads_str + '\n')
elif get_backend() == 'tensorflow':
if self.frequency > 0 and (
(self.unit == 'batch' and
(training_context['current_batch'] + 1) % self.frequency == 0)
or
(self.unit == 'step' and
(training_context['steps'] + 1) % self.frequency == 0) or
(self.unit == 'epoch'
and training_context['current_batch'] == 0 and
(training_context['current_epoch'] + 1) % self.frequency
== 0)):
grad_dict = OrderedDict()
if 'grads_state' not in training_context:
training_context['grads_state'] = OrderedDict()
if training_context['current_batch'] == 0 and training_context[
'current_epoch'] > 0:
# relocate the first/ last layers
self.first_layer = OrderedDict()
self.last_layer = OrderedDict()
if len(self.first_layer) == 0 and len(self.last_layer) == 0:
if training_context['current_model'][
-1].__class__.__name__ == 'ModuleDict':
self.is_modulefict = True
for k, v in training_context['current_model'][
-1].items():
last_layer_name = ''
for name, module in v.named_modules():
if len([
pk
for pk, pv in module._parameters.items(
) if 'bias' not in pk
and pv is not None and pv.trainable
]) > 0:
last_layer_name = module.relative_name
if last_layer_name != '':
self.last_layer[last_layer_name] = k
first_layer_name = ''
last_layer_name = ''
for k, v in training_context[
'current_model'].named_modules():
if len([
pk
for pk, pv in v._parameters.items() if 'bias'
not in pk and pv is not None and pv.trainable
]) > 0:
if first_layer_name == '':
first_layer_name = v.relative_name
self.first_layer[
first_layer_name] = 'first_layer'
if not self.is_modulefict:
last_layer_name = v.relative_name
if last_layer_name != '' and not self.is_modulefict:
self.last_layer[last_layer_name] = 'last_layer'
grads_and_vars = list(training_context['grads_and_vars'])
grads_dict = OrderedDict()
for grad, var in grads_and_vars:
grads_dict[var.ref()] = to_numpy(grad)
for name, module in training_context[
'current_model'].named_modules():
if module.relative_name in self.first_layer or module.relative_name in self.last_layer:
grads_data = [
np.abs(np.reshape(grads_dict[pv.ref()], -1))
for pk, pv in module._parameters.items()
if 'bias' not in pk and pv.trainable
and grads_dict[pv.ref()] is not None
]
weights_data = [
np.abs(np.reshape(to_numpy(pv.value()), -1))
for pk, pv in module._parameters.items()
if 'bias' not in pk and pv.trainable
]
if ctx.enable_tensorboard and ctx.summary_writer is not None:
ctx.summary_writer.add_histogram(
training_context['training_name'] +
'/gradients/' +
self.first_layer[module.relative_name]
if module.relative_name in self.first_layer
else self.last_layer[module.relative_name],
np.concatenate(grads_data, axis=0),
training_context['steps'])
ctx.summary_writer.add_histogram(
training_context['training_name'] +
'/weights/' +
self.first_layer[module.relative_name]
if module.relative_name in self.first_layer
else self.last_layer[module.relative_name],
np.concatenate(weights_data, axis=0),
training_context['steps'])
if len(grads_data) > 0:
grads_data = np.concatenate(grads_data,
axis=0).mean()
else:
grads_data = None
if module.relative_name in self.first_layer:
training_context['grads_state'][
'first_layer'] = grads_data
elif module.relative_name in self.last_layer:
training_context['grads_state'][self.last_layer[
module.relative_name]] = grads_data
if len(training_context['grads_state']) > 0:
grad_str = yellow_color('{0:<16s}'.format(
training_context['current_model'].name) + '|'.join([
'{0} gradients: {1:<8.3e} '.format(k, v)
for k, v in training_context['grads_state'].items(
) if isinstance(v, numbers.Number)
]))
self.lines.append(grad_str + '\n')
