def on_batch_end(self, data: Data) -> None:
y_true, y_pred = to_number(data[self.true_key]), to_number(data[self.pred_key])
if y_true.shape[-1] > 1 and y_true.ndim > 1:
y_true = np.argmax(y_true, axis=-1)
if y_pred.shape[-1] > 1:
y_pred = np.argmax(y_pred, axis=-1)
else:
y_pred = np.round(y_pred)
assert y_pred.size == y_true.size
self.correct += np.sum(y_pred.ravel() == y_true.ravel())
self.total += len(y_pred.ravel())
def on_batch_end(self, data: Data) -> None:
y_true, y_pred = to_number(data[self.true_key]), to_number(data[self.pred_key])
self.binary_classification = y_pred.shape[-1] == 1
if y_true.shape[-1] > 1 and y_true.ndim > 1:
y_true = np.argmax(y_true, axis=-1)
if y_pred.shape[-1] > 1:
y_pred = np.argmax(y_pred, axis=-1)
else:
y_pred = np.round(y_pred)
assert y_pred.size == y_true.size
self.y_pred.extend(y_pred.ravel())
self.y_true.extend(y_true.ravel())
def on_batch_end(self, data: Data) -> None:
y_true, y_pred = to_number(data[self.true_key]), to_number(
data[self.pred_key])
batch_size = y_true.shape[0]
y_true, y_pred = y_true.reshape((batch_size, -1)), y_pred.reshape(
(batch_size, -1))
prediction_label = (y_pred >= self.threshold).astype(np.int32)
intersection = np.sum(y_true * prediction_label, axis=-1)
area_sum = np.sum(y_true, axis=-1) + np.sum(prediction_label, axis=-1)
dice = (2. * intersection + self.smooth) / (area_sum + self.smooth)
self.dice.extend(list(dice))
def write_embeddings(
self,
mode: str,
embeddings: Iterable[Tuple[str, Tensor, Optional[List[Any]],
Optional[Tensor]]],
step: int,
):
"""Write embeddings (like UMAP) to TensorBoard.
Args:
mode: The current mode of execution ('train', 'eval', 'test', 'infer').
embeddings: A collection of quadruplets like [("key", <features>, [<label1>, ...], <label_images>)].
Features are expected to be batched, and if labels and/or label images are provided they should have the
same batch dimension as the features.
step: The current training step.
"""
for key, features, labels, label_imgs in embeddings:
flat = to_number(reshape(features, [features.shape[0], -1]))
if not isinstance(label_imgs, (torch.Tensor, type(None))):
label_imgs = to_tensor(label_imgs, 'torch')
if len(label_imgs.shape) == 4:
label_imgs = permute(label_imgs, [0, 3, 1, 2])
self.summary_writers[mode].add_embedding(mat=flat,
metadata=labels,
label_img=label_imgs,
tag=key,
global_step=step)
def get_smoothed_masks(
self,
batch: Dict[str, Any],
stdev_spread: float = .15,
nsamples: int = 25,
nintegration: Optional[int] = None,
magnitude: bool = True) -> Dict[str, Union[Tensor, np.ndarray]]:
"""Generates smoothed greyscale saliency mask(s) from a given `batch` of data.
Args:
batch: An input batch of data.
stdev_spread: Amount of noise to add to the input, as fraction of the total spread (x_max - x_min).
nsamples: Number of samples to average across to get the smooth gradient.
nintegration: Number of samples to compute when integrating (None to disable).
magnitude: If true, computes the sum of squares of gradients instead of just the sum.
Returns:
Greyscale saliency mask(s) smoothed via the SmoothGrad method.
"""
# Shallow copy batch since we're going to modify its contents later
batch = {key: val for key, val in batch.items()}
model_inputs = [batch[ins] for ins in self.model_inputs]
stdevs = [
to_number(stdev_spread *
(reduce_max(ins) - reduce_min(ins))).item()
for ins in model_inputs
]
# Adding noise to the image might cause the max likelihood class value to change, so need to keep track of
# which class we're comparing to
response = self._get_mask(batch)
for gather_key, output_key in zip(self.gather_keys,
self.model_outputs):
batch[gather_key] = response[output_key]
if magnitude:
for key in self.outputs:
response[key] = response[key] * response[key]
for _ in range(nsamples - 1):
noisy_batch = {key: batch[key] for key in self.gather_keys}
for idx, input_name in enumerate(self.model_inputs):
noise = random_normal_like(model_inputs[idx], std=stdevs[idx])
x_plus_noise = model_inputs[idx] + noise
noisy_batch[input_name] = x_plus_noise
grads_and_preds = self._get_mask(
noisy_batch
) if not nintegration else self._get_integrated_masks(
noisy_batch, nsamples=nintegration)
for name in self.outputs:
grad = grads_and_preds[name]
if magnitude:
response[name] += grad * grad
else:
response[name] += grad
for key in self.outputs:
grad = response[key]
response[key] = self._convert_for_visualization(grad / nsamples)
return response
def on_batch_end(self, data: Data) -> None:
y_true, y_pred = to_number(data[self.true_key]), to_number(
data[self.pred_key])
if y_true.shape[-1] > 1 and y_true.ndim > 1:
y_true = np.argmax(y_true, axis=-1)
if y_pred.shape[-1] > 1:
y_pred = np.argmax(y_pred, axis=-1)
else:
y_pred = np.round(y_pred)
assert y_pred.size == y_true.size
batch_confusion = confusion_matrix(y_true,
y_pred,
labels=list(
range(0, self.num_classes)))
if self.matrix is None:
self.matrix = batch_confusion
else:
self.matrix += batch_confusion
def write_scalars(self, mode: str, scalars: Iterable[Tuple[str, Any]],
step: int) -> None:
"""Write summaries of scalars to TensorBoard.
Args:
mode: The current mode of execution ('train', 'eval', 'test', 'infer').
scalars: A collection of pairs like [("key", val), ("key2", val2), ...].
step: The current training step.
"""
for key, val in scalars:
self.summary_writers[mode].add_scalar(tag=key,
scalar_value=to_number(val),
global_step=step)
def write_images(self, mode: str, images: Iterable[Tuple[str, Any]],
step: int) -> None:
"""Write images to TensorBoard.
Args:
mode: The current mode of execution ('train', 'eval', 'test', 'infer').
images: A collection of pairs like [("key", image1), ("key2", image2), ...].
step: The current training step.
"""
for key, img in images:
if isinstance(img, ImgData):
img = img.paint_figure()
if isinstance(img, plt.Figure):
self.summary_writers[mode].add_figure(tag=key,
figure=img,
global_step=step)
else:
self.summary_writers[mode].add_images(
tag=key,
img_tensor=to_number(img),
global_step=step,
dataformats='NCHW'
if isinstance(img, torch.Tensor) else 'NHWC')
def _print_message(self,
header: str,
data: Data,
log_epoch: bool = False) -> None:
"""Print a log message to the screen, and record the `data` into the `system` summary.
Args:
header: The prefix for the log message.
data: A collection of data to be recorded.
log_epoch: Whether epoch information should be included in the log message.
"""
log_message = header
if log_epoch:
log_message += "epoch: {}; ".format(self.system.epoch_idx)
self.system.write_summary('epoch', self.system.epoch_idx)
for key, val in data.read_logs().items():
val = to_number(val)
self.system.write_summary(key, val)
if val.size > 1:
log_message += "\n{}:\n{};".format(
key, np.array2string(val, separator=','))
else:
log_message += "{}: {}; ".format(key, str(val))
print(log_message)
def on_batch_end(self, data: Data):
# begin of reading det and gt
pred = to_number(
data[self.pred_key]) # pred is [batch, nms_max_outputs, 7]
pred = self._reshape_pred(pred)
gt = to_number(
data[self.true_key]
) # gt is np.array (batch, box, 5), box dimension is padded
gt = self._reshape_gt(gt)
ground_truth_bb = []
for gt_item in gt:
idx_in_batch, x1, y1, w, h, label = gt_item
label = int(label)
id_epoch = self._get_id_in_epoch(idx_in_batch)
self.batch_image_ids.append(id_epoch)
self.image_ids.append(id_epoch)
tmp_dict = {
'idx': id_epoch,
'x1': x1,
'y1': y1,
'w': w,
'h': h,
'label': label
}
ground_truth_bb.append(tmp_dict)
predicted_bb = []
for pred_item in pred:
idx_in_batch, x1, y1, w, h, label, score = pred_item
label = int(label)
id_epoch = self.ids_batch_to_epoch[idx_in_batch]
self.image_ids.append(id_epoch)
tmp_dict = {
'idx': id_epoch,
'x1': x1,
'y1': y1,
'w': w,
'h': h,
'label': label,
'score': score
}
predicted_bb.append(tmp_dict)
for dict_elem in ground_truth_bb:
self.gt[dict_elem['idx'], dict_elem['label']].append(dict_elem)
for dict_elem in predicted_bb:
self.det[dict_elem['idx'], dict_elem['label']].append(dict_elem)
# end of reading det and gt
# compute iou matrix, matrix index is (img_id, cat_id), each element in matrix has shape (num_det, num_gt)
self.ious = {(img_id, cat_id):
self.compute_iou(self.det[img_id, cat_id],
self.gt[img_id, cat_id])
for img_id in self.batch_image_ids
for cat_id in self.categories}
for cat_id in self.categories:
for img_id in self.batch_image_ids:
self.evalimgs[(cat_id,
img_id)] = self.evaluate_img(cat_id, img_id)
def show_image(im: Union[np.ndarray, Tensor],
axis: plt.Axes = None,
fig: plt.Figure = None,
title: Optional[str] = None,
color_map: str = "inferno",
stack_depth: int = 0) -> Optional[plt.Figure]:
"""Plots a given image onto an axis. The repeated invocation of this function will cause figure plot overlap.
If `im` is 2D and the length of second dimension are 4 or 5, it will be viewed as bounding box data (x0, y0, w, h,
<label>).
```python
boxes = np.array([[0, 0, 10, 20, "apple"],
[10, 20, 30, 50, "dog"],
[40, 70, 200, 200, "cat"],
[0, 0, 0, 0, "not_shown"],
[0, 0, -10, -20, "not_shown2"]])
img = np.zeros((150, 150))
fig, axis = plt.subplots(1, 1)
fe.util.show_image(img, fig=fig, axis=axis) # need to plot image first
fe.util.show_image(boxes, fig=fig, axis=axis)
```
Users can also directly plot text
```python
fig, axis = plt.subplots(1, 1)
fe.util.show_image("apple", fig=fig, axis=axis)
```
Args:
axis: The matplotlib axis to plot on, or None for a new plot.
fig: A reference to the figure to plot on, or None if new plot.
im: The image (width X height) / bounding box / text to display.
title: A title for the image.
color_map: Which colormap to use for greyscale images.
stack_depth: Multiple images can be drawn onto the same axis. When stack depth is greater than zero, the `im`
will be alpha blended on top of a given axis.
Returns:
plotted figure. It will be the same object as user have provided in the argument.
"""
if axis is None:
fig, axis = plt.subplots(1, 1)
axis.axis('off')
# Compute width of axis for text font size
bbox = axis.get_window_extent().transformed(fig.dpi_scale_trans.inverted())
width, height = bbox.width * fig.dpi, bbox.height * fig.dpi
space = min(width, height)
if not hasattr(im, 'shape') or len(im.shape) < 2:
# text data
im = to_number(im)
if hasattr(im, 'shape') and len(im.shape) == 1:
im = im[0]
im = im.item()
if isinstance(im, bytes):
im = im.decode('utf8')
text = "{}".format(im)
axis.text(0.5,
0.5,
im,
ha='center',
transform=axis.transAxes,
va='center',
wrap=False,
family='monospace',
fontsize=min(45, space // len(text)))
elif len(im.shape) == 2 and (im.shape[1] == 4 or im.shape[1] == 5):
# Bounding Box Data. Should be (x0, y0, w, h, <label>)
boxes = []
im = to_number(im)
color = ["m", "r", "c", "g", "y", "b"][stack_depth % 6]
for box in im:
# Unpack the box, which may or may not have a label
x0 = int(box[0])
y0 = int(box[1])
width = int(box[2])
height = int(box[3])
label = None if len(box) < 5 else str(box[4])
# Don't draw empty boxes, or invalid box
if width <= 0 or height <= 0:
continue
r = Rectangle((x0, y0),
width=width,
height=height,
fill=False,
edgecolor=color,
linewidth=3)
boxes.append(r)
if label:
axis.text(r.get_x() + 3,
r.get_y() + 3,
label,
ha='left',
va='top',
color=color,
fontsize=max(8, min(14, width // len(label))),
fontweight='bold',
family='monospace')
pc = PatchCollection(boxes, match_original=True)
axis.add_collection(pc)
else:
if isinstance(im, torch.Tensor) and len(im.shape) > 2:
# Move channel first to channel last
channels = list(range(len(im.shape)))
channels.append(channels.pop(0))
im = im.permute(*channels)
# image data
im = to_number(im)
if np.issubdtype(im.dtype, np.integer):
# im is already in int format
im = im.astype(np.uint8)
elif np.max(im) <= 1 and np.min(im) >= 0: # im is [0,1]
im = (im * 255).astype(np.uint8)
elif np.min(im) >= -1 and np.max(im) <= 1: # im is [-1, 1]
im = ((im + 1) * 127.5).astype(np.uint8)
else: # im is in some arbitrary range, probably due to the Normalize Op
ma = abs(
np.max(im,
axis=tuple([i for i in range(len(im.shape) - 1)])
if len(im.shape) > 2 else None))
mi = abs(
np.min(im,
axis=tuple([i for i in range(len(im.shape) - 1)])
if len(im.shape) > 2 else None))
im = (((im + mi) / (ma + mi)) * 255).astype(np.uint8)
# matplotlib doesn't support (x,y,1) images, so convert them to (x,y)
if len(im.shape) == 3 and im.shape[2] == 1:
im = np.reshape(im, (im.shape[0], im.shape[1]))
alpha = 1 if stack_depth == 0 else 0.3
if len(im.shape) == 2:
axis.imshow(im, cmap=plt.get_cmap(name=color_map), alpha=alpha)
else:
axis.imshow(im, alpha=alpha)
if title is not None:
axis.set_title(title,
fontsize=min(20, 1 + width // len(title)),
family='monospace')
return fig
def on_epoch_end(self, data: Data) -> None:
mode = self.system.mode
if self.n_found[mode] > 0:
if self.n_required[mode] > 0:
# We are keeping a user-specified number of samples
self.samples[mode] = {
key: concat(val)[:self.n_required[mode]]
for key, val in self.samples[mode].items()
}
else:
# We are keeping one batch of data
self.samples[mode] = {
key: val[0]
for key, val in self.samples[mode].items()
}
# even if you haven't found n_required samples, you're at end of epoch so no point trying to collect more
self.n_found[mode] = 0
self.n_required[mode] = 0
masks = self.salnet.get_masks(self.samples[mode])
smoothed, integrated, smint = {}, {}, {}
if self.smoothing:
smoothed = self.salnet.get_smoothed_masks(self.samples[mode],
nsamples=self.smoothing)
if self.integrating:
if isinstance(self.integrating, Tuple):
n_integration, n_smoothing = self.integrating
else:
n_integration = self.integrating
n_smoothing = self.smoothing
integrated = self.salnet.get_integrated_masks(
self.samples[mode], nsamples=n_integration)
if n_smoothing:
smint = self.salnet.get_smoothed_masks(
self.samples[mode],
nsamples=n_smoothing,
nintegration=n_integration)
# Arrange the outputs
args = {}
if self.class_key:
classes = self.samples[mode][self.class_key]
if self.label_mapping:
classes = np.array([
self.label_mapping[clazz]
for clazz in to_number(squeeze(classes))
])
args[self.class_key] = classes
for key in self.model_outputs:
classes = masks[key]
if self.label_mapping:
classes = np.array([
self.label_mapping[clazz]
for clazz in to_number(squeeze(classes))
])
args[key] = classes
sal = smint or integrated or smoothed or masks
for key, val in self.samples[mode].items():
if key is not self.class_key:
args[key] = val
# Create a linear combination of the original image, the saliency mask, and the product of the two in
# order to highlight regions of importance
min_val = reduce_min(val)
diff = reduce_max(val) - min_val
for outkey in self.outputs:
args["{} {}".format(
key, outkey)] = (0.3 * (sal[outkey] *
(val - min_val) + min_val) +
0.3 * val + 0.4 * sal[outkey] * diff +
min_val)
for key in self.outputs:
args[key] = masks[key]
if smoothed:
args["Smoothed {}".format(key)] = smoothed[key]
if integrated:
args["Integrated {}".format(key)] = integrated[key]
if smint:
args["SmInt {}".format(key)] = smint[key]
result = ImgData(colormap="inferno", **args)
data.write_without_log(self.outputs[0], result)