def gather_from_batch(tensor: Tensor, indices: Tensor) -> Tensor: """Gather specific indices from a batch of data. This method can be useful if you need to compute gradients based on a specific subset of a tensor's output values. The `indices` will automatically be cast to the correct type (tf, torch, np) based on the type of the `tensor`. This method can be used with Numpy data: ```python ind = np.array([1, 0, 1]) n = np.array([[0, 1], [2, 3], [4, 5]]) b = fe.backend.gather_from_batch(n, ind) # [1, 2, 5] n = np.array([[[0, 1], [2, 3]], [[4, 5], [6, 7]], [[8, 9], [10, 11]]]) b = fe.backend.gather_from_batch(n, ind) # [[2, 3], [4, 5], [10, 11]] ``` This method can be used with TensorFlow tensors: ```python ind = tf.constant([1, 0, 1]) t = tf.constant([[0, 1], [2, 3], [4, 5]]) b = fe.backend.gather_from_batch(t, ind) # [1, 2, 5] t = tf.constant([[[0, 1], [2, 3]], [[4, 5], [6, 7]], [[8, 9], [10, 11]]]) b = fe.backend.gather_from_batch(t, ind) # [[2, 3], [4, 5], [10, 11]] ``` This method can be used with PyTorch tensors: ```python ind = torch.tensor([1, 0, 1]) p = torch.tensor([[0, 1], [2, 3], [4, 5]]) b = fe.backend.gather_from_batch(p, ind) # [1, 2, 5] p = torch.tensor([[[0, 1], [2, 3]], [[4, 5], [6, 7]], [[8, 9], [10, 11]]]) b = fe.backend.gather_from_batch(p, ind) # [[2, 3], [4, 5], [10, 11]] ``` Args: tensor: A tensor of shape (batch, d1, ..., dn). indices: A tensor of shape (batch, ) or (batch, 1) indicating which indices should be selected. Returns: A tensor of shape (batch, d2, ..., dn) containing the elements from `tensor` at the given `indices`. Raises: ValueError: If `tensor` is an unacceptable data type. """ if tf.is_tensor(tensor): indices = to_tensor(indices, 'tf') indices = tf.cast(indices, tf.int64) if len(indices.shape) == 1: # Indices not batched indices = expand_dims(indices, 1) return tf.gather_nd(tensor, indices=indices, batch_dims=1) elif isinstance(tensor, torch.Tensor): return tensor[torch.arange(tensor.shape[0]), squeeze(indices)] elif isinstance(tensor, np.ndarray): return tensor[np.arange(tensor.shape[0]), squeeze(indices)] else: raise ValueError("Unrecognized tensor type {}".format(type(tensor)))
def gather(tensor: Tensor, indices: Tensor) -> Tensor: """Gather specific indices from a tensor. The `indices` will automatically be cast to the correct type (tf, torch, np) based on the type of the `tensor`. This method can be used with Numpy data: ```python ind = np.array([1, 0, 1]) n = np.array([[0, 1], [2, 3], [4, 5]]) b = fe.backend.gather(n, ind) # [[2, 3], [0, 1], [2, 3]] n = np.array([[[0, 1], [2, 3]], [[4, 5], [6, 7]], [[8, 9], [10, 11]]]) b = fe.backend.gather(n, ind) # [[[4, 5], [6, 7]], [[0, 1], [2, 3]], [[4, 5], [6, 7]]] ``` This method can be used with TensorFlow tensors: ```python ind = tf.constant([1, 0, 1]) t = tf.constant([[0, 1], [2, 3], [4, 5]]) b = fe.backend.gather(t, ind) # [[2, 3], [0, 1], [2, 3]] t = tf.constant([[[0, 1], [2, 3]], [[4, 5], [6, 7]], [[8, 9], [10, 11]]]) b = fe.backend.gather(t, ind) # [[[4, 5], [6, 7]], [[0, 1], [2, 3]], [[4, 5], [6, 7]]] ``` This method can be used with PyTorch tensors: ```python ind = torch.tensor([1, 0, 1]) p = torch.tensor([[0, 1], [2, 3], [4, 5]]) b = fe.backend.gather(p, ind) # [[2, 3], [0, 1], [2, 3]] p = torch.tensor([[[0, 1], [2, 3]], [[4, 5], [6, 7]], [[8, 9], [10, 11]]]) b = fe.backend.gather(p, ind) # [[[4, 5], [6, 7]], [[0, 1], [2, 3]], [[4, 5], [6, 7]]] ``` Args: tensor: A tensor to gather values from. indices: A tensor indicating which indices should be selected. These represent locations along the 0 axis. Returns: A tensor containing the elements from `tensor` at the given `indices`. Raises: ValueError: If `tensor` is an unacceptable data type. """ if tf.is_tensor(tensor): indices = to_tensor(indices, 'tf') indices = tf.cast(indices, tf.int64) return tf.gather(tensor, indices=squeeze(indices), axis=0) elif isinstance(tensor, torch.Tensor): return tensor[squeeze(indices).type(torch.int64)] elif isinstance(tensor, np.ndarray): return np.take(tensor, squeeze(indices).astype('int64'), axis=0) else: raise ValueError("Unrecognized tensor type {}".format(type(tensor)))
def _weight_to_image( weight: Tensor, kernel_channels_last: bool = False) -> Optional[Tensor]: """Logs a weight as a TensorBoard image. Implementation from TensorFlow codebase, would have invoked theirs directly but they didn't make it a static method. """ w_img = squeeze(weight) shape = backend.int_shape(w_img) if len(shape) == 1: # Bias case w_img = reshape(w_img, [1, shape[0], 1, 1]) elif len(shape) == 2: # Dense layer kernel case if shape[0] > shape[1]: w_img = permute(w_img, [0, 1]) shape = backend.int_shape(w_img) w_img = reshape(w_img, [1, shape[0], shape[1], 1]) elif len(shape) == 3: # ConvNet case if kernel_channels_last: # Switch to channels_first to display every kernel as a separate images w_img = permute(w_img, [2, 0, 1]) w_img = expand_dims(w_img, axis=-1) elif len(shape) == 4: # Conv filter with multiple input channels if kernel_channels_last: # Switch to channels first to display kernels as separate images w_img = permute(w_img, [3, 2, 0, 1]) w_img = reduce_sum( abs(w_img), axis=1) # Sum over the each channel within the kernel w_img = expand_dims(w_img, axis=-1) shape = backend.int_shape(w_img) # Not possible to handle 3D convnets etc. if len(shape) == 4 and shape[-1] in [1, 3, 4]: return w_img
def on_epoch_end(self, data: Data) -> None: mode = self.system.mode if self.n_found[mode] > 0: if self.n_required[mode] > 0: # We are keeping a user-specified number of samples self.samples[mode] = { key: concat(val)[:self.n_required[mode]] for key, val in self.samples[mode].items() } else: # We are keeping one batch of data self.samples[mode] = { key: val[0] for key, val in self.samples[mode].items() } # even if you haven't found n_required samples, you're at end of epoch so no point trying to collect more self.n_found[mode] = 0 self.n_required[mode] = 0 masks = self.salnet.get_masks(self.samples[mode]) smoothed, integrated, smint = {}, {}, {} if self.smoothing: smoothed = self.salnet.get_smoothed_masks(self.samples[mode], nsamples=self.smoothing) if self.integrating: if isinstance(self.integrating, Tuple): n_integration, n_smoothing = self.integrating else: n_integration = self.integrating n_smoothing = self.smoothing integrated = self.salnet.get_integrated_masks( self.samples[mode], nsamples=n_integration) if n_smoothing: smint = self.salnet.get_smoothed_masks( self.samples[mode], nsamples=n_smoothing, nintegration=n_integration) # Arrange the outputs args = {} if self.class_key: classes = self.samples[mode][self.class_key] if self.label_mapping: classes = np.array([ self.label_mapping[clazz] for clazz in to_number(squeeze(classes)) ]) args[self.class_key] = classes for key in self.model_outputs: classes = masks[key] if self.label_mapping: classes = np.array([ self.label_mapping[clazz] for clazz in to_number(squeeze(classes)) ]) args[key] = classes sal = smint or integrated or smoothed or masks for key, val in self.samples[mode].items(): if key is not self.class_key: args[key] = val # Create a linear combination of the original image, the saliency mask, and the product of the two in # order to highlight regions of importance min_val = reduce_min(val) diff = reduce_max(val) - min_val for outkey in self.outputs: args["{} {}".format( key, outkey)] = (0.3 * (sal[outkey] * (val - min_val) + min_val) + 0.3 * val + 0.4 * sal[outkey] * diff + min_val) for key in self.outputs: args[key] = masks[key] if smoothed: args["Smoothed {}".format(key)] = smoothed[key] if integrated: args["Integrated {}".format(key)] = integrated[key] if smint: args["SmInt {}".format(key)] = smint[key] result = ImgData(colormap="inferno", **args) data.write_without_log(self.outputs[0], result)
def on_epoch_end(self, data: Data) -> None: # Keep only the user-specified number of samples images = concat(self.images)[:self.n_samples or self.n_found] _, height, width = get_image_dims(images) grads = to_number(concat(self.grads)[:self.n_samples or self.n_found]) if tf.is_tensor(images): grads = np.moveaxis(grads, source=-1, destination=1) # grads should be channel first args = {} labels = None if not self.labels else concat( self.labels)[:self.n_samples or self.n_found] if labels is not None: if len(labels.shape) > 1: labels = argmax(labels, axis=-1) if self.label_mapping: labels = np.array([ self.label_mapping[clazz] for clazz in to_number(squeeze(labels)) ]) args[self.true_label_key] = labels preds = None if not self.preds else concat( self.preds)[:self.n_samples or self.n_found] if preds is not None: if len(preds.shape) > 1: preds = argmax(preds, axis=-1) if self.label_mapping: preds = np.array([ self.label_mapping[clazz] for clazz in to_number(squeeze(preds)) ]) args[self.pred_label_key] = preds args[self.image_key] = images # Clear memory self._reset() # Make the image # TODO: In future maybe allow multiple different grads to have side-by-side comparisons of classes components = [np.mean(grads, axis=1)] components = [np.maximum(component, 0) for component in components] masks = [] for component_batch in components: img_batch = [] for img in component_batch: img = cv2.resize(img, (width, height)) img = img - np.min(img) img = img / np.max(img) img = cv2.cvtColor( cv2.applyColorMap(np.uint8(255 * img), cv2.COLORMAP_JET), cv2.COLOR_BGR2RGB) img = np.float32(img) / 255 img_batch.append(img) img_batch = np.array(img_batch, dtype=np.float32) # Switch to channel first for pytorch if isinstance(images, torch.Tensor): img_batch = np.moveaxis(img_batch, source=-1, destination=1) masks.append(img_batch) components = [ images + mask for mask in masks ] # This seems to work even if the image is 1 channel instead of 3 components = [image / reduce_max(image) for image in components] for elem in components: args[self.grad_key] = elem result = ImgData(**args) data.write_without_log(self.outputs[0], result)
def on_epoch_end(self, data: Data) -> None: # Keep only the user-specified number of samples images = concat(self.images)[:self.n_samples or self.n_found] _, height, width = get_image_dims(images) activations = to_number( concat(self.activations)[:self.n_samples or self.n_found]) if tf.is_tensor(images): activations = np.moveaxis( activations, source=-1, destination=1) # Activations should be channel first args = {} labels = None if not self.labels else concat( self.labels)[:self.n_samples or self.n_found] if labels is not None: if len(labels.shape) > 1: labels = argmax(labels, axis=-1) if self.label_mapping: labels = np.array([ self.label_mapping[clazz] for clazz in to_number(squeeze(labels)) ]) args[self.true_label_key] = labels preds = None if not self.preds else concat( self.preds)[:self.n_samples or self.n_found] if preds is not None: if len(preds.shape) > 1: preds = argmax(preds, axis=-1) if self.label_mapping: preds = np.array([ self.label_mapping[clazz] for clazz in to_number(squeeze(preds)) ]) args[self.pred_label_key] = preds args[self.image_key] = images # Clear memory self._reset() # Make the image n_components, batch_component_image = self._project_2d(activations) components = [] # component x image (batch x image) for component_idx in range(n_components): batch = [] for base_image, component_image in zip(images, batch_component_image): if len(component_image) > component_idx: mask = component_image[component_idx] mask = cv2.resize(mask, (width, height)) mask = mask - np.min(mask) mask = mask / np.max(mask) mask = cv2.cvtColor( cv2.applyColorMap(np.uint8(255 * mask), cv2.COLORMAP_JET), cv2.COLOR_BGR2RGB) mask = np.float32(mask) / 255 # switch to channel first for pytorch if isinstance(base_image, torch.Tensor): mask = np.moveaxis(mask, source=-1, destination=1) new_image = base_image + mask new_image = new_image / reduce_max(new_image) else: # There's no component for this image, so display an empty image here new_image = np.ones_like(base_image) batch.append(new_image) components.append(np.array(batch, dtype=np.float32)) for idx, elem in enumerate(components): args[f"Component {idx}"] = elem result = ImgData(**args) data.write_without_log(self.outputs[0], result)