def write_embeddings(
self,
mode: str,
embeddings: Iterable[Tuple[str, Tensor, Optional[List[Any]],
Optional[Tensor]]],
step: int,
):
"""Write embeddings (like UMAP) to TensorBoard.
Args:
mode: The current mode of execution ('train', 'eval', 'test', 'infer').
embeddings: A collection of quadruplets like [("key", <features>, [<label1>, ...], <label_images>)].
Features are expected to be batched, and if labels and/or label images are provided they should have the
same batch dimension as the features.
step: The current training step.
"""
for key, features, labels, label_imgs in embeddings:
flat = to_number(reshape(features, [features.shape[0], -1]))
if not isinstance(label_imgs, (torch.Tensor, type(None))):
label_imgs = to_tensor(label_imgs, 'torch')
if len(label_imgs.shape) == 4:
label_imgs = permute(label_imgs, [0, 3, 1, 2])
self.summary_writers[mode].add_embedding(mat=flat,
metadata=labels,
label_img=label_imgs,
tag=key,
global_step=step)
def transform(
self,
data: Dict[str, Any],
mode: str,
epoch: int = 1,
ds_id: str = '',
target_type: str = 'np') -> Union[Dict[str, Any], FilteredData]:
"""Apply all pipeline operations on a given data instance for the specified `mode` and `epoch`.
Args:
data: Input data in dictionary format.
mode: The execution mode in which to run. This can be "train", "eval", "test" or "infer".
epoch: The epoch index to run. Note that epoch indices are 1-indexed.
ds_id: The current dataset id.
target_type: What kind of tensor(s) to create. One of "tf", "torch", or "np".
Returns:
The transformed data.
"""
data = deepcopy(data)
instance_ops, batch_spec, batch_ops = self._get_op_split(mode=mode,
epoch=epoch,
ds_id=ds_id)
state = {'mode': mode}
op_data = forward_numpyop(instance_ops, data, state)
if isinstance(op_data, FilteredData):
return op_data
data = batch_spec.collate_fn([data])
op_data = forward_numpyop(batch_ops, data, state, batched='torch')
if isinstance(op_data, FilteredData):
return op_data
return to_tensor(data, target_type=target_type)
def _configure_loader(
self, loader: Union[DataLoader, tf.data.Dataset]
) -> Union[DataLoader, tf.data.Dataset]:
"""A method to configure a given dataloader for use with this Estimator's Network.
This method will ensure that the `loader` returns the correct data type (tf.Tensor or torch.Tensor) depending on
the requirements of the Network. It also handles issues with multi-gpu data sharding.
Args:
loader: A data loader to be modified.
Returns:
The potentially modified dataloader to be used for training.
"""
new_loader = loader
if isinstance(new_loader, DataLoader) and isinstance(
self.network, TFNetwork):
add_batch = bool(new_loader.batch_size)
if hasattr(loader, 'fe_postprocess_fn'
) and loader.fe_postprocess_fn is not None:
# The user is manually batching data and running ops on data batches. No reliable way to shortcut this
# since ops might require specific batch composition.
data_instance = next(iter(loader))
add_batch = False
else:
# No batch-based ops so we can try and just use the OpDataset to more quickly get our data summary
data_instance = loader.dataset[0]
if isinstance(data_instance, list):
# This is a batched dataset
data_instance = data_instance[0]
add_batch = True
if isinstance(data_instance, FilteredData):
# We got unlucky and drew filtered data as the zeroth element. Fall back to a slower but more robust
# analysis of the batch
data_instance = next(iter(loader))
add_batch = False
data_instance = to_tensor(data_instance, target_type="tf")
data_type = to_type(data_instance)
data_shape = to_shape(data_instance,
add_batch=add_batch,
exact_shape=False)
new_loader = tf.data.Dataset.from_generator(
lambda: loader, data_type, output_shapes=data_shape)
new_loader = new_loader.prefetch(1)
if isinstance(new_loader, tf.data.Dataset):
if self.system.train_steps_per_epoch and self.system.mode == "train":
new_loader = new_loader.take(self.system.train_steps_per_epoch)
if self.system.eval_steps_per_epoch and self.system.mode == "eval":
new_loader = new_loader.take(self.system.eval_steps_per_epoch)
if isinstance(tf.distribute.get_strategy(),
tf.distribute.MirroredStrategy) and isinstance(
self.network, TFNetwork) and not isinstance(
new_loader, DistributedDataset):
# The default autoshard policy is file, changing it to data to avoid warning
options = tf.data.Options()
options.experimental_distribute.auto_shard_policy = tf.data.experimental.AutoShardPolicy.DATA
new_loader = new_loader.with_options(options)
new_loader = tf.distribute.get_strategy(
).experimental_distribute_dataset(new_loader)
return new_loader
def _configure_loader(self, loader: Union[DataLoader, tf.data.Dataset]) -> Union[DataLoader, tf.data.Dataset]:
"""A method to configure a given dataloader for use with this Estimator's Network.
This method will ensure that the `loader` returns the correct data type (tf.Tensor or torch.Tensor) depending on
the requirements of the Network. It also handles issues with multi-gpu data sharding.
Args:
loader: A data loader to be modified.
Returns:
The potentially modified dataloader to be used for training.
"""
new_loader = loader
if isinstance(new_loader, DataLoader) and isinstance(self.network, TFNetwork):
add_batch = True
if hasattr(loader.dataset, "dataset") and isinstance(loader.dataset.dataset, BatchDataset):
add_batch = False
batch = to_tensor(loader.dataset[0], target_type="tf")
data_type = to_type(batch)
data_shape = to_shape(batch, add_batch=add_batch, exact_shape=False)
new_loader = tf.data.Dataset.from_generator(lambda: loader, data_type, output_shapes=data_shape)
new_loader = new_loader.prefetch(1)
if isinstance(new_loader, tf.data.Dataset):
if self.system.max_train_steps_per_epoch and self.system.mode == "train":
new_loader = new_loader.take(self.system.max_train_steps_per_epoch)
if self.system.max_eval_steps_per_epoch and self.system.mode == "eval":
new_loader = new_loader.take(self.system.max_eval_steps_per_epoch)
if isinstance(tf.distribute.get_strategy(),
tf.distribute.MirroredStrategy) and not isinstance(new_loader, DistributedDataset):
new_loader = tf.distribute.get_strategy().experimental_distribute_dataset(new_loader)
return new_loader
def build(self, framework: str) -> None:
labels = hadamard(self.code_length).astype(np.float32)
labels[np.arange(0, self.code_length, 2),
0] = -1 # Make first column alternate
labels = labels[:self.n_classes]
self.labels = to_tensor(labels, target_type=framework)
max_prob = 0.99999 # This will only be approximate since the first column is alternating
power = 1.0
self.eps = to_tensor(np.array((self.code_length + 1) * math.pow(
(1.0 - max_prob) / (max_prob * (self.n_classes - 1)), 1 / power),
dtype=np.float32),
target_type=framework)
if framework == "torch":
self.labels = self.labels.to(
"cuda:0" if torch.cuda.is_available() else "cpu")
self.eps = self.eps.to(
"cuda:0" if torch.cuda.is_available() else "cpu")
def forward_numpyop(ops: List[NumpyOp],
data: MutableMapping[str, Any],
state: Dict[str, Any],
batched: Optional[str] = None) -> Optional[FilteredData]:
"""Call the forward function for list of NumpyOps, and modify the data dictionary in place.
Args:
ops: A list of NumpyOps to execute.
data: The data dictionary.
state: Information about the current execution context, ex. {"mode": "train"}. Must contain at least the mode.
batched: Whether the `data` is batched or not. If it is batched, provide the string ('tf', 'torch', or 'np')
indicating which type of tensors the batch contains.
"""
if batched:
# Cast data to Numpy before performing batch forward
for key, val in data.items():
data[key] = to_tensor(val, target_type='np')
for op in ops:
op_data = get_inputs_by_op(op, data, copy_on_write=op.in_place_edits)
try:
op_data = op.forward_batch(
op_data, state) if batched else op.forward(op_data, state)
except ValueError as err:
if err.args[0] == 'assignment destination is read-only':
# If the numpy error text changes we'll need to make adjustments in the future
op.in_place_edits = True
op_data = get_inputs_by_op(op,
data,
copy_on_write=op.in_place_edits)
op_data = op.forward_batch(
op_data, state) if batched else op.forward(op_data, state)
else:
raise err
if isinstance(op_data, FilteredData):
return op_data
if isinstance(op, Delete):
for key in op.inputs:
del data[key]
if op.outputs:
write_outputs_by_op(op, data, op_data)
if batched:
# Cast data back to original tensor type after performing batch forward
for key, val in data.items():
data[key] = to_tensor(val, target_type=batched, shared_memory=True)
return None
def _configure_tensor(self, loader: Union[DataLoader, tf.data.Dataset], batch: Dict[str, Any]) -> Dict[str, Any]:
"""A function to convert a batch of tf.Tensors to torch.Tensors if required.
Returns:
Either the original `batch`, or the `batch` converted to torch.Tensors if required.
"""
if isinstance(loader, tf.data.Dataset) and isinstance(self.network, TorchNetwork):
batch = to_tensor(batch, target_type="torch")
return batch
def feed_forward(model: Union[tf.keras.Model, torch.nn.Module],
x: Union[Tensor, np.ndarray],
training: bool = True) -> Tensor:
"""Run a forward step on a given model.
This method can be used with TensorFlow models:
```python
m = fe.architecture.tensorflow.LeNet(classes=2)
x = tf.ones((3,28,28,1)) # (batch, height, width, channels)
b = fe.backend.feed_forward(m, x) # [[~0.5, ~0.5], [~0.5, ~0.5], [~0.5, ~0.5]]
```
This method can be used with PyTorch models:
```python
m = fe.architecture.pytorch.LeNet(classes=2)
x = torch.ones((3,1,28,28)) # (batch, channels, height, width)
b = fe.backend.feed_forward(m, x) # [[~0.5, ~0.5], [~0.5, ~0.5], [~0.5, ~0.5]]
```
Args:
model: A neural network to run the forward step through.
x: An input tensor for the `model`. This value will be auto-cast to either a tf.Tensor or torch.Tensor as
applicable for the `model`.
training: Whether this forward step is part of training or not. This may impact the behavior of `model` layers
such as dropout.
Returns:
The result of `model(x)`.
Raises:
ValueError: If `model` is an unacceptable data type.
"""
if isinstance(model, tf.keras.Model):
if not tf.is_tensor(x):
x = to_tensor(x, "tf")
x = model(x, training=training)
elif isinstance(model, torch.nn.Module):
model.train(mode=training)
if not isinstance(x, torch.Tensor):
x = to_tensor(x, "torch")
x = model(x)
else:
raise ValueError("Unrecognized model instance {}".format(type(model)))
return x
def _configure_tensor(self, loader: Union[DataLoader, tf.data.Dataset], batch: Dict[str, Any]) -> Dict[str, Any]:
"""A function to convert a batch of tf.Tensors to torch.Tensors if required.
Returns:
Either the original `batch`, or the `batch` converted to torch.Tensors if required.
"""
# TODO - if user has torch loader but custom collate that doesn't return torch tensor, need to cast here
if isinstance(loader, tf.data.Dataset) and isinstance(self.network, TorchNetwork):
batch = to_tensor(batch, target_type="torch")
return batch
def gather_from_batch(tensor: Tensor, indices: Tensor) -> Tensor:
"""Gather specific indices from a batch of data.
This method can be useful if you need to compute gradients based on a specific subset of a tensor's output values.
The `indices` will automatically be cast to the correct type (tf, torch, np) based on the type of the `tensor`.
This method can be used with Numpy data:
```python
ind = np.array([1, 0, 1])
n = np.array([[0, 1], [2, 3], [4, 5]])
b = fe.backend.gather_from_batch(n, ind) # [1, 2, 5]
n = np.array([[[0, 1], [2, 3]], [[4, 5], [6, 7]], [[8, 9], [10, 11]]])
b = fe.backend.gather_from_batch(n, ind) # [[2, 3], [4, 5], [10, 11]]
```
This method can be used with TensorFlow tensors:
```python
ind = tf.constant([1, 0, 1])
t = tf.constant([[0, 1], [2, 3], [4, 5]])
b = fe.backend.gather_from_batch(t, ind) # [1, 2, 5]
t = tf.constant([[[0, 1], [2, 3]], [[4, 5], [6, 7]], [[8, 9], [10, 11]]])
b = fe.backend.gather_from_batch(t, ind) # [[2, 3], [4, 5], [10, 11]]
```
This method can be used with PyTorch tensors:
```python
ind = torch.tensor([1, 0, 1])
p = torch.tensor([[0, 1], [2, 3], [4, 5]])
b = fe.backend.gather_from_batch(p, ind) # [1, 2, 5]
p = torch.tensor([[[0, 1], [2, 3]], [[4, 5], [6, 7]], [[8, 9], [10, 11]]])
b = fe.backend.gather_from_batch(p, ind) # [[2, 3], [4, 5], [10, 11]]
```
Args:
tensor: A tensor of shape (batch, d1, ..., dn).
indices: A tensor of shape (batch, ) or (batch, 1) indicating which indices should be selected.
Returns:
A tensor of shape (batch, d2, ..., dn) containing the elements from `tensor` at the given `indices`.
Raises:
ValueError: If `tensor` is an unacceptable data type.
"""
if tf.is_tensor(tensor):
indices = to_tensor(indices, 'tf')
indices = tf.cast(indices, tf.int64)
if len(indices.shape) == 1: # Indices not batched
indices = expand_dims(indices, 1)
return tf.gather_nd(tensor, indices=indices, batch_dims=1)
elif isinstance(tensor, torch.Tensor):
return tensor[torch.arange(tensor.shape[0]), squeeze(indices)]
elif isinstance(tensor, np.ndarray):
return tensor[np.arange(tensor.shape[0]), squeeze(indices)]
else:
raise ValueError("Unrecognized tensor type {}".format(type(tensor)))
def gather(tensor: Tensor, indices: Tensor) -> Tensor:
"""Gather specific indices from a tensor.
The `indices` will automatically be cast to the correct type (tf, torch, np) based on the type of the `tensor`.
This method can be used with Numpy data:
```python
ind = np.array([1, 0, 1])
n = np.array([[0, 1], [2, 3], [4, 5]])
b = fe.backend.gather(n, ind) # [[2, 3], [0, 1], [2, 3]]
n = np.array([[[0, 1], [2, 3]], [[4, 5], [6, 7]], [[8, 9], [10, 11]]])
b = fe.backend.gather(n, ind) # [[[4, 5], [6, 7]], [[0, 1], [2, 3]], [[4, 5], [6, 7]]]
```
This method can be used with TensorFlow tensors:
```python
ind = tf.constant([1, 0, 1])
t = tf.constant([[0, 1], [2, 3], [4, 5]])
b = fe.backend.gather(t, ind) # [[2, 3], [0, 1], [2, 3]]
t = tf.constant([[[0, 1], [2, 3]], [[4, 5], [6, 7]], [[8, 9], [10, 11]]])
b = fe.backend.gather(t, ind) # [[[4, 5], [6, 7]], [[0, 1], [2, 3]], [[4, 5], [6, 7]]]
```
This method can be used with PyTorch tensors:
```python
ind = torch.tensor([1, 0, 1])
p = torch.tensor([[0, 1], [2, 3], [4, 5]])
b = fe.backend.gather(p, ind) # [[2, 3], [0, 1], [2, 3]]
p = torch.tensor([[[0, 1], [2, 3]], [[4, 5], [6, 7]], [[8, 9], [10, 11]]])
b = fe.backend.gather(p, ind) # [[[4, 5], [6, 7]], [[0, 1], [2, 3]], [[4, 5], [6, 7]]]
```
Args:
tensor: A tensor to gather values from.
indices: A tensor indicating which indices should be selected. These represent locations along the 0 axis.
Returns:
A tensor containing the elements from `tensor` at the given `indices`.
Raises:
ValueError: If `tensor` is an unacceptable data type.
"""
if tf.is_tensor(tensor):
indices = to_tensor(indices, 'tf')
indices = tf.cast(indices, tf.int64)
return tf.gather(tensor, indices=squeeze(indices), axis=0)
elif isinstance(tensor, torch.Tensor):
return tensor[squeeze(indices).type(torch.int64)]
elif isinstance(tensor, np.ndarray):
return np.take(tensor, squeeze(indices).astype('int64'), axis=0)
else:
raise ValueError("Unrecognized tensor type {}".format(type(tensor)))
def transform(self, data: Dict[str, Any], mode: str, epoch: int = 1) -> Dict[str, Any]:
"""Run a forward step through the Network on an element of data.
Args:
data: The element to data to use as input.
mode: The mode in which to run the transform. One of 'train', 'eval', 'test', or 'infer'.
epoch: The epoch in which to run the transform.
Returns:
prediction_data overlaid on the input `data`.
"""
self.load_epoch(mode, epoch, warmup=False, eager=True)
data = to_tensor(data, target_type=self.target_type)
data, prediction = self.run_step(data)
self.unload_epoch()
return {**data, **prediction}
def transform(self, data: Dict[str, Any], mode: str, epoch: int = 1) -> Dict[str, Any]:
"""Run a forward step through the Network on an element of data.
Args:
data: The element to data to use as input.
mode: The mode in which to run the transform. One of 'train', 'eval', 'test', or 'infer'.
epoch: The epoch in which to run the transform.
Returns:
(batch_data, prediction_data)
"""
self.load_epoch(mode, epoch, warmup=False)
data = to_tensor(data, "torch")
data, prediction = self.run_step(data)
self.unload_epoch()
data.update(prediction)
return data
def transform(self, data: Dict[str, Any], mode: str, epoch: int = 1) -> Dict[str, Any]:
"""Run a forward step through the Network on an element of data.
Args:
data: The element to data to use as input.
mode: The mode in which to run the transform. One of 'train', 'eval', 'test', or 'infer'.
epoch: The epoch in which to run the transform.
Returns:
(batch_data, prediction_data)
"""
self.load_epoch(mode, epoch, warmup=False)
data = to_tensor(data, target_type="tf")
data, prediction = self.run_step(data)
self.unload_epoch()
# handle tensorflow multi-gpu inferencing issue, it will replicate data on each device
if isinstance(tf.distribute.get_strategy(), tf.distribute.MirroredStrategy):
prediction = self._subsample_data(prediction, get_batch_size(data))
data.update(prediction)
return data
def benchmark(self,
mode: str = "train",
epoch: int = 1,
ds_id: Optional[str] = None,
num_steps: int = 1000,
log_interval: int = 100,
detailed: bool = True) -> None:
"""Benchmark the pipeline processing speed.
Args:
mode: The execution mode to benchmark. This can be 'train', 'eval' or 'test'.
epoch: The epoch index to benchmark. Note that epoch indices are 1-indexed.
ds_id: The ds_id to benchmark. If None, all ds_ids will be benchmarked.
num_steps: The number of steps over which to perform the benchmark.
log_interval: The logging interval.
detailed: Whether to display the detailed time used by each operator.
"""
if ds_id is None:
ds_ids = self.get_ds_ids(epoch=epoch, mode=mode)
else:
ds_ids = [ds_id]
for ds_id in ds_ids:
with self(mode=mode,
epoch=epoch,
ds_id=ds_id,
steps_per_epoch=num_steps) as loader:
if isinstance(loader, tf.data.Dataset):
loader = loader.take(num_steps)
start = time.perf_counter()
for idx, _ in enumerate(loader, start=1):
if idx % log_interval == 0:
duration = time.perf_counter() - start
iters_per_sec = log_interval / duration
ds_str = f"Dataset: {ds_id}, " if ds_id else ""
print(
"FastEstimator-Benchmark ({}): {}Step: {}, Epoch: {}, Steps/sec: {}"
.format(mode.capitalize(), ds_str, idx, epoch,
iters_per_sec))
start = time.perf_counter()
# Pipeline Operations Benchmarking when using FEDataset
if isinstance(loader, FEDataLoader) and isinstance(
loader.dataset, OpDataset) and detailed:
# (n_visited, duration)
duration_list = np.zeros(shape=(len(self.ctx_ops) + 1 +
len(self.ctx_batch_ops),
2))
data_len = len(loader.dataset)
ds_str = f", Dataset: {ds_id}" if ds_id else ""
print(
"\nBreakdown of time taken by Pipeline Operations (Mode: {}, Epoch: {}{})\n"
.format(mode.capitalize(), epoch, ds_str))
extra_memory_management_time = 0
for _ in range(log_interval):
filtered = False
batch = []
index = np.random.randint(data_len)
items = deepcopy(loader.dataset.dataset[index])
if isinstance(items, list):
while not batch:
filtered = False
# BatchDataset may randomly sample the same elements multiple times, avoid reprocessing
unique_samples = set()
for item in items:
if id(item) not in unique_samples:
for i, op in enumerate(self.ctx_ops):
start = time.perf_counter()
op_data = forward_numpyop(
[op], item,
{'mode': loader.dataset.mode})
duration = time.perf_counter(
) - start
duration_list[i][0] += 1
duration_list[i][1] += duration
if isinstance(
op_data, FilteredData):
filtered = True
break
unique_samples.add(id(item))
if not filtered:
batch = items
else:
while len(batch) < (self.ctx_batch_size or 1):
filtered = False
for i, op in enumerate(self.ctx_ops):
start = time.perf_counter()
op_data = forward_numpyop([op], items,
{'mode': mode})
duration = time.perf_counter() - start
duration_list[i][0] += 1
duration_list[i][1] += duration
if isinstance(op_data, FilteredData):
filtered = True
break
if not filtered:
batch.append(items)
index = np.random.randint(data_len)
items = deepcopy(loader.dataset.dataset[index])
if not filtered:
# Perform the batching
start = time.perf_counter()
batch = self.ctx_batch_info.collate_fn(batch)
duration = time.perf_counter() - start
duration_list[len(self.ctx_ops)][0] += 1
duration_list[len(self.ctx_ops)][1] += duration
# Perform batch ops
start = time.perf_counter()
# Transform to numpy to not bias against the first op in the batch_op chain
batch = to_tensor(batch, target_type='np')
extra_memory_management_time += time.perf_counter(
) - start
for i, op in enumerate(self.ctx_batch_ops,
start=len(self.ctx_ops) +
1):
start = time.perf_counter()
op_data = forward_numpyop([op],
data=batch,
state={'mode': mode},
batched='np')
duration = time.perf_counter() - start
duration_list[i][0] += 1
duration_list[i][1] += duration
if isinstance(op_data, FilteredData):
break
# Count extra time needed to cast data back to torch
start = time.perf_counter()
to_tensor(batch,
target_type='torch',
shared_memory=True)
extra_memory_management_time += time.perf_counter(
) - start
if self.ctx_batch_ops:
# Extra memory management penalty is only incurred when using batch ops
duration_list[len(
self.ctx_ops)][1] += extra_memory_management_time
total_time = np.sum(duration_list[:, 1])
normalized_times_ms = 1000 * duration_list[:, 1] / np.maximum(
duration_list[:, 0], 1)
op_names = ["Op"]
for op in self.ctx_ops + [self.ctx_batch_info
] + self.ctx_batch_ops:
if isinstance(op, Sometimes) and op.op:
op_names.append(op.__class__.__name__ + " (" +
op.op.__class__.__name__ + ")")
elif isinstance(op, Repeat) and op.op:
op_names.append(op.__class__.__name__ + " (" +
op.op.__class__.__name__ + ")")
elif isinstance(op, OneOf) and op.ops:
op_names.append(op.__class__.__name__ + " (" +
", ".join([
sub_op.__class__.__name__
for sub_op in op.ops
]) + ")")
elif isinstance(op, Fuse) and op.ops:
op_names.append(op.__class__.__name__ + " (" +
", ".join([
sub_op.__class__.__name__
for sub_op in op.ops
]) + ")")
elif isinstance(op, Batch):
op_names.append("<Collating Batch>")
else:
op_names.append(op.__class__.__name__)
max_op_len = max(len(op_name) for op_name in op_names)
max_in_len = max([
len(", ".join(op.inputs)) for op in self.ctx_ops +
[self.ctx_batch_info] + self.ctx_batch_ops
] + [len("Inputs")])
max_out_len = max([
len(", ".join(op.outputs)) for op in self.ctx_ops +
[self.ctx_batch_info] + self.ctx_batch_ops
] + [len("Outputs")])
ms_visit_len = max(
len("{:.3f}".format(max(normalized_times_ms))),
len("ms / Visit"))
visit_len = max(len(f"{int(np.max(duration_list[:, 0]))}"),
len("Visits"))
print("{}: {}: {}: {}: {}: {}".format(
"Op".ljust(max_op_len + 1),
"Inputs".ljust(max_in_len + 1),
"Outputs".ljust(max_out_len + 1),
"ms / Visit".ljust(ms_visit_len + 1),
"Visits".ljust(visit_len + 1),
"Time (Total)".rjust(12)))
print("-" * (max_op_len + max_in_len + max_out_len +
visit_len + 37))
for i, op in enumerate(self.ctx_ops +
[self.ctx_batch_info] +
self.ctx_batch_ops):
print("{}: {}: {}: {}: {}: {:11.2f}%".format(
op_names[i + 1].ljust(max_op_len + 1),
", ".join(op.inputs).ljust(max_in_len + 1),
", ".join(op.outputs).ljust(max_out_len + 1),
"{:.3f}".format(
normalized_times_ms[i]).ljust(ms_visit_len +
1),
str(int(duration_list[i][0])).ljust(visit_len + 1),
100 * duration_list[i][1] / total_time))
if self.ctx_batch_ops:
penalty = round(
100 * (duration_list[len(self.ctx_ops)][1] -
extra_memory_management_time) /
duration_list[len(self.ctx_ops)][1], 1)
print(
f"\nNote that collation time would be cut by ~{penalty}% if there were no batched ops."
)
print("\n") # to make printing more obvious
def iwd(tensor: Tensor,
power: float = 1.0,
max_prob: float = 0.95,
pairwise_distance: float = 1.0,
eps: Optional[Tensor] = None) -> Tensor:
"""Compute the Inverse Weighted Distance from the given input.
This can be used as an activation function for the final layer of a neural network instead of softmax. For example,
instead of: model.add(layers.Dense(classes, activation='softmax')), you could use:
model.add(layers.Dense(classes, activation=lambda x: iwd(tf.nn.sigmoid(x))))
This method can be used with Numpy data:
```python
n = np.array([[0.5]*5, [0]+[1]*4])
b = fe.backend.iwd(n) # [[0.2, 0.2, 0.2, 0.2, 0.2], [0.95, 0.0125, 0.0125, 0.0125, 0.0125]]
```
This method can be used with TensorFlow tensors:
```python
t = tf.constant([[0.5]*5, [0]+[1]*4])
b = fe.backend.iwd(n) # [[0.2, 0.2, 0.2, 0.2, 0.2], [0.95, 0.0125, 0.0125, 0.0125, 0.0125]]
```
This method can be used with PyTorch tensors:
```python
p = torch.tensor([[0.5]*5, [0]+[1]*4])
b = fe.backend.iwd(n) # [[0.2, 0.2, 0.2, 0.2, 0.2], [0.95, 0.0125, 0.0125, 0.0125, 0.0125]]
```
Args:
tensor: The input value. Should be of shape (Batch, C) where every element in C corresponds to a (non-negative)
distance to a target class.
power: The power to raise the inverse distances to. 1.0 results in a fairly intuitive probability output. Larger
powers can widen regions of certainty, whereas values between 0 and 1 can widen regions of uncertainty.
max_prob: The maximum probability to assign to a class estimate when it is distance zero away from the target.
For numerical stability this must be less than 1.0. We have found that using smaller values like 0.95 can
lead to natural adversarial robustness.
pairwise_distance: The distance to any other class when the distance to a target class is zero. For example, if
you have a perfect match for class 'a', what distance should be reported to class 'b'. If you have a metric
where this isn't constant, just use an approximate expected distance. In that case `max_prob` will only give
you approximate control over the true maximum probability.
eps: The numeric stability constant to be used when d approaches zero. If None then it will be computed using
`max_prob` and `pairwise_distance`. If not None, then `max_prob` and `pairwise_distance` will be ignored.
Returns:
A probability distribution of shape (Batch, C) where smaller distances from `tensor` correspond to larger
probabilities.
"""
if eps is None:
eps = np.array(pairwise_distance * math.pow(
(1.0 - max_prob) / (max_prob * (tensor.shape[-1] - 1)), 1 / power),
dtype=TENSOR_TO_NP_DTYPE[tensor.dtype])
eps = to_tensor(
eps,
target_type='torch' if isinstance(tensor, torch.Tensor) else
'tf' if tf.is_tensor(tensor) else 'np')
if isinstance(eps, torch.Tensor):
eps = eps.to("cuda:0" if torch.cuda.is_available() else "cpu")
tensor = maximum(tensor, eps)
tensor = tensor_pow(1.0 / tensor, power)
tensor = tensor / reshape(reduce_sum(tensor, axis=-1), shape=[-1, 1])
return tensor
