def __init__(self, filepath, transform=None):
data = None
labels = []
# Open the training data
if filepath[-1] == "_":
for idx in range(1, 6):
filename = filepath + str(idx)
with open(filename, 'rb') as f:
data_dict = pickle.load(f, encoding='latin-1')
if idx == 1:
data = data_dict['data']
else:
data = np.vstack((data, data_dict['data']))
labels.extend(data_dict['labels'])
data = data.reshape(-1, 3, 32, 32).astype("float")
data = data.transpose((0, 2, 3, 1)) # convert to HWC
labels = np.array(labels)
# Open the testing data or one training batch
else:
with open(filepath, 'rb') as f:
test_data_dict = pickle.load(f, encoding='latin-1')
data = test_data_dict['data']
data = data.reshape(data.shape[0], 3, 32, 32).astype("float")
data = data.transpose((0, 2, 3, 1)) # convert to HWC
labels = np.array(test_data_dict['labels'])
if transform is not None:
data = transform(data)
self.data = nets.Tensor(data)
self.labels = nets.Tensor(labels)
def test_ops(self):
tensor1 = nets.Tensor([[2, 4], [6, 8]])
tensor2 = nets.Tensor([[1, 3], [5, 7]])
tensor1 + tensor2
tensor1 += 1
tensor1 * tensor2
tensor1.T
def __init__(self, path_data, path_label, transform=None):
data = self._load_mnist(path_data, header_size=16).reshape(
(-1, 28, 28))
if transform is not None:
data = transform(data)
self.data = nets.Tensor(data)
self.labels = nets.Tensor(self._load_mnist(path_label, header_size=8))
def test_init(self):
# Instantiate a tensor
tensor = nets.Tensor(1)
tensor = nets.Tensor(tensor)
tensor = nets.Tensor(np.eye(3))
# Check the data
data = np.ones((100, 100))
tensor = nets.Tensor(data)
assert np.array_equal(tensor.data, data)
tensor = nets.Tensor(data, requires_grad=True)
assert tensor.requires_grad == True
assert np.array_equal(tensor.grad.data, np.zeros_like(data))
def argmax(t, axis=None):
r"""Get the indices of maximum elements from a ``Tensor``.
Args:
t (Tensor): tensor get maximum indices from
axis (int, optional): index of the axis. Default is ``None``.
Returns:
Tensor
"""
t = nets.to_tensor(t)
if axis is None:
return nets.Tensor(np.unravel_index(np.argmax(t.data), t.shape))
else:
return nets.Tensor(np.argmax(t.data, axis=axis))
def transpose(t, indices=None):
r"""Permutation a tensor object.
Args:
t (Tensor):
indices (tuple, optional): index to transpose.
Returns:
Tensor
"""
t = nets.to_tensor(t)
if indices is None:
indices = tuple(range(t.ndim - 1, -1, -1))
data = t.data.transpose(indices)
requires_grad = t.requires_grad
hooks = []
if requires_grad:
def grad_fn(grad):
indices_back = tuple(inv_permutation(indices))
grad = grad.transpose(indices_back)
return grad
hooks.append(Hook(t, grad_fn))
return nets.Tensor(data, requires_grad, hooks)
def forward(self, tensor1, tensor2):
scalars_to_device(self.cond, tensor1, tensor2)
nc = numpy_or_cupy(tensor1, tensor2)
data = nc.where(self.cond.data, tensor1.data, tensor2.data)
requires_grad = tensor1.requires_grad or tensor2.requires_grad
device = tensor1.device
return nets.Tensor(data, requires_grad=requires_grad, device=device)
def pow(t, power):
r"""Power a tensor-like object.
.. math::
T_{out} = T^2
Args:
t (Tensor like): reference tensor
power (int): power to elevate a tensor
Returns:
Tensor
"""
assert type(
power
) == int, "unsupported type {} for power. Currently supported type: int".format(
type(power))
t = nets.to_tensor(t)
data = t.data**power
requires_grad = t.requires_grad
hooks = []
# Update the gradient
if requires_grad:
hooks.append(Hook(t, lambda grad: grad * power * t.data**(power - 1)))
return nets.Tensor(data, requires_grad, hooks)
def dot(t1, t2):
r"""Dot product of two matrices.
.. math::
T_{out} = (t_{i, j}^{[out]})_{i, j} \quad where \quad t_{i, j}^{[out]} = \sum_{k=1}^{n} t_{i, k}^{[1]} \times
t_{k, j}^{[2]}
Args:
t1 (Tensor like)
t2 (Tensor like)
Returns:
Tensor
"""
data = t1.data @ t2.data
requires_grad = t1.requires_grad or t2.requires_grad
hooks = []
if t1.requires_grad:
def grad_fn1(grad):
return grad @ t2.data.T
hooks.append(Hook(t1, grad_fn1))
if t2.requires_grad:
def grad_fn2(grad):
return t1.data.T @ grad
hooks.append(Hook(t2, grad_fn2))
return nets.Tensor(data, requires_grad, hooks)
def tanh(t):
# type: (Tensor) -> Tensor
r"""``tanh`` standard function, definead as:
.. math::
\text{tanh}(T) = \frac{e^{T} - e^{-T}}{e^{T} + e^{-T}}
Shape:
- input: x (numpy.array): input to compute the ``tanh`` function on.
- output: y (numpy.array): ``tanh`` output, with the same shape than :math:`T`.
.. image:: images/functional_tanh.png
Examples::
>>> import nets
>>> in_array = np.array([-5, 2, 6, -2, 4])
>>> out_array = nets.tanh(in_array)
See :class:`~nets.nn.activation.Tanh` for the activation implementation.
"""
data = np.tanh(t.data)
requires_grad = t.requires_grad
hooks = []
if requires_grad:
hooks.append(Hook(t, lambda grad: grad * (1 - data * data)))
return nets.Tensor(data, requires_grad, hooks)
def dropout(t, prob=0.5):
r"""Zeros elements from a ``Tensor`` with a probability ``prob``.
.. math::
\text{dropout}(T) = T \times Z \quad \text{where} Z = (z_{i})_{i} \quad and z_i =
\begin{cases}
1, &\quad p \ge prob \\
0, &\quad else.
\end{cases}
Args:
t (Tensor): tensor to zeros
prob (float [0, 1]): probability to zero an element
Returns:
Tensor: input tensor with some zeros
"""
# Randomly generates number following a uniform distribution between [0, 1]
probabilities = np.random.uniform(low=0.0, high=1.0, size=t.shape)
# Generate a mask of (0, 1). 0 means probabilities[index] > prob, 1 else.
mask = np.where(probabilities > prob, 0, 1)
mask = nets.Tensor(mask)
# Applies the mask to the tensor to get the dropout (elementwise multiplication)
t_drop = t * mask
return t_drop
def slice(t, indices):
r"""Slice a tensor from given indices.
Args:
t (Tensor): tensor to slice
idxs (tuple, int, :): indices to extract data
Returns:
Tensor
"""
t = nets.to_tensor(t)
if isinstance(indices, nets.Tensor):
indices = indices.data
data = t.data[indices]
requires_grad = t.requires_grad
hooks = []
if requires_grad:
def grad_fn(grad):
bigger_grad = np.zeros_like(t.data)
if grad.shape != bigger_grad.shape:
bigger_grad[indices] = grad
else:
bigger_grad = grad
return bigger_grad
hooks.append(Hook(t, grad_fn))
return nets.Tensor(data, requires_grad, hooks)
def max(t, axis=None):
r"""Get the maximum from a ``Tensor``.
Args:
t (Tensor): tensor to transform
axis (int, optional): index of the axis to search. Default is ``None``.
Returns:
Tensor
"""
t = nets.to_tensor(t)
data = np.max(t.data, axis=axis)
requires_grad = t.requires_grad
hooks = []
if requires_grad:
def grad_fn(grad):
bigger_grad = np.zeros_like(t.data)
if axis is None:
# If there is no axis, the argmax is the location of he maximum single element
max_indices = np.unravel_index(np.argmax(t.data), t.shape)
bigger_grad[max_indices] = grad
else:
# If there is an axis, we reconstruct the bigger matrix by 'rolling' on this axis
max_indices = np.argmax(t.data, axis=axis)
for i, roll in enumerate(np.rollaxis(bigger_grad, axis)):
roll += (max_indices == i).astype(int) * grad
return bigger_grad
hooks.append(Hook(t, grad_fn))
return nets.Tensor(data, requires_grad, hooks)
def concatenate(iterable):
r"""Concatenate multiples ``Tensor`` from an iterable.
.. note::
The ``Tensor`` in ``iterable`` should and must have the same shape.
Args:
iterable (tuple, list): list containing ``Tensor`` to concatenate.
Returns:
Tensor: the concatenation of all ``Tensor``.
"""
assert isinstance(iterable, ITERABLE), f'iterable type {type(iterable)} unsupported for `concatenate` function.' \
f'Types currently supported are list, tuple.'
requires_grad = False
hooks = []
data = np.array([])
for idx, t in enumerate(iterable):
t = nets.to_tensor(t)
requires_grad = t.requires_grad or requires_grad
if data.size == 0:
data = t.data
else:
data = np.concatenate((data, t.data))
if t.requires_grad:
def grad_fn(grad):
return grad[idx:idx + t.shape[0]]
hooks.append(Hook(t, grad_fn))
return nets.Tensor(data, requires_grad, hooks)
def inverse(t):
r"""Inverse a tensor-like object.
.. math::
T_{out} = \frac{1}{T}
Args:
t (Tensor like): tensor to inverse.
Returns:
Tensor
"""
t = nets.to_tensor(t)
requires_grad = t.requires_grad
hooks = []
if requires_grad:
def grad_fn(grad):
r"""Update the gradient for the inverse operation, :math:`grad = grad \times \frac{-1}{T^2}`.
Shape:
- inputs (np.ndarray): upstream gradient.
- outputs (np.ndarray): downstream gradient.
"""
return -1 / (t.data**2) * grad
hooks.append(Hook(t, grad_fn))
return nets.Tensor(1 / t.data, requires_grad, hooks)
def forward(self, predictions, labels):
assert labels.dtype == int, "unsupported labels type {} for cross entropy loss".format(
predictions.dtype)
batch_size, _ = predictions.shape
predictions = nets.softmax(predictions, axis=1)
cost = nets.Tensor(-1 / batch_size,
device=predictions.device) * nets.sum(
nets.log(predictions) * labels)
return cost
def astype(t, new_type):
"""Create a range of values.
Args:
new_type (str): new type of the data.
Returns:
Tensor
"""
data = t.data.astype(new_type)
return nets.Tensor(data, requires_grad=t.requires_grad, device=t.device)
def forward(self, inputs):
r"""
Computes the forward pass of a vanilla RNN.
.. math::
\begin{align*}
h_{0} &= 0 \\
h_t &= \text{tanh}(x \cdot W_{ih} + h_{t-1} \cdot W_{hh} + b_{h}) \\
y &= h_t \cdot W_{ho} + b_{o} \\
\end{align*}
Args:
inputs (Tensor): sequence of inputs to be processed
Returns:
outputs (Tensor): predictions :math:`y`.
hidden_states (Tensor): concatenation of all hidden states :math:`h_t`.
"""
hidden_states = nets.Tensor([self.weight_h0])
outputs = nets.Tensor([])
# Initialize hidden_cell_0 (with zeros)
hidden_state = hidden_states[0]
# For each element in input sequence
for t in range(inputs.shape[0]):
# Compute new hidden state
hidden_state = nets.tanh(
nets.dot(inputs[t], self.weight_ih) +
nets.dot(hidden_state, self.weight_hh) + self.bias_h)
# Compute output
out = nets.sigmoid(
nets.dot(hidden_state, self.weight_ho) + self.bias_o)
# Save results and continue
outputs = nets.append(outputs, out)
hidden_states = nets.append(hidden_states, hidden_state)
# Save in the cache (for manual back-propagation)
self._cache['hidden_states'] = hidden_states
return outputs, hidden_states
def ones(shape, requires_grad=False, device='cpu', **kwargs):
"""Create a ones tensor of a given shape.
Args:
shape (tuple): shape of the 0-tensor.
requires_grad (bool): if ``True`` will track gradients.
device (str): name of the device where the tensor is located. Default to ``'cpu'``.
Returns:
Tensor
"""
if device == 'cpu':
data = np.ones(shape, **kwargs)
else:
data = cp.ones(shape, **kwargs)
return nets.Tensor(data, requires_grad=requires_grad, device=device)
def arange(*args, requires_grad=False, device='cpu', **kwargs):
"""Create a range of values.
Args:
requires_grad (bool): if ``True`` will track gradients.
device (str): name of the device where the tensor is located. Default to ``'cpu'``.
Returns:
Tensor
"""
if device == 'cpu':
data = np.arange(*args, **kwargs)
else:
data = cp.arange(*args, **kwargs)
return nets.Tensor(data, requires_grad=requires_grad, device=device)
def eye(size, requires_grad=False, device='cpu', **kwargs):
"""Create an eye matrix.
Args:
size (int): size of the matrix.
requires_grad (bool): if ``True`` will track gradients.
device (str): name of the device where the tensor is located. Default to ``'cpu'``.
Returns:
Tensor
"""
if device == 'cpu':
data = np.eye(size, **kwargs)
else:
data = cp.eye(size, **kwargs)
return nets.Tensor(data, requires_grad=requires_grad, device=device)
def eq(t, other):
r"""Return a boolean tensor for *equal* condition.
.. math::
\text{gt}_{\text{other}}(t) = t == other
Args:
t (Tensor): tensor to compare
other (Tensor like): object to compare the tensor
Returns:
Tensor
"""
t = nets.to_tensor(t)
other = nets.to_tensor(other)
cond = t.data == other.data
return nets.Tensor(cond, device=t.device)
def argmax(t, axis=None):
r"""Get the indices of maximum elements from a tensor.
Args:
t (Tensor): tensor get maximum indices from
axis (int, optional): index of the axis. Default is ``None``.
Returns:
Tensor
"""
t = nets.to_tensor(t)
nc = numpy_or_cupy(t)
if axis is None:
data = nc.unravel_index(nc.argmax(t.data), t.shape)
else:
data = nc.argmax(t.data, axis=axis)
return nets.Tensor(data, device=t.device)
def lt(t, other):
r"""Return a boolean tensor for *lower than* condition.
.. math::
\text{gt}_{\text{other}}(t) = t < other
Args:
t (Tensor): tensor to compare
other (Tensor like): object to compare the tensor
Returns:
Tensor
"""
t = nets.to_tensor(t)
other = nets.to_tensor(other)
data = t.data < other.data
return nets.Tensor(data, device=t.device)
def ge(t, other):
r"""Return a boolean tensor for *greater or equal* condition.
.. math::
\text{gt}_{\text{other}}(t) = t \ge other
Args:
t (Tensor): tensor to compare
other (Tensor like): object to compare the tensor
Returns:
Tensor
"""
t = nets.to_tensor(t)
other = nets.to_tensor(other)
data = t.data >= other.data
return nets.Tensor(data, device=t.device)
def rollaxis(t, axis, start=0):
"""Roll the specified axis backwards, until it lies in a given position.
Args:
t (Tensor): Input tensor.
axis (int): The axis to be rolled.
The positions of the other axes do not change relative to one another.
start (int, optional): When ``start <= axis``, the axis is rolled back until it lies in this position.
When ``start > axis``, the axis is rolled until it lies before this position.
The default, 0, results in a "complete" roll.
Returns:
Tensor
"""
nc = numpy_or_cupy(t)
data = nc.rollaxis(t.data, axis, start=start)
return nets.Tensor(data, requires_grad=t.requires_grad, device=t.device)
def reshape(t, shape):
r"""Reshape a ``Tensor``.
Args:
t (Tensor): tensor to transform
shape (tuple): new shape of ``t``
Returns:
Tensor
"""
t = nets.to_tensor(t)
data = t.data.reshape(shape)
requires_grad = t.requires_grad
hooks = []
if requires_grad:
hooks.append(Hook(t, lambda grad: grad.reshape(t.shape)))
return nets.Tensor(data, requires_grad, hooks)
def pad(t, padding, constant_values=0):
r"""Reshape a ``Tensor`` to a bigger size and add a ``padding`` on the side, with a ``0`` constant value.
Args:
t (Tensor): tensor to transform
padding (tuple): padding dimensions
constant_values (scalar, optional): scalar affected in the padding
Returns:
Tensor
"""
t = nets.to_tensor(t)
data = np.pad(t.data, pad_width=padding, constant_values=constant_values)
requires_grad = t.requires_grad
hooks = []
if requires_grad:
hooks.append(Hook(t, lambda grad: numpy_unpad(grad, padding)))
return nets.Tensor(data, requires_grad, hooks)
def where(cond, t1, t2):
r"""Transformation regarding a condition.
.. math::
T_{out} =
\begin{cases}
T_1, &\quad if \quad condition \\
T_2, &\quad else.
\end{cases}
Args:
cond (bool): condition to merge two tensors
t1 (Tensor): input tensor to merge
t2 (Tensor): input tensor to merge
Returns:
Tensor
"""
t1 = nets.to_tensor(t1)
t2 = nets.to_tensor(t2)
# TODO: handle broadcasting with where(): sum across the broadcast dimension
assert t1.shape == t2.shape, f"tensors should have the same shape. Got t1.shape={t1.shape}, t2.shape={t2.shape}"
cond = nets.to_array(cond)
data = np.where(cond, t1.data, t2.data)
requires_grad = t1.requires_grad or t2.requires_grad
hooks = []
if t1.requires_grad:
def grad_fn(grad):
return grad * np.where(cond, 1, 0)
hooks.append(Hook(t1, grad_fn))
if t2.requires_grad:
def grad_fn(grad):
return grad * np.where(cond, 0, 1)
hooks.append(Hook(t2, grad_fn))
return nets.Tensor(data, requires_grad, hooks)
def append(t, value):
r"""Append multiples ``Tensor`` from an iterable.
.. note::
The ``Tensor`` in ``iterable`` should and must have the same shape.
Args:
t (Tensor): list containing ``Tensor`` to concatenate.
Returns:
Tensor: the concatenation of all ``Tensor``.
"""
t = nets.to_tensor(t)
value = nets.to_tensor(value)
requires_grad = False
hooks = []
requires_grad = t.requires_grad or value.requires_grad
if t.size == 0:
data = [value.data]
elif value.size == 0:
data = [t.data]
else:
data = t.data.tolist()
data.append(value.data)
if t.requires_grad:
def grad_fn(grad):
return grad[:-1]
hooks.append(Hook(t, grad_fn))
if value.requires_grad:
def grad_fn(grad):
return grad[-1]
hooks.append(Hook(value, grad_fn))
return nets.Tensor(data, requires_grad, hooks)
