def setup(self):
size = self.x.array.size
newshape_size = np.prod(self.newshape)
self.shape = tuple(d if d != -1 else -size // newshape_size
for d in self.newshape)
self.array = ca.zeros(self.shape)
self.grad_array = ca.zeros(self.shape)
def setup(self):
a_shp = self.a.shape
b_shp = self.b.shape
concat_size = a_shp[self.axis] + b_shp[self.axis]
self.a_size = a_shp[self.axis]
self.shape = (a_shp[:self.axis] + (concat_size, ) +
a_shp[self.axis + 1:])
self.array = ca.zeros(self.shape)
self.grad_array = ca.zeros(self.shape)
def setup(self):
shape = self.inputs[0].shape
for expr in self.inputs:
if shape != expr.shape:
raise ValueError('shape mismatch: %s and %s' %
(shape, expr.shape))
self.shape = (self.n_sources, ) + shape
self.array = ca.zeros(self.shape)
self.grad_array = ca.zeros(self.shape)
def setup(self):
try:
# XXX: don't be lazy
self.shape = ca.dot(self.lhs.array, self.rhs.array).shape
except ValueError:
raise ValueError('Shape mismatch: %s and %s for %s. LHS: %s RHS: '
'%s.' % (self.lhs.shape, self.rhs.shape,
self, self.lhs, self.rhs))
self.array = ca.zeros(self.shape)
self.grad_array = ca.zeros(self.shape)
def setup(self):
if len(self.x.shape) == 1:
self.shape = (1, ) + self.x.shape
elif len(self.x.shape) == 2:
self.shape = tuple(reversed(self.x.shape))
else:
raise ValueError('invalid shape for transpose: %s' %
str(self.x.shape))
self.array = ca.zeros(self.shape)
self.grad_array = ca.zeros(self.shape)
def setup(self):
x_shape = self.x.shape
batch_size, n_channels = x_shape[:2]
self.weights.setup((self.n_filters, n_channels) + self.filter_shape)
if self.bias is not None:
self.bias.setup((1, self.n_filters, 1, 1))
img_shape = self.img_out_shape(x_shape[2:], self.filter_shape,
self.strides, self.padding)
self.shape = (batch_size, self.n_filters) + img_shape
self.array = ca.zeros(self.shape)
self.grad_array = ca.zeros(self.shape)
def setup(self):
super(BatchNormalization, self).setup()
if len(self.out_shape) != 2:
raise ValueError('Only 1D data supported')
reduced_shape = 1, self.out_shape[1]
self.running_mean = ca.zeros(reduced_shape)
self.running_std = ca.ones(reduced_shape)
self._tmp_batch_centered = ca.zeros(self.out_shape)
self._tmp_batch_inv_std = ca.zeros(reduced_shape)
if self.affine:
self.gamma.setup(reduced_shape)
self.beta.setup(reduced_shape)
def setup(self):
super(BatchNormalization, self).setup()
reduced_shape = (1, ) + self.out_shape[1:]
if self.running_mean is None:
self.running_mean = ca.zeros(reduced_shape)
self.running_std = ca.ones(reduced_shape)
if self.affine:
self.gamma.setup(reduced_shape)
self.beta.setup(reduced_shape)
else:
if self.running_mean.shape != reduced_shape:
raise ValueError('New input shape is not compatible')
self._tmp_batch_inv_std = ca.zeros(reduced_shape)
self._tmp_batch_centered = ca.zeros(self.out_shape)
def setup(self):
x_shape = self.x.shape
batch_size, n_channels = x_shape[:2]
self.weights.setup((n_channels, self.n_filters) + self.filter_shape)
if self.bias is not None:
self.bias.setup((1, self.n_filters, 1, 1))
img_shape = self.img_out_shape(x_shape[2:], self.filter_shape,
self.strides, self.padding)
self.shape = (batch_size, self.n_filters) + img_shape
self.array = ca.zeros(self.shape)
self.grad_array = ca.zeros(self.shape)
# make sure conv_op is initialized
self.conv_op.fprop(self.grad_array, self.weights.array,
convout=self.x.grad_array)
def setup(self):
super(BatchNormalization, self).setup()
reduced_shape = (1,) + self.out_shape[1:]
if self.running_mean is None:
self.running_mean = ca.zeros(reduced_shape)
self.running_std = ca.ones(reduced_shape)
if self.affine:
self.gamma.setup(reduced_shape)
self.beta.setup(reduced_shape)
else:
if self.running_mean.shape != reduced_shape:
raise ValueError('New input shape is not compatible')
self._tmp_batch_inv_std = ca.zeros(reduced_shape)
self._tmp_batch_centered = ca.zeros(self.out_shape)
def setup(self):
self.shape = np.add(np.zeros(self.lhs.shape),
np.zeros(self.rhs.shape)).shape
size = np.prod(self.shape)
if self.lhs_orig.bpropable and size > self.lhs_orig.array.size:
self.lhs = Broadcast(self.lhs_orig.shape,
self.shape)(self.lhs_orig)
self.inputs[0] = self.lhs
self.lhs.setup()
if self.rhs_orig.bpropable and size > self.rhs_orig.array.size:
self.rhs = Broadcast(self.rhs_orig.shape,
self.shape)(self.rhs_orig)
self.inputs[1] = self.rhs
self.rhs.setup()
self.array = ca.zeros(self.shape)
self.grad_array = ca.zeros(self.shape)
def setup(self):
super(SpatialBatchNormalization, self).setup()
if len(self.out_shape) != 4:
raise ValueError('Only 4D data supported')
reduced_shape = 1, self.out_shape[1], 1, 1
if self.running_mean is None:
self.running_mean = ca.zeros(reduced_shape)
self.running_std = ca.ones(reduced_shape)
if self.affine:
self.gamma.setup(reduced_shape)
self.beta.setup(reduced_shape)
else:
if self.running_mean.shape != reduced_shape:
raise ValueError('New input shape is not compatible')
self._tmp_batch_inv_std = ca.zeros(reduced_shape)
self._tmp_batch_centered = ca.zeros(self.out_shape)
def setup(self):
self.shape = np.add(np.zeros(self.lhs.shape),
np.zeros(self.rhs.shape)).shape
size = np.prod(self.shape)
if self.lhs_orig.bpropable and size > self.lhs_orig.array.size:
self.lhs = Broadcast(self.lhs_orig.shape,
self.shape)(self.lhs_orig)
self.inputs[0] = self.lhs
self.lhs.setup()
if self.rhs_orig.bpropable and size > self.rhs_orig.array.size:
self.rhs = Broadcast(self.rhs_orig.shape,
self.shape)(self.rhs_orig)
self.inputs[1] = self.rhs
self.rhs.setup()
self.array = ca.zeros(self.shape)
self.grad_array = ca.zeros(self.shape)
def setup_from_shape(self, in_shape):
batch_size = in_shape[0]
self.shape = (batch_size,)
self.array = ca.zeros(self.shape)
self.grad_array = ca.ones(self.shape)
self.axis = tuple(range(1, len(in_shape)))
self.bcast_shape = tuple([batch_size, ] + [1, ]*len(self.axis))
def setup(self):
super(SpatialBatchNormalization, self).setup()
if len(self.out_shape) != 4:
raise ValueError('Only 4D data supported')
reduced_shape = 1, self.out_shape[1], 1, 1
if self.running_mean is None:
self.running_mean = ca.zeros(reduced_shape)
self.running_std = ca.ones(reduced_shape)
if self.affine:
self.gamma.setup(reduced_shape)
self.beta.setup(reduced_shape)
else:
if self.running_mean.shape != reduced_shape:
raise ValueError('New input shape is not compatible')
self._tmp_batch_inv_std = ca.zeros(reduced_shape)
self._tmp_batch_centered = ca.zeros(self.out_shape)
def setup(self):
out_shape = self.x.out_shape
for i in range(self.n_splits):
self.outputs[i].out_shape = out_shape
self.outputs[i].out = self.x.out
self.outputs[i].bpropable = self.bpropable
if self.bpropable:
self.outputs[i].out_grad = ca.zeros(out_shape)
def setup(self):
self.shape = self.x.shape
for i in range(self.n_splits):
self.outputs[i].shape = self.shape
self.outputs[i].array = self.x.array
self.outputs[i].bpropable = self.bpropable
if self.bpropable:
self.outputs[i].grad_array = ca.zeros(self.shape)
def setup(self):
self.shape = self.x.shape
for i in range(self.n_splits):
self.outputs[i].shape = self.shape
self.outputs[i].array = self.x.array
self.outputs[i].bpropable = self.bpropable
if self.bpropable:
self.outputs[i].grad_array = ca.zeros(self.shape)
def setup(self):
out_shape = self.x.out_shape
for i in range(self.n_splits):
self.outputs[i].out_shape = out_shape
self.outputs[i].out = self.x.out
self.outputs[i].bpropable = self.bpropable
if self.bpropable:
self.outputs[i].out_grad = ca.zeros(out_shape)
def setup(self):
b, c, h, w = self.imgs.out_shape
b_, f = self.feats.out_shape
if b != b_:
raise ValueError('batch size mismatch')
self.out_shape = (b, c+f, h, w)
self.out = ca.empty(self.out_shape)
self.out_grad = ca.empty(self.out_shape)
self.tmp = ca.zeros((b, f, h, w))
def setup_from_shape(self, in_shape):
batch_size = in_shape[0]
self.shape = (batch_size, )
self.array = ca.zeros(self.shape)
self.grad_array = ca.ones(self.shape)
self.axis = tuple(range(1, len(in_shape)))
self.bcast_shape = tuple([
batch_size,
] + [
1,
] * len(self.axis))
def setup(self):
splits = [0] + self.splits + [self.x.shape[0]]
self.slices = []
for i in range(len(splits) - 1):
self.slices.append((splits[i], splits[i + 1]))
for i, (start, end) in enumerate(self.slices):
shape = (end - start, ) + self.x.shape[1:]
self.outputs[i].shape = shape
self.outputs[i].array = self.x.array[start:end]
if self.bpropable:
self.outputs[i].grad_array = ca.zeros(shape)
def setup(self):
splits = [0] + self.splits + [self.x.out_shape[0]]
self.slices = []
for i in range(len(splits) - 1):
self.slices.append((splits[i], splits[i + 1]))
for i, (start, end) in enumerate(self.slices):
out_shape = (end - start, ) + self.x.out_shape[1:]
self.outputs[i].out_shape = out_shape
self.outputs[i].out = self.x.out[start:end, :]
self.outputs[i].bpropable = self.bpropable
if self.bpropable:
self.outputs[i].out_grad = ca.zeros(out_shape)
def setup(self):
splits = [0] + self.splits + [self.x.out_shape[0]]
self.slices = []
for i in range(len(splits) - 1):
self.slices.append((splits[i], splits[i+1]))
for i, (start, end) in enumerate(self.slices):
out_shape = (end-start,) + self.x.out_shape[1:]
self.outputs[i].out_shape = out_shape
self.outputs[i].out = self.x.out[start:end, :]
self.outputs[i].bpropable = self.bpropable
if self.bpropable:
self.outputs[i].out_grad = ca.zeros(out_shape)
def _update(self):
# Forward propagation
next_x = self.x.array
x_feats = [None]*len(self.layers)
x_grams = [None]*len(self.layers)
for l, layer in enumerate(self.layers):
next_x = layer.fprop(next_x)
if self.subject_weights[l] > 0:
x_feats[l] = next_x
if self.style_weights[l] > 0:
x_feats[l] = next_x
x_grams[l] = gram_matrix(next_x)
# Backward propagation
grad = ca.zeros_like(next_x)
loss = ca.zeros(1)
for l, layer in reversed(list(enumerate(self.layers))):
if self.subject_weights[l] > 0:
diff = x_feats[l] - self.subject_feats[l]
norm = ca.sum(ca.fabs(diff)) + 1e-8
weight = float(self.subject_weights[l]) / norm
grad += diff * weight
loss += 0.5*weight*ca.sum(diff**2)
if self.style_weights[l] > 0:
diff = x_grams[l] - self.style_grams[l]
n_channels = diff.shape[0]
x_feat = ca.reshape(x_feats[l], (n_channels, -1))
style_grad = ca.reshape(ca.dot(diff, x_feat), x_feats[l].shape)
norm = ca.sum(ca.fabs(style_grad))
weight = float(self.style_weights[l]) / norm
style_grad *= weight
grad += style_grad
loss += 0.25*weight*ca.sum(diff**2)
grad = layer.bprop(grad)
if self.tv_weight > 0:
x = ca.reshape(self.x.array, (3, 1) + grad.shape[2:])
tv = self.tv_conv.fprop(x, self.tv_kernel)
tv *= self.tv_weight
grad -= ca.reshape(tv, grad.shape)
ca.copyto(self.x.grad_array, grad)
return loss
def _update(self):
# Forward propagation
next_x = self.x.array
x_feats = [None] * len(self.layers)
for l, layer in enumerate(self.layers):
next_x = layer.fprop(next_x)
if self.subject_weights[l] > 0 or self.style_weights[l] > 0:
x_feats[l] = next_x
# Backward propagation
grad = ca.zeros_like(next_x)
loss = ca.zeros(1)
for l, layer in reversed(list(enumerate(self.layers))):
if self.subject_weights[l] > 0:
diff = x_feats[l] - self.subject_feats[l]
norm = ca.sum(ca.fabs(diff)) + 1e-8
weight = float(self.subject_weights[l]) / norm
grad += diff * weight
loss += 0.5 * weight * ca.sum(diff**2)
if self.style_weights[l] > 0:
diff = gram_matrix(x_feats[l]) - self.style_grams[l]
n_channels = diff.shape[0]
x_feat = ca.reshape(x_feats[l], (n_channels, -1))
style_grad = ca.reshape(ca.dot(diff, x_feat), x_feats[l].shape)
norm = ca.sum(ca.fabs(style_grad))
weight = float(self.style_weights[l]) / norm
style_grad *= weight
grad += style_grad
loss += 0.25 * weight * ca.sum(diff**2)
grad = layer.bprop(grad)
if self.tv_weight > 0:
x = ca.reshape(self.x.array, (3, 1) + grad.shape[2:])
tv = self.tv_conv.fprop(x, self.tv_kernel)
tv *= self.tv_weight
grad -= ca.reshape(tv, grad.shape)
ca.copyto(self.x.grad_array, grad)
return loss
def setup(self):
self.shape = self.x.shape[:1]
self.array = ca.zeros(self.shape, dtype=int_)
def setup(self):
self.shape = self.x.shape + (self.n_classes,)
self.array = ca.zeros(self.shape)
def __init__(self, shape):
self.shape = shape
self.array = ca.zeros(shape)
def setup(self):
super(SpatialDropout, self).setup()
self.mask_shape = self.shape[:2] + (1, 1)
self._tmp_mask = ca.zeros(self.mask_shape, dtype=ca.int_)
def setup(self):
self.array = ca.zeros(self.shape)
self.grad_array = ca.zeros(self.broadcast_shape)
def setup(self):
self.shape = self.x.shape
self.array = ca.zeros(self.shape)
self.grad_array = ca.zeros(self.shape)
def setup(self):
self.shape = self.x.shape
self.array = ca.zeros(self.shape)
self.grad_array = ca.zeros(self.shape)
def setup(self):
super(SpatialDropout, self).setup()
self.mask_shape = self.shape[:2] + (1, 1)
self._tmp_mask = ca.zeros(self.mask_shape, dtype=ca.int_)
def __init__(self, shape, low=0.0, high=1.0):
self.shape = shape
self.low = low
self.high = high
self.array = ca.zeros(self.shape)
def __init__(self, array):
if isinstance(array, np.ndarray):
array = ca.array(array)
self.shape = array.shape
self.array = array
self.grad_array = ca.zeros(self.shape)
def __init__(self, shape, mu=0.0, sigma=1.0):
self.shape = shape
self.mu = mu
self.sigma = sigma
self.array = ca.zeros(self.shape)
def setup(self):
shape = self.x.shape[1:]
for i in range(self.n_splits):
self.outputs[i].shape = shape
self.outputs[i].array = self.x.array[i]
self.outputs[i].grad_array = ca.zeros(shape)
def setup(self):
self.shape = ca.sum(self.x.array, axis=self.axis,
keepdims=self.keepdims).shape
self.array = ca.zeros(self.shape)
self.grad_array = ca.zeros(self.shape)
def setup(self):
self.shape = ca.nnet.rescale(self.x.array, self.factor,
self.method).shape
self.array = ca.zeros(self.shape)
self.grad_array = ca.zeros(self.shape)
def setup(self):
shape = self.x.shape
self.shape = (shape[0], np.prod(shape[1:]))
self.array = ca.zeros(self.shape)
self.grad_array = ca.zeros(self.shape)
def setup(self):
super(Dropout, self).setup()
self.mask_shape = self.shape
self._tmp_mask = ca.zeros(self.mask_shape, dtype=ca.int_)
def setup(self):
self.shape = self.pool_op.fprop(self.x.array).shape
self.array = ca.zeros(self.shape)
self.grad_array = ca.zeros(self.shape)
def setup(self):
self.shape = self.x.shape + (self.n_classes, )
self.array = ca.zeros(self.shape)
def __init__(self, shape):
self.shape = shape
self.array = ca.zeros(shape)
def __init__(self, array):
if isinstance(array, np.ndarray):
array = ca.array(array)
self.shape = array.shape
self.array = array
self.grad_array = ca.zeros(self.shape)
def setup(self):
super(Dropout, self).setup()
self.mask_shape = self.shape
self._tmp_mask = ca.zeros(self.mask_shape, dtype=ca.int_)
def setup(self):
self.array = ca.zeros(self.shape)
self.grad_array = ca.zeros(self.broadcast_shape)
def setup(self):
batch_size, n_in = self.x.shape
self.shape = (batch_size, self.n_out)
self.array = ca.zeros(self.shape)
self.grad_array = ca.zeros(self.shape)
self.weights.setup((n_in, self.n_out))