def setup(self):
size = self.x.out.size
newshape_size = np.prod(self.newshape)
self.out_shape = tuple(d if d != -1 else -size//newshape_size
for d in self.newshape)
self.out = ca.empty(self.out_shape)
self.out_grad = ca.empty(self.out_shape)
def split(arr, a_size, axis=0, out_a=None, out_b=None):
shp = arr.shape
ndim = arr.ndim
da = a_size
db = shp[axis] - a_size
out_a_shp = shp[:axis] + (da,) + shp[axis + 1 :]
out_b_shp = shp[:axis] + (db,) + shp[axis + 1 :]
if out_a is None:
out_a = ca.empty(out_a_shp, dtype=arr.dtype)
else:
if out_a.shape != out_a_shp:
raise ValueError("shape mismatch")
if out_b is None:
out_b = ca.empty(out_b_shp, dtype=arr.dtype)
else:
if out_b.shape != out_b_shp:
raise ValueError("shape mismatch")
if ndim < 3:
shp = shp + (1,) * (3 - ndim)
elif ndim > 3:
if axis == 0:
shp = shp[axis], prod(shp[1:]), 1
elif axis + 1 == ndim:
shp = 1, prod(shp[:axis]), shp[axis]
axis = 2
else:
shp = prod(shp[:axis]), shp[axis], prod(shp[axis + 1 :])
axis = 1
d0, d1, d2 = shp
array_ops._split(arr._data, axis, d0, d1, d2, da, db, out_a._data, out_b._data)
return out_a, out_b
def bprop(self, imgs, filters, convout_d, to_filters=True, to_imgs=True,
filters_d=None, imgs_d=None):
if imgs is None:
imgs = self.last_imgs
b, c, _, _ = imgs.shape
f, c_filters, _, _ = filters.shape
b_convout, f_convout, _, _ = convout_d.shape
if b != b_convout:
raise ValueError('batch mismatch')
if f != f_convout:
raise ValueError('filter mismatch')
if c != c_filters:
raise ValueError('channel mismatch')
if imgs.dtype != filters.dtype != convout_d.dtype:
raise ValueError('dtype mismatch')
if filters_d is None:
filters_d = ca.empty(filters.shape, dtype=filters.dtype)
if imgs_d is None:
imgs_d = ca.empty(imgs.shape, dtype=imgs.dtype)
conv_bc01_bprop(imgs=imgs,
convout_d=convout_d,
filters=filters,
padding=self.padding,
strides=self.strides,
imgs_grad=imgs_d,
filters_grad=filters_d)
return filters_d, imgs_d
def setup(self):
if not (isinstance(self.out, ca.ndarray)
and self.out.shape == self.out_shape):
self.out = ca.empty(self.out_shape)
if not (isinstance(self.out_grad, ca.ndarray)
and self.out_grad.shape == self.out_shape):
self.out_grad = ca.empty(self.out_shape)
def setup(self):
x_shape = self.x.out_shape
self.out_shape = (x_shape[0], self.n_out)
self.out = ca.empty(self.out_shape)
self.out_grad = ca.empty(self.out_shape)
self.weights.setup((x_shape[1], self.n_out))
self.bias.setup(self.n_out)
def fprop(self, imgs, poolout=None):
poolout_shape = self.output_shape(imgs.shape)
if poolout is None:
poolout = ca.empty(poolout_shape, dtype=imgs.dtype)
else:
if poolout_shape != poolout.shape:
raise ValueError('poolout.shape does not match result')
if imgs.dtype != poolout.dtype:
raise ValueError('dtype mismatch')
img_shape = imgs.shape[-2:]
if self.impl == 'masked':
n_imgs = np.prod(imgs.shape[:-2])
if self.mask is None or self.mask.shape != poolout_shape:
self.mask = ca.empty(poolout_shape, dtype=np.dtype('int32'))
nnet._max_pool_b01(
imgs._data, n_imgs, img_shape, self.win_shape, self.padding,
self.strides, poolout._data, self.mask._data
)
elif self.impl == 'cudnn':
n_imgs, n_channels = imgs.shape[:2]
self.last_imgs = imgs
self.last_poolout = poolout
self.pool_cudnn.fprop(
imgs._data, n_imgs, n_channels, img_shape, poolout._data
)
return poolout
def split(arr, a_size, axis=0, out_a=None, out_b=None):
shp = arr.shape
ndim = arr.ndim
da = a_size
db = shp[axis] - a_size
out_a_shp = shp[:axis] + (da, ) + shp[axis + 1:]
out_b_shp = shp[:axis] + (db, ) + shp[axis + 1:]
if out_a is None:
out_a = ca.empty(out_a_shp, dtype=arr.dtype)
else:
if out_a.shape != out_a_shp:
raise ValueError('shape mismatch')
if out_b is None:
out_b = ca.empty(out_b_shp, dtype=arr.dtype)
else:
if out_b.shape != out_b_shp:
raise ValueError('shape mismatch')
if ndim < 3:
shp = shp + (1, ) * (3 - ndim)
elif ndim > 3:
if axis == 0:
shp = shp[axis], prod(shp[1:]), 1
elif axis + 1 == ndim:
shp = 1, prod(shp[:axis]), shp[axis]
axis = 2
else:
shp = prod(shp[:axis]), shp[axis], prod(shp[axis + 1:])
axis = 1
d0, d1, d2 = shp
array_ops._split(arr._data, axis, d0, d1, d2, da, db, out_a._data,
out_b._data)
return out_a, out_b
def setup(self):
if not (isinstance(self.out, ca.ndarray)
and self.out.shape == self.out_shape):
self.out = ca.empty(self.out_shape)
if not (isinstance(self.out_grad, ca.ndarray)
and self.out_grad.shape == self.out_shape):
self.out_grad = ca.empty(self.out_shape)
def setup(self):
size = self.x.out.size
newshape_size = np.prod(self.newshape)
self.out_shape = tuple(d if d != -1 else -size // newshape_size
for d in self.newshape)
self.out = ca.empty(self.out_shape)
self.out_grad = ca.empty(self.out_shape)
def fprop(self, imgs, poolout=None):
poolout_shape = self.output_shape(imgs.shape)
if poolout is None:
poolout = ca.empty(poolout_shape, dtype=imgs.dtype)
else:
if poolout_shape != poolout.shape:
raise ValueError('poolout.shape does not match result')
if imgs.dtype != poolout.dtype:
raise ValueError('dtype mismatch')
img_shape = imgs.shape[-2:]
if self.impl == 'cudarray':
n_imgs = np.prod(imgs.shape[:-2])
if self.method == 'max':
if self.mask is None or self.mask.shape != poolout_shape:
self.mask = ca.empty(poolout_shape,
dtype=np.dtype('int32'))
nnet._max_pool_b01(imgs._data, n_imgs, img_shape,
self.win_shape, self.padding, self.strides,
poolout._data, self.mask._data)
else:
nnet._avg_pool_b01(imgs._data, n_imgs, img_shape,
self.win_shape, self.padding, self.strides,
poolout._data)
else:
n_imgs, n_channels = imgs.shape[:2]
self.last_imgs = imgs
self.last_poolout = poolout
self.pool_cudnn.fprop(imgs._data, imgs.shape, poolout._data)
return poolout
def avg_running_time(fun, reps):
# Memory allocation forces GPU synchronization
ca.empty(1)
start_time = time.time()
for _ in range(reps):
fun()
ca.empty(1)
return float(time.time() - start_time) / reps
def avg_running_time(fun, reps):
# Memory allocation forces GPU synchronization
ca.empty(1)
start_time = time.time()
for _ in range(reps):
fun()
ca.empty(1)
return float(time.time() - start_time) / reps
def setup(self):
a_shp = self.a.out_shape
b_shp = self.b.out_shape
concat_size = a_shp[self.axis] + b_shp[self.axis]
self.a_size = a_shp[self.axis]
self.out_shape = (a_shp[:self.axis] + (concat_size,) +
a_shp[self.axis+1:])
self.out = ca.empty(self.out_shape)
self.out_grad = ca.empty(self.out_shape)
def bprop(self,
imgs,
filters,
convout_d,
to_filters=True,
to_imgs=True,
filters_d=None,
imgs_d=None):
if imgs is not None:
b, c, img_h, img_w = imgs.shape
else:
b, c, img_h, img_w = self.imgs_shape
if filters is not None:
f, c, filter_h, filter_w = filters.shape
if imgs_d is not None:
b, c, img_h, img_w = imgs_d.shape
b_convout, f_convout, convout_h, convout_w = convout_d.shape
imgs_shape = self.imgs_shape
img_shape = imgs_shape[2:]
filter_shape = (filter_h, filter_w)
if to_filters:
if filters_d is None:
filters_d = ca.empty(filters.shape, dtype=filters.dtype)
else:
if filters_d.shape != filters.shape:
raise ValueError('filters_d.shape does not match result')
if filters_d.dtype != filters.dtype:
raise ValueError('dtype mismatch')
if self.impl == 'cudarray':
nnet._conv_bc01_matmul_bprop_filters(
imgs._data, convout_d._data, b, c, f, img_shape,
filter_shape, self.padding, self.strides, filters_d._data)
if to_imgs:
if imgs_d is None:
imgs_d = ca.empty(imgs_shape, dtype=convout_d.dtype)
elif imgs is not None:
if imgs_d.shape != imgs.shape:
raise ValueError('imgs_d.shape does not match result')
if imgs_d.dtype != imgs.dtype:
raise ValueError('dtype mismatch')
if self.impl == 'cudarray':
nnet._conv_bc01_matmul_bprop_imgs(filters._data,
convout_d._data, b, c, f,
img_shape, filter_shape,
self.padding, self.strides,
imgs_d._data)
if self.impl == 'cudnn':
imgs_ = None if imgs is None else imgs._data
imgs_d_ = imgs_d._data if to_imgs else None
filters_ = filters_d._data if to_filters else None
self.conv_cudnn.bprop(imgs_, filters._data, convout_d._data,
imgs_d_, filters_)
return filters_d, imgs_d
def setup(self):
b, c, h, w = self.imgs.out_shape
b_, f = self.feats.out_shape
if b != b_:
raise ValueError('batch size mismatch')
self.out_shape = (b, c+f, h, w)
self.out = ca.empty(self.out_shape)
self.out_grad = ca.empty(self.out_shape)
self.tmp = ca.zeros((b, f, h, w))
def setup(self):
shape = self.inputs[0].out_shape
for expr in self.inputs:
if shape != expr.out_shape:
raise ValueError('shape mismatch: %s and %s'
% (shape, expr.out_shape))
self.out_shape = (self.n_sources,) + shape
self.out = ca.empty(self.out_shape)
self.out_grad = ca.empty(self.out_shape)
def setup(self):
a_shp = self.a.out_shape
b_shp = self.b.out_shape
concat_size = a_shp[self.axis] + b_shp[self.axis]
self.a_size = a_shp[self.axis]
self.out_shape = (a_shp[:self.axis] + (concat_size, ) +
a_shp[self.axis + 1:])
self.out = ca.empty(self.out_shape)
self.out_grad = ca.empty(self.out_shape)
def setup(self):
shape = self.inputs[0].out_shape
for expr in self.inputs:
if shape != expr.out_shape:
raise ValueError('shape mismatch: %s and %s' %
(shape, expr.out_shape))
self.out_shape = (self.n_sources, ) + shape
self.out = ca.empty(self.out_shape)
self.out_grad = ca.empty(self.out_shape)
def setup(self):
try:
# XXX: don't be lazy
self.out_shape = ca.dot(self.lhs.out, self.rhs.out).shape
except ValueError:
raise ValueError('Shape mismatch: %s and %s for %s. LHS: %s RHS: '
'%s.' % (self.lhs.out.shape, self.rhs.out.shape,
self, self.lhs, self.rhs))
self.out = ca.empty(self.out_shape)
self.out_grad = ca.empty(self.out_shape)
def setup(self):
if len(self.x.out_shape) == 1:
self.out_shape = (1, ) + self.x.out_shape
elif len(self.x.out_shape) == 2:
self.out_shape = tuple(reversed(self.x.out_shape))
else:
raise ValueError('invalid shape for transpose: %s' %
str(self.x.shape))
self.out = ca.empty(self.out_shape)
self.out_grad = ca.empty(self.out_shape)
def setup(self):
if len(self.x.out_shape) == 1:
self.out_shape = (1,) + self.x.out_shape
elif len(self.x.out_shape) == 2:
self.out_shape = tuple(reversed(self.x.out_shape))
else:
raise ValueError('invalid shape for transpose: %s'
% str(self.x.shape))
self.out = ca.empty(self.out_shape)
self.out_grad = ca.empty(self.out_shape)
def setup(self):
x_shape = self.x.out_shape
batch_size, n_channels = x_shape[:2]
self.weights.setup((self.n_filters, n_channels) + self.filter_shape)
self.bias.setup((1, self.n_filters, 1, 1))
out_shape = convout_shape(x_shape[2:], self.filter_shape, self.strides,
self.padding)
self.out_shape = (batch_size, self.n_filters) + out_shape
self.out = ca.empty(self.out_shape)
self.out_grad = ca.empty(self.out_shape)
def setup(self):
x_shape = self.x.out_shape
batch_size, n_channels = x_shape[:2]
self.weights.setup((self.n_filters, n_channels) + self.filter_shape)
self.bias.setup((1, self.n_filters, 1, 1))
out_shape = convout_shape(x_shape[2:], self.filter_shape,
self.strides, self.padding)
self.out_shape = (batch_size, self.n_filters) + out_shape
self.out = ca.empty(self.out_shape)
self.out_grad = ca.empty(self.out_shape)
def setup(self):
x_shape = self.x.out_shape
batch_size, n_channels = x_shape[:2]
self.weights.setup((n_channels, self.n_filters) + self.filter_shape)
if self.bias is not None:
self.bias.setup((1, self.n_filters, 1, 1))
out_shape = self.img_out_shape(x_shape[2:], self.filter_shape, self.strides, self.padding)
self.out_shape = (batch_size, self.n_filters) + out_shape
self.out = ca.empty(self.out_shape)
self.out_grad = ca.empty(self.out_shape)
# make sure conv_op is initialized
self.conv_op.fprop(self.out_grad, self.weights.array, convout=self.x.out_grad)
def bprop(self, imgs, filters, convout_d, to_filters=True, to_imgs=True,
filters_d=None, imgs_d=None):
b, c, img_h, img_w = imgs.shape
f, c_filters, filter_h, filter_w = filters.shape
b_convout, f_convout, convout_h, convout_w = convout_d.shape
img_shape = (img_h, img_w)
filter_shape = (filter_h, filter_w)
if b != b_convout:
raise ValueError('batch mismatch')
if f != f_convout:
raise ValueError('filter mismatch')
if c != c_filters:
raise ValueError('channel mismatch')
if imgs.dtype != filters.dtype != convout_d.dtype:
raise ValueError('dtype mismatch')
if to_filters:
if filters_d is None:
filters_d = ca.empty(filters.shape, dtype=filters.dtype)
else:
if filters_d.shape != filters.shape:
raise ValueError('filters_d.shape does not match result')
if filters_d.dtype != filters.dtype:
raise ValueError('dtype mismatch')
if self.impl == 'matmul':
nnet._conv_bc01_matmul_bprop_filters(
imgs._data, convout_d._data, b, c, f, img_shape,
filter_shape, self.padding, self.strides, filters_d._data
)
if to_imgs:
if imgs_d is None:
imgs_d = ca.empty(imgs.shape, dtype=imgs.dtype)
else:
if imgs_d.shape != imgs.shape:
raise ValueError('imgs_d.shape does not match result')
if imgs_d.dtype != imgs.dtype:
raise ValueError('dtype mismatch')
if self.impl == 'matmul':
nnet._conv_bc01_matmul_bprop_imgs(
filters._data, convout_d._data, b, c, f, img_shape,
filter_shape, self.padding, self.strides, imgs_d._data
)
if self.impl == 'cudnn':
imgs_ = imgs_d._data if to_imgs else None
filters_ = filters_d._data if to_filters else None
self.conv_cudnn.bprop(imgs._data, filters._data, convout_d._data,
imgs_, filters_)
return filters_d, imgs_d
def bprop(self, imgs, filters, convout_d, to_filters=True, to_imgs=True,
filters_d=None, imgs_d=None):
b, c, img_h, img_w = imgs.shape
f, c_filters, filter_h, filter_w = filters.shape
b_convout, f_convout, convout_h, convout_w = convout_d.shape
img_shape = (img_h, img_w)
filter_shape = (filter_h, filter_w)
if b != b_convout:
raise ValueError('batch mismatch')
if f != f_convout:
raise ValueError('filter mismatch')
if c != c_filters:
raise ValueError('channel mismatch')
if imgs.dtype != filters.dtype != convout_d.dtype:
raise ValueError('dtype mismatch')
if to_filters:
if filters_d is None:
filters_d = ca.empty(filters.shape, dtype=filters.dtype)
else:
if filters_d.shape != filters.shape:
raise ValueError('filters_d.shape does not match result')
if filters_d.dtype != filters.dtype:
raise ValueError('dtype mismatch')
if self.impl == 'matmul':
nnet._conv_bc01_matmul_bprop_filters(
imgs._data, convout_d._data, b, c, f, img_shape,
filter_shape, self.padding, self.strides, filters_d._data
)
if to_imgs:
if imgs_d is None:
imgs_d = ca.empty(imgs.shape, dtype=imgs.dtype)
else:
if imgs_d.shape != imgs.shape:
raise ValueError('imgs_d.shape does not match result')
if imgs_d.dtype != imgs.dtype:
raise ValueError('dtype mismatch')
if self.impl == 'matmul':
nnet._conv_bc01_matmul_bprop_imgs(
filters._data, convout_d._data, b, c, f, img_shape,
filter_shape, self.padding, self.strides, imgs_d._data
)
if self.impl == 'cudnn':
imgs_ = imgs_d._data if to_imgs else None
filters_ = filters_d._data if to_filters else None
self.conv_cudnn.bprop(imgs._data, filters._data, convout_d._data,
imgs_, filters_)
return filters_d, imgs_d
def batches(self):
x = ca.empty(self.x_shape, dtype=self.x.dtype)
y = ca.empty(self.y_shape, dtype=self.y.dtype)
for start, stop in self._batch_slices():
if stop > start:
x_np = self.x[start:stop]
y_np = self.y[start:stop]
else:
x_np = np.concatenate((self.x[start:], self.x[:stop]))
y_np = np.concatenate((self.y[start:], self.y[:stop]))
ca.copyto(x, x_np)
ca.copyto(y, y_np)
yield x, y
def batches(self):
x = ca.empty(self.x_shape, dtype=self.x.dtype)
y = ca.empty(self.y_shape, dtype=self.y.dtype)
for start, stop in self._batch_slices():
if stop > start:
x_np = self.x[start:stop]
y_np = self.y[start:stop]
else:
x_np = np.concatenate((self.x[start:], self.x[:stop]))
y_np = np.concatenate((self.y[start:], self.y[:stop]))
ca.copyto(x, x_np)
ca.copyto(y, y_np)
yield x, y
def bprop(self, imgs, filters, convout_d, to_filters=True, to_imgs=True,
filters_d=None, imgs_d=None):
if imgs is not None:
b, c, img_h, img_w = imgs.shape
else:
b, c, img_h, img_w = self.imgs_shape
if filters is not None:
f, c, filter_h, filter_w = filters.shape
if imgs_d is not None:
b, c, img_h, img_w = imgs_d.shape
b_convout, f_convout, convout_h, convout_w = convout_d.shape
imgs_shape = self.imgs_shape
img_shape = imgs_shape[2:]
filter_shape = (filter_h, filter_w)
if to_filters:
if filters_d is None:
filters_d = ca.empty(filters.shape, dtype=filters.dtype)
else:
if filters_d.shape != filters.shape:
raise ValueError('filters_d.shape does not match result')
if filters_d.dtype != filters.dtype:
raise ValueError('dtype mismatch')
if self.impl == 'cudarray':
nnet._conv_bc01_matmul_bprop_filters(
imgs._data, convout_d._data, b, c, f, img_shape,
filter_shape, self.padding, self.strides, filters_d._data
)
if to_imgs:
if imgs_d is None:
imgs_d = ca.empty(imgs_shape, dtype=convout_d.dtype)
elif imgs is not None:
if imgs_d.shape != imgs.shape:
raise ValueError('imgs_d.shape does not match result')
if imgs_d.dtype != imgs.dtype:
raise ValueError('dtype mismatch')
if self.impl == 'cudarray':
nnet._conv_bc01_matmul_bprop_imgs(
filters._data, convout_d._data, b, c, f, img_shape,
filter_shape, self.padding, self.strides, imgs_d._data
)
if self.impl == 'cudnn':
imgs_ = None if imgs is None else imgs._data
imgs_d_ = imgs_d._data if to_imgs else None
filters_ = filters_d._data if to_filters else None
self.conv_cudnn.bprop(imgs_, filters._data, convout_d._data,
imgs_d_, filters_)
return filters_d, imgs_d
def setup(self):
x_shape = self.x.out_shape
batch_size, n_channels = x_shape[:2]
self.weights.setup((n_channels, self.n_filters) + self.filter_shape)
self.bias.setup((1, self.n_filters, 1, 1))
out_shape = self.img_out_shape(x_shape[2:], self.filter_shape,
self.strides, self.padding)
self.out_shape = (batch_size, self.n_filters) + out_shape
self.out = ca.empty(self.out_shape)
self.out_grad = ca.empty(self.out_shape)
# make sure conv_op is initialized
self.conv_op.fprop(self.out_grad,
self.weights.array,
convout=self.x.out_grad)
def fprop(self, imgs, filters, convout=None):
b, c, img_h, img_w = imgs.shape
f, c_filters, filter_h, filter_w = filters.shape
if c != c_filters:
raise ValueError('channel mismatch')
if imgs.dtype != filters.dtype:
raise ValueError('dtype mismatch')
convout_shape = self.output_shape(imgs.shape, f, (filter_h, filter_w))
if convout is None:
convout = ca.empty(convout_shape, dtype=imgs.dtype)
else:
if convout.shape != convout_shape:
raise ValueError('convout.shape does not match result')
if convout.dtype != imgs.dtype:
raise ValueError('dtype mismatch')
conv_bc01(imgs=imgs,
filters=filters,
padding=self.padding,
strides=self.strides,
convout=convout)
self.last_imgs = imgs
return convout
def fprop(self, imgs, filters, convout=None):
b, c, img_h, img_w = imgs.shape
f, c_filters, filter_h, filter_w = filters.shape
if c != c_filters:
raise ValueError('channel mismatch')
if imgs.dtype != filters.dtype:
raise ValueError('dtype mismatch')
convout_shape = self.output_shape(imgs.shape, f, (filter_h, filter_w))
if convout is None:
convout = ca.empty(convout_shape, dtype=imgs.dtype)
else:
if convout.shape != convout_shape:
raise ValueError('convout.shape does not match result')
if convout.dtype != imgs.dtype:
raise ValueError('dtype mismatch')
conv_bc01(imgs=imgs,
filters=filters,
padding=self.padding,
strides=self.strides,
convout=convout)
self.last_imgs = imgs
return convout
def fprop(self, imgs, filters, convout=None):
b, c, img_h, img_w = imgs.shape
f, c_filters, filter_h, filter_w = filters.shape
if c != c_filters:
raise ValueError('channel mismatch')
if imgs.dtype != filters.dtype:
raise ValueError('dtype mismatch')
img_shape = (img_h, img_w)
self.imgs_shape = imgs.shape
filter_shape = (filter_h, filter_w)
convout_shape = self.output_shape(imgs.shape, f, (filter_h, filter_w))
if convout is None:
convout = ca.empty(convout_shape, dtype=imgs.dtype)
else:
if convout.shape != convout_shape:
raise ValueError('convout.shape does not match result')
if convout.dtype != imgs.dtype:
raise ValueError('dtype mismatch')
if self.impl == 'cudarray':
nnet._conv_bc01_matmul(imgs._data, filters._data, b, c, f,
img_shape, filter_shape, self.padding,
self.strides, convout._data)
else:
self.conv_cudnn.fprop(imgs._data, filters._data, b, c, f,
img_shape, filter_shape, convout._data)
return convout
def bprop(self, img_shape, poolout_d, imgs_d=None):
n_imgs_shape = poolout_d.shape[:-2]
imgs_shape = n_imgs_shape + img_shape
if imgs_d is None:
imgs_d = ca.empty(imgs_shape, dtype=poolout_d.dtype)
else:
if imgs_d.shape != imgs_d.shape:
raise ValueError('poolout.shape does not match result')
if imgs_d.dtype != poolout_d.dtype:
raise ValueError('dtype mismatch')
if self.impl == 'cudarray':
n_imgs = np.prod(n_imgs_shape)
if self.method == 'max':
nnet._max_pool_b01_bprop(poolout_d._data, self.mask._data,
n_imgs, img_shape, self.win_shape,
self.padding, self.strides,
imgs_d._data)
else:
nnet._avg_pool_b01_bprop(poolout_d._data, n_imgs, img_shape,
self.win_shape, self.padding,
self.strides, imgs_d._data)
else:
self.pool_cudnn.bprop(self.last_imgs._data,
self.last_poolout._data, poolout_d._data,
imgs_d._data)
return imgs_d
def bprop(self, y_grad, h_grad):
n = self.n_hidden
h_grad = h_grad + y_grad
c_grad = h_grad * self._tmp_u
u_grad = h_grad * (self._tmp_c - self._tmp_h_tm1)
h_grad *= 1 - self._tmp_u
c_grad = ca.ascontiguousarray(ca.transpose(c_grad))
u_grad = ca.ascontiguousarray(ca.transpose(u_grad))
c_grad = self.act_c.bprop(c_grad)
ca.sum(c_grad, axis=1, keepdims=True, out=self.b_c.grad_array)
u_grad = self.act_u.bprop(u_grad)
ca.sum(u_grad, axis=1, keepdims=True, out=self.b_u.grad_array)
r_grad = c_grad * self._tmp_h_c
r_grad = self.act_r.bprop(r_grad)
ca.sum(r_grad, axis=1, keepdims=True, out=self.b_r.grad_array)
stack_grad = ca.empty((self.n_hidden * 3, y_grad.shape[0]))
stack_grad[:n, :] = r_grad
stack_grad[n : n * 2, :] = u_grad
stack_grad[n * 2 : n * 3, :] = c_grad
ca.dot(self._tmp_x.T, stack_grad.T, out=self.w_x.grad_array)
x_grad = ca.dot(stack_grad.T, self.w_x.array.T)
stack_grad[n * 2 : n * 3, :] *= self._tmp_r
ca.dot(self._tmp_h_tm1.T, stack_grad.T, out=self.w_h.grad_array)
h_grad += ca.dot(stack_grad.T, self.w_h.array.T)
ca.clip(h_grad, -self.clip, self.clip, out=h_grad)
return {"x_grad": x_grad, "h_grad": h_grad}
def concatenate(a, b, axis=0, out=None):
ndim = a.ndim
a_shp = a.shape
b_shp = b.shape
d_concat = a_shp[axis] + b_shp[axis]
out_shp = a_shp[:axis] + (d_concat,) + a_shp[axis + 1 :]
if out is None:
out = ca.empty(out_shp, dtype=a.dtype)
else:
if out.shape != out_shp:
raise ValueError("shape mismatch")
da = a_shp[axis]
db = b_shp[axis]
if ndim < 3:
a_shp = a_shp + (1,) * (3 - ndim)
b_shp = b_shp + (1,) * (3 - ndim)
elif ndim > 3:
if axis == 0:
a_shp = a_shp[axis], prod(a_shp[1:]), 1
b_shp = b_shp[axis], prod(b_shp[1:]), 1
elif axis + 1 == ndim:
a_shp = 1, prod(a_shp[:axis]), a_shp[axis]
b_shp = 1, prod(b_shp[:axis]), b_shp[axis]
axis = 2
else:
a_shp = prod(a_shp[:axis]), a_shp[axis], prod(a_shp[axis + 1 :])
b_shp = prod(b_shp[:axis]), b_shp[axis], prod(b_shp[axis + 1 :])
axis = 1
d0, d1, d2 = a_shp[:axis] + (d_concat,) + a_shp[axis + 1 :]
array_ops._concatenate(a._data, b._data, axis, d0, d1, d2, da, db, out._data)
return out
def fprop(self, imgs, filters, convout=None):
b, c, img_h, img_w = imgs.shape
f, c_filters, filter_h, filter_w = filters.shape
if c != c_filters:
raise ValueError('channel mismatch')
if imgs.dtype != filters.dtype:
raise ValueError('dtype mismatch')
img_shape = (img_h, img_w)
self.imgs_shape = imgs.shape
filter_shape = (filter_h, filter_w)
convout_shape = self.output_shape(imgs.shape, f, (filter_h, filter_w))
if convout is None:
convout = ca.empty(convout_shape, dtype=imgs.dtype)
else:
if convout.shape != convout_shape:
raise ValueError('convout.shape does not match result')
if convout.dtype != imgs.dtype:
raise ValueError('dtype mismatch')
if self.impl == 'cudarray':
nnet._conv_bc01_matmul(
imgs._data, filters._data, b, c, f, img_shape, filter_shape,
self.padding, self.strides, convout._data
)
else:
self.conv_cudnn.fprop(
imgs._data, filters._data, b, c, f, img_shape, filter_shape,
convout._data
)
return convout
def bprop(self, img_shape, poolout_d, imgs_d=None):
n_imgs_shape = poolout_d.shape[:-2]
imgs_shape = n_imgs_shape + img_shape
if imgs_d is None:
imgs_d = ca.empty(imgs_shape, dtype=poolout_d.dtype)
else:
if imgs_d.shape != imgs_d.shape:
raise ValueError('poolout.shape does not match result')
if imgs_d.dtype != poolout_d.dtype:
raise ValueError('dtype mismatch')
if self.impl == 'cudarray':
n_imgs = np.prod(n_imgs_shape)
if self.method == 'max':
nnet._max_pool_b01_bprop(
poolout_d._data, self.mask._data, n_imgs, img_shape,
self.win_shape, self.padding, self.strides, imgs_d._data
)
else:
nnet._avg_pool_b01_bprop(
poolout_d._data, n_imgs, img_shape, self.win_shape,
self.padding, self.strides, imgs_d._data
)
else:
self.pool_cudnn.bprop(
self.last_imgs._data, self.last_poolout._data, poolout_d._data,
imgs_d._data
)
return imgs_d
def bprop(self, y_grad, h_grad):
n = self.n_hidden
h_grad = h_grad + y_grad
c_grad = h_grad * self._tmp_u
u_grad = h_grad * (self._tmp_c - self._tmp_h_tm1)
h_grad *= (1 - self._tmp_u)
c_grad = ca.ascontiguousarray(ca.transpose(c_grad))
u_grad = ca.ascontiguousarray(ca.transpose(u_grad))
c_grad = self.act_c.bprop(c_grad)
ca.sum(c_grad, axis=1, keepdims=True, out=self.b_c.grad_array)
u_grad = self.act_u.bprop(u_grad)
ca.sum(u_grad, axis=1, keepdims=True, out=self.b_u.grad_array)
r_grad = c_grad * self._tmp_h_c
r_grad = self.act_r.bprop(r_grad)
ca.sum(r_grad, axis=1, keepdims=True, out=self.b_r.grad_array)
stack_grad = ca.empty((self.n_hidden*3, y_grad.shape[0]))
stack_grad[:n, :] = r_grad
stack_grad[n:n*2, :] = u_grad
stack_grad[n*2:n*3, :] = c_grad
ca.dot(self._tmp_x.T, stack_grad.T, out=self.w_x.grad_array)
x_grad = ca.dot(stack_grad.T, self.w_x.array.T)
stack_grad[n*2:n*3, :] *= self._tmp_r
ca.dot(self._tmp_h_tm1.T, stack_grad.T, out=self.w_h.grad_array)
h_grad += ca.dot(stack_grad.T, self.w_h.array.T)
ca.clip(h_grad, -self.clip, self.clip, out=h_grad)
return {'x_grad': x_grad, 'h_grad': h_grad}
def concatenate(a, b, axis=0, out=None):
ndim = a.ndim
a_shp = a.shape
b_shp = b.shape
d_concat = a_shp[axis] + b_shp[axis]
out_shp = a_shp[:axis] + (d_concat, ) + a_shp[axis + 1:]
if out is None:
out = ca.empty(out_shp, dtype=a.dtype)
else:
if out.shape != out_shp:
raise ValueError('shape mismatch')
da = a_shp[axis]
db = b_shp[axis]
if ndim < 3:
a_shp = a_shp + (1, ) * (3 - ndim)
b_shp = b_shp + (1, ) * (3 - ndim)
elif ndim > 3:
if axis == 0:
a_shp = a_shp[axis], prod(a_shp[1:]), 1
b_shp = b_shp[axis], prod(b_shp[1:]), 1
elif axis + 1 == ndim:
a_shp = 1, prod(a_shp[:axis]), a_shp[axis]
b_shp = 1, prod(b_shp[:axis]), b_shp[axis]
axis = 2
else:
a_shp = prod(a_shp[:axis]), a_shp[axis], prod(a_shp[axis + 1:])
b_shp = prod(b_shp[:axis]), b_shp[axis], prod(b_shp[axis + 1:])
axis = 1
d0, d1, d2 = a_shp[:axis] + (d_concat, ) + a_shp[axis + 1:]
array_ops._concatenate(a._data, b._data, axis, d0, d1, d2, da, db,
out._data)
return out
def batches(self):
x1 = ca.empty(self.x_shape, dtype=self.x.dtype)
x2 = ca.empty_like(x1)
y = ca.empty(self.y_shape, dtype=self.y.dtype)
for start, stop in self._batch_slices():
if stop > start:
x1_np = self.x[start:stop]
x2_np = self.x2[start:stop]
y_np = self.y[start:stop]
else:
x1_np = np.concatenate((self.x[start:], self.x[:stop]))
x2_np = np.concatenate((self.x[start:], self.x[:stop]))
y_np = np.concatenate((self.y[start:], self.y[:stop]))
ca.copyto(x1, x1_np)
ca.copyto(x2, x2_np)
ca.copyto(y, y_np)
yield x1, x2, y
def batches(self):
x1 = ca.empty(self.x_shape, dtype=self.x.dtype)
x2 = ca.empty_like(x1)
y = ca.empty(self.y_shape, dtype=self.y.dtype)
for start, stop in self._batch_slices():
if stop > start:
x1_np = self.x[start:stop]
x2_np = self.x2[start:stop]
y_np = self.y[start:stop]
else:
x1_np = np.concatenate((self.x[start:], self.x[:stop]))
x2_np = np.concatenate((self.x[start:], self.x[:stop]))
y_np = np.concatenate((self.y[start:], self.y[:stop]))
ca.copyto(x1, x1_np)
ca.copyto(x2, x2_np)
ca.copyto(y, y_np)
yield x1, x2, y
def batches(self):
x = ca.empty(self.x_shape, dtype=dp.float_)
y = ca.empty(self.y_shape, dtype=dp.float_)
for start, stop in self._batch_slices():
if stop > start:
x_np = self.x[start:stop]
y_np = self.y[start:stop]
else:
x_np = np.concatenate((self.x[start:], self.x[:stop]))
y_np = np.concatenate((self.y[start:], self.y[:stop]))
if random.randint(0, 1) == 0:
x_np = x_np[:, :, :, ::-1]
x_np = img_transform(x_np, to_bc01=False)
x_np = np.ascontiguousarray(x_np)
ca.copyto(x, x_np)
ca.copyto(y, y_np)
yield {'x': x, 'y': y}
def batches(self):
x = ca.empty(self.x_shape, dtype=dp.float_)
y = ca.empty(self.y_shape, dtype=dp.float_)
for start, stop in self._batch_slices():
if stop > start:
x_np = self.x[start:stop]
y_np = self.y[start:stop]
else:
x_np = np.concatenate((self.x[start:], self.x[:stop]))
y_np = np.concatenate((self.y[start:], self.y[:stop]))
if random.randint(0, 1) == 0:
x_np = x_np[:, :, :, ::-1]
x_np = img_transform(x_np, to_bc01=False)
x_np = np.ascontiguousarray(x_np)
ca.copyto(x, x_np)
ca.copyto(y, y_np)
yield x, y
def setup(self):
splits = [0] + self.splits + [self.x.out_shape[0]]
self.slices = []
for i in range(len(splits) - 1):
self.slices.append((splits[i], splits[i+1]))
for i, (start, end) in enumerate(self.slices):
out_shape = (end-start,) + self.x.out_shape[1:]
self.outputs[i].out_shape = out_shape
self.outputs[i].out = self.x.out[start:end, :]
self.outputs[i].out_grad = ca.empty(out_shape)
def one_hot_encode(labels, n_classes, out=None):
out_shape = (labels.size, n_classes)
if labels.dtype != np.dtype('int32'):
raise ValueError('labels.dtype must be int')
if out is None:
out = ca.empty(out_shape)
else:
if out.shape != out_shape:
raise ValueError('shape mismatch')
nnet._one_hot_encode(labels._data, n_classes, out_shape[0], out._data)
return out
def one_hot_decode(one_hot, out=None):
out_shape = (one_hot.shape[0],)
if out is None:
out = ca.empty(out_shape, dtype=np.dtype('int32'))
else:
if out.dtype != np.dtype('int32'):
raise ValueError('out.dtype must be int')
if out.shape != out_shape:
raise ValueError('shape mismatch')
ca.argmax(one_hot, axis=1, out=out)
return out
def one_hot_decode(one_hot, out=None):
out_shape = (one_hot.shape[0], )
if out is None:
out = ca.empty(out_shape, dtype=np.dtype('int32'))
else:
if out.dtype != np.dtype('int32'):
raise ValueError('out.dtype must be int')
if out.shape != out_shape:
raise ValueError('shape mismatch')
ca.argmax(one_hot, axis=1, out=out)
return out
def one_hot_encode(labels, n_classes, out=None):
out_shape = (labels.size, n_classes)
if labels.dtype != np.dtype('int32'):
raise ValueError('labels.dtype must be int')
if out is None:
out = ca.empty(out_shape)
else:
if out.shape != out_shape:
raise ValueError('shape mismatch')
nnet._one_hot_encode(labels._data, n_classes, out_shape[0], out._data)
return out
def unary(op, x, out=None):
out_shape = x.shape
if out is None:
out = cudarray.empty(out_shape, dtype=x.dtype)
else:
if not out_shape == out.shape:
raise ValueError('out.shape does not match result')
if not x.dtype == out.dtype:
raise ValueError('dtype mismatch')
n = x.size
elementwise._unary(op, x._data, n, out._data)
return out
def clip(a, a_min, a_max, out=None):
out_shape = a.shape
if out is None:
out = cudarray.empty(out_shape, dtype=a.dtype)
else:
if not out_shape == out.shape:
raise ValueError('out.shape does not match result')
if not a.dtype == out.dtype:
raise ValueError('dtype mismatch')
n = a.size
elementwise._clip(a._data, a_min, a_max, n, out._data)
return out
def clip(a, a_min, a_max, out=None):
out_shape = a.shape
if out is None:
out = cudarray.empty(out_shape, dtype=a.dtype)
else:
if not out_shape == out.shape:
raise ValueError('out.shape does not match result')
if not a.dtype == out.dtype:
raise ValueError('dtype mismatch')
n = a.size
elementwise._clip(a._data, a_min, a_max, n, out._data)
return out
def unary(op, x, out=None):
out_shape = x.shape
if out is None:
out = cudarray.empty(out_shape, dtype=x.dtype)
else:
if not out_shape == out.shape:
raise ValueError('out.shape does not match result')
if not x.dtype == out.dtype:
raise ValueError('dtype mismatch')
n = x.size
elementwise._unary(op, x._data, n, out._data)
return out
def bprop(self,
imgs,
filters,
convout_d,
to_filters=True,
to_imgs=True,
filters_d=None,
imgs_d=None):
if imgs is None:
imgs = self.last_imgs
b, c, _, _ = imgs.shape
f, c_filters, _, _ = filters.shape
b_convout, f_convout, _, _ = convout_d.shape
if b != b_convout:
raise ValueError('batch mismatch')
if f != f_convout:
raise ValueError('filter mismatch')
if c != c_filters:
raise ValueError('channel mismatch')
if imgs.dtype != filters.dtype != convout_d.dtype:
raise ValueError('dtype mismatch')
if filters_d is None:
filters_d = ca.empty(filters.shape, dtype=filters.dtype)
if imgs_d is None:
imgs_d = ca.empty(imgs.shape, dtype=imgs.dtype)
conv_bc01_bprop(imgs=imgs,
convout_d=convout_d,
filters=filters,
padding=self.padding,
strides=self.strides,
imgs_grad=imgs_d,
filters_grad=filters_d)
return filters_d, imgs_d
def fprop(self, imgs, poolout=None):
poolout_shape = self.output_shape(imgs.shape)
if poolout is None:
poolout = ca.empty(poolout_shape, dtype=imgs.dtype)
else:
if poolout_shape != poolout.shape:
raise ValueError('poolout.shape does not match result')
if imgs.dtype != poolout.dtype:
raise ValueError('dtype mismatch')
if self.mask is None or self.mask.shape[:-1] != poolout_shape:
self.mask = ca.empty(poolout_shape + (2,), dtype=np.dtype('int_'))
pool_bc01(imgs=imgs,
win_shape=self.win_shape,
strides=self.strides,
padding=self.padding,
poolout=poolout,
type=self.method,
switches=self.mask)
return poolout
def setup(self):
try:
self.out_shape = np.add(np.zeros_like(self.lhs.out),
np.zeros_like(self.rhs.out)).shape
if self.lhs_bprop and np.prod(self.out_shape) > self.lhs.out.size:
self.lhs = Broadcast(self.lhs.out_shape,
self.out_shape)(self.lhs)
self.inputs = [self.lhs, self.rhs]
self.lhs.setup()
if self.rhs_bprop and np.prod(self.out_shape) > self.rhs.out.size:
self.rhs = Broadcast(self.rhs.out_shape,
self.out_shape)(self.rhs)
self.rhs_broadcast = True
self.inputs = [self.lhs, self.rhs]
self.rhs.setup()
except ValueError:
raise
raise ValueError('Shape mismatch: %s and %s for %s. LHS: %s RHS: '
'%s.' % (self.lhs.out.shape, self.rhs.out.shape,
self, self.lhs, self.rhs))
self.out = ca.empty(self.out_shape)
self.out_grad = ca.empty(self.out_shape)
def fprop(self, imgs, poolout=None):
poolout_shape = self.output_shape(imgs.shape)
if poolout is None:
poolout = ca.empty(poolout_shape, dtype=imgs.dtype)
else:
if poolout_shape != poolout.shape:
raise ValueError('poolout.shape does not match result')
if imgs.dtype != poolout.dtype:
raise ValueError('dtype mismatch')
if self.mask is None or self.mask.shape[:-1] != poolout_shape:
self.mask = ca.empty(poolout_shape + (2,), dtype=np.dtype('int_'))
pool_bc01(imgs=imgs,
win_shape=self.win_shape,
strides=self.strides,
padding=self.padding,
poolout=poolout,
type=self.method,
switches=self.mask)
return poolout