def _create_cc(self, x, gy, hint, y, ws, ksize, stride, pad, cover_all, e):
self.ksize = ksize
self.stride = stride
self.pad = pad
self.cover_all = cover_all
self.x = array(x, m.memory.nchw, e)
gy = array(gy, m.memory.nchw, e)
if self.alg_kind is pooling_max:
gy_md = y.memory.get_primitive_desc().desc()
else:
gy_md = gy.memory.get_primitive_desc().desc()
gx_md = m.desc(x.shape, m.memory.f32, m.memory.any)
# x_md = self.x.memory.get_primitive_desc().desc()
n, c, h, w = x.shape
sy, sx = _pair(stride)
kh, kw = _pair(ksize)
p_upper, p_left = _pair(pad)
yh = conv.get_conv_outsize(h, kh, sy, p_upper, cover_all=cover_all)
assert yh > 0, 'Height in the output should be positive.'
yw = conv.get_conv_outsize(w, kw, sx, p_left, cover_all=cover_all)
assert yw > 0, 'Width in the output should be positive.'
p_down = sy * (yh - 1) + kh - h - p_upper
p_right = sx * (yw - 1) + kw - w - p_left
cc_d = pooling_backward.desc(self.alg_kind, gx_md, gy_md, stride,
ksize, (p_upper, p_left),
(p_down, p_right), zero)
cc_pd = pooling_backward.primitive_desc(cc_d, e, hint)
gx = mdarray(cc_pd.diff_src_primitive_desc())
if self.alg_kind is pooling_max:
# For max pooling reorder y if needed
outputs = reorder_if_must(gy, y.memory.get_primitive_desc(), e,
self.dag_)
if len(outputs) == 2:
self.reordered_gy, self.itm_arr = outputs[:2]
else:
self.reordered_gy = outputs[0]
self.dag_.push_back(
pooling_backward.pooling_backward(
cc_pd, at(self.reordered_gy.memory), at(ws.memory),
gx.memory))
else:
# There is no workspace for average pooling
self.dag_.push_back(
pooling_backward.pooling_backward(cc_pd, at(gy.memory),
gx.memory))
self._hint = hint
self.gy = gy
self.outputs = gx,
def _create_cc(self, xs, gy, hint, axis, e):
self.axis = axis
gy = array(gy[0], m.memory.nchw, e)
fmt = m.memory.nchw
gy_mpd = gy.memory.get_primitive_desc()
offsets = (0, 0, 0, 0)
self.outputs = ()
for x in xs:
view_pd = view.primitive_desc(gy_mpd, x.shape, offsets)
fmt = m.get_fmt(gy_mpd)
assert x.dtype == numpy.dtype('float32')
gx = mdarray(x.shape, m.memory.f32, fmt, e)
# gx = mdarray(x.memory.get_primitive_desc())
# gx = array(x, m.memory.nchw, e)
reorder_pd = r.primitive_desc(view_pd.dst_primitive_desc(),
gx.memory.get_primitive_desc())
reorder_prim = r.reorder(reorder_pd, at(gy.memory), gx.memory)
self.dag_.push_back(reorder_prim)
self.outputs += (gx, )
new_off = offsets[axis] + x.shape[axis]
offsets = offsets[:axis] + (new_off, ) + offsets[axis + 1:]
self.gy = gy
self.xs = xs
self._hint = hint
def _create_cc(self, x, dropout_ratio, e=Engine()):
self.dropout_op = dropout_f32(dropout_ratio)
self.mask = np.ndarray(shape=x.shape, dtype=np.float32)
self._mask = array(self.mask, _format(self.mask.ndim), e)
self._hint = mdarray(self.x.memory.get_primitive_desc())
def __init__(self, inputs, dropout_ratio, pos=(0, 0), e=Engine()):
super(DropoutForward, self).__init__()
self.x = array(inputs[0], _format(inputs[0].ndim), Engine())
if self.new:
self._create_cc(inputs[0], dropout_ratio, e)
def cpu_cosim_dump_inner(self, in_data, out_grad=None):
cd = None
if out_grad is None:
cd = cdump.cosim_dump(cdump.cdump_op_lrn_forward)
else:
cd = cdump.cosim_dump(cdump.cdump_op_lrn_backward)
x = array(in_data[0], m.memory.nchw, Engine())
cd.dump_memory(cdump.cdump_src_memory, x.memory)
if out_grad is not None:
gy = array(out_grad[0], m.memory.nchw, Engine())
cd.dump_memory(cdump.cdump_diff_dst_memory, gy.memory)
cd.dump_int_parms(cdump.cdump_lrn_local_size, 1, self.n)
cd.dump_double_parms(cdump.cdump_lrn_doulbe_parms, 3, self.k,
self.alpha, self.beta)
def __init__(self, dropout_op, mask, gy, hint, pos=(0, 0), e=Engine()):
super(DropoutBackward, self).__init__()
self._dropout_op = dropout_op
self._mask = mask
self.gy = array(gy[0], _format(gy[0].ndim), e)
if self.new:
self._create_cc(hint)
def cpu_cosim_dump_inner(self, in_data, out_grad=None):
cd = None
if out_grad is None:
cd = cdump.cosim_dump(cdump.cdump_op_max_pooling_forward)
else:
cd = cdump.cosim_dump(cdump.cdump_op_max_pooling_backward)
x = array(in_data[0], m.memory.nchw, Engine())
cd.dump_memory(cdump.cdump_src_memory, x.memory)
if out_grad is not None:
gy = array(out_grad[0], m.memory.nchw, Engine())
cd.dump_memory(cdump.cdump_diff_dst_memory, gy.memory)
cd.dump_int_parms(cdump.cdump_max_pooling_int_parms, 8,
self.kh, self.kw, self.sy, self.sx, self.ph, self.pw,
pooling_max, 1 if self.cover_all else 0)
def _create_cc(self, inputs, fwd_x, gy, hint, flags, eps, mean, var, e):
self.train = configuration.config.train
self.flags = flags
self.eps = eps
x, gamma, beta = inputs[:3]
# self.x = array(x, m.memory.nchw, e)
self.x = fwd_x
x_mpd = self.x.memory.get_primitive_desc()
x_md = x_mpd.desc()
gy = array(gy, m.memory.nchw, e)
outputs = reorder_if_must(gy, x_mpd, e, self.dag_)
if len(outputs) == 2:
self.gy_src = gy
gy, self.itm_arr = outputs[:2]
else:
self.gy_src = gy
gy = outputs[0]
gy_md = gy.memory.get_primitive_desc().desc()
cc_d = bn_backward.desc(backward, gy_md, x_md, eps, flags)
cc_pd = bn_backward.primitive_desc(cc_d, e, hint)
gx = mdarray(self.x.memory.get_primitive_desc(), gy.memory)
if flags & use_scale_shift:
w = numpy.concatenate((gamma, beta), axis=0).reshape((2, -1))
self.w = array(w, m.memory.nc, e)
self.mean = array(mean, m.memory.x, e)
self.var = array(var, m.memory.x, e)
self.gw = mdarray(cc_pd.diff_weights_primitive_desc())
bwd_p = bn_backward.batch_normalization_backward(
cc_pd, at(self.x.memory), at(self.mean.memory),
at(self.var.memory), at(gy.memory), at(self.w.memory),
gx.memory, self.gw.memory)
else:
bwd_p = bn_backward.batch_normalization_backward(
cc_pd, at(self.x.memory), at(self.mean.memory),
at(self.var.memory), at(gy.memory), gx.memory)
self.dag_.push_back(bwd_p)
self._hint = hint
self.gy = gy
self.outputs = gx, self.gw
def _create_cc(self, x, gy, hint, ws, n, k, alpha, beta, e):
self.n = n
self.k = k
self.alpha = alpha
self.beta = beta
self.x = array(x, m.memory.nchw, e)
x_md = self.x.memory.get_primitive_desc().desc()
# TODO: check avx512?
gy = array(gy, m.memory.nchw, e)
gy_md = gy.memory.get_primitive_desc().desc()
cc_d = lrn_backward.desc(lrn_across_channels, x_md, gy_md, n, alpha,
beta, k)
cc_pd = lrn_backward.primitive_desc(cc_d, e, hint)
gx = mdarray(cc_pd.diff_src_primitive_desc())
self.dag_.push_back(
lrn_backward.lrn_backward(cc_pd, at(self.x.memory), at(gy.memory),
at(ws.memory), gx.memory))
self._hint = hint
self.gy = gy
self.outputs = gx,
def mkl_sum(xs, func=None):
e = Engine()
xarrays = () # prevent the obj from gc
xs_arrays = () # prevent the obj from gc
itm_arr = None # prvent the obj from gc
xs_mpdl = m.mpd_list()
xs_pl = ()
scales = m.vectord()
pl = primitive_list()
for i in range(len(xs)):
xarray = array(xs[i], _x_format(xs[i].ndim), e)
xmpd = xarray.memory.get_primitive_desc()
if i == 0:
xmpd_best = xmpd
else:
if m.get_fmt(xmpd) > m.get_fmt(xmpd_best):
xmpd_best = xmpd
xs_arrays += (xarray,)
for x in xs_arrays:
outputs = reorder_if_must(x, xmpd_best, e, pl)
if len(outputs) == 2:
xarray, itm_arr = outputs[:2]
else:
xarray = outputs[0]
xarrays += (xarray,)
scales.push_back(1.0)
xs_mpdl.push_back(xarray.memory.get_primitive_desc())
xs_pl += (at(xarray.memory), )
cc_pd = sum.primitive_desc(scales, xs_mpdl)
if func is not None and hasattr(func, 'hint'): # this is only used for grad accumulate currently
cc = ComputeComplex.get_bd_cc(func.hint, pos=(func.rank, func.fanout))
if cc is not None:
y = cc.gy
else:
y = mdarray(cc_pd.dst_primitive_desc())
else:
y = mdarray(cc_pd.dst_primitive_desc())
pl.push_back(sum.sum(cc_pd, xs_pl, y.memory))
s = Stream()
s.submit(pl)
s.wait()
return y
def _create_cc(self, x, ksize, stride, pad, cover_all, e):
self.ksize = ksize
self.stride = stride
self.pad = pad
self.cover_all = cover_all
self.x = array(x, m.memory.nchw, e)
# TODO: check avx512?
n, c, h, w = x.shape
sy, sx = _pair(stride)
kh, kw = _pair(ksize)
p_upper, p_left = _pair(pad)
yh = conv.get_conv_outsize(h, kh, sy, p_upper, cover_all=cover_all)
assert yh > 0, 'Height in the output should be positive.'
yw = conv.get_conv_outsize(w, kw, sx, p_left, cover_all=cover_all)
assert yw > 0, 'Width in the output should be positive.'
y_shape = (n, c, yh, yw)
p_down = sy * (yh - 1) + kh - h - p_upper
p_right = sx * (yw - 1) + kw - w - p_left
y_md = m.desc(y_shape, m.memory.f32, m.memory.any)
x_md = self.x.memory.get_primitive_desc().desc()
cc_d = pooling_forward.desc(forward_training, self.alg_kind, x_md,
y_md, stride, ksize, (p_upper, p_left),
(p_down, p_right), zero)
cc_pd = pooling_forward.primitive_desc(cc_d, e)
y = mdarray(cc_pd.dst_primitive_desc())
if self.alg_kind is pooling_max:
ws = mdarray(cc_pd.workspace_primitive_desc())
self.dag_.push_back(
pooling_forward.pooling_forward(cc_pd, at(self.x.memory),
y.memory, ws.memory))
else:
# There is no workspace for average pooling
ws = None
self.dag_.push_back(
pooling_forward.pooling_forward(cc_pd, at(self.x.memory),
y.memory))
self._hint = cc_pd
self.outputs = y,
self.ws = ws
def _create_cc(self, x, n, k, alpha, beta, e):
self.n = n
self.k = k
self.alpha = alpha
self.beta = beta
# TODO: check avx512?
self.x = array(x, m.memory.nchw, e)
x_md = self.x.memory.get_primitive_desc().desc()
cc_d = lrn_forward.desc(forward_training, lrn_across_channels, x_md, n,
alpha, beta, k)
cc_pd = lrn_forward.primitive_desc(cc_d, e)
y = mdarray(cc_pd.dst_primitive_desc())
ws = mdarray(cc_pd.workspace_primitive_desc())
self.dag_.push_back(
lrn_forward.lrn_forward(cc_pd, at(self.x.memory), ws.memory,
y.memory))
self._hint = cc_pd
self.outputs = y,
self.ws = ws
def _create_cc(self, xs, axis, e):
self.axis = axis
xarrays = ()
axis_dim = 0
xs_mpdl = m.mpd_list()
# xs_pl = primitive_list()
xs_pl = ()
for x in xs:
axis_dim += x.shape[1]
xarray = array(x, m.memory.nchw, e)
xarrays += (xarray, )
xs_mpdl.push_back(xarray.memory.get_primitive_desc())
# xs_pl.push_back(xarray.memory)
xs_pl += (at(xarray.memory), )
cc_pd = concat.primitive_desc(axis, xs_mpdl)
y = mdarray(cc_pd.dst_primitive_desc())
self.dag_.push_back(concat.concat(cc_pd, xs_pl, y.memory))
self._hint = cc_pd
self.outputs = y,
self.xarrays = xarrays
def _create_cc(self, inputs, eps, mean, var, e):
self.eps = eps
self.mean = None
self.var = None
self.w = None
self.train = configuration.config.train
x, gamma, beta = inputs[:3]
fmt_desired = m.get_desired_format(x.shape[1])
x = array(x, m.memory.nchw, e)
# x = array(x, fmt_desired, e)
assert x.dtype == numpy.dtype('float32')
x_desired_md = m.desc(x.shape, m.memory.f32, fmt_desired)
x_desired_mpd = m.primitive_desc(x_desired_md, e)
outputs = reorder_if_must(x, x_desired_mpd, e, self.dag_)
if len(outputs) == 2:
self.x, self.itm_arr = outputs[:2]
self.x_src = x
else:
self.x = outputs[0]
self.x_src = x
w = numpy.concatenate((gamma, beta), axis=0).reshape((2, -1))
self.numpy_w = w
self.w = array(w, m.memory.nc, e)
scale_shift = True
self.flags = use_scale_shift
if mean is None:
fwd_prop_kind = forward_training
global_stats = False
else:
fwd_prop_kind = forward_scoring
self.flags |= use_global_stats
global_stats = True
self.mean = array(mean, m.memory.x, e)
self.var = array(var, m.memory.x, e)
x_md = self.x.memory.get_primitive_desc().desc()
cc_d = bn_forward.desc(fwd_prop_kind, x_md, eps, self.flags)
cc_pd = bn_forward.primitive_desc(cc_d, e)
y = mdarray(cc_pd.dst_primitive_desc())
# TODO reorder weight
# if scale_shift is True:
# w = mdarray(cc_pd.weights_primitive_desc())
if scale_shift is True and global_stats is False:
self.mean = mdarray(cc_pd.mean_primitive_desc())
self.var = mdarray(cc_pd.variance_primitive_desc())
if (not configuration.config.train) and (not global_stats):
if scale_shift is True:
bnf = bn_forward.batch_normalization_forward(
cc_pd, at(self.x.memory), at(self.w.memory), y.memory)
else:
bnf = bn_forward.batch_normalization_forward(
cc_pd, at(self.x.memory), y.memory)
elif global_stats is True:
if scale_shift is True:
bnf = bn_forward.batch_normalization_forward(
cc_pd, at(self.x.memory), at(self.mean.memory),
at(self.var.memory), at(self.w.memory), y.memory)
else:
bnf = bn_forward.batch_normalization_forward(
cc_pd, at(self.x.memory), self.mean.memory,
self.var.memory, y.memory)
else:
if scale_shift is True:
bnf = bn_forward.batch_normalization_forward(
cc_pd, at(self.x.memory), at(self.w.memory), y.memory,
self.mean.memory, self.var.memory)
else:
bnf = bn_forward.batch_normalization_forward(
cc_pd, at(self.x.memory), y.memory, self.mean.memory,
self.var.memory)
self.dag_.push_back(bnf)
self._hint = cc_pd
self.outputs = y, self.flags, self.mean, self.var
