def test_ReduceL2(tmpdir, dtype): with C.default_options(dtype=dtype): data = np.array( [[[1, 2], [3, 4]], [[5, 6], [7, 8]], [[9, 10], [11, 12]]], dtype=dtype) model = C.reduce_l2(data, 0) verify_no_input(model, tmpdir, 'ReduceL2_0')
def gaussian_mdn_phi(target, mu, sigma, ndim: int): """ Calculates phi between the target tensor and the network prediction Does not assumes independence between components of target. Arguments: target: target tensor with shape (ndim, ) mu: means of gaussian mdn with shape (nmix, ndim) sigma: sigma of gaussian mdn nmix (int): number of mixtures ndim (int): number of dimensions in gaussian Returns: :class:`~cntk.ops.functions.Function` """ if not len(mu.shape) == 2: raise ValueError("mu {0} must have shape (nmix, ndim)".format(mu.shape)) t = C.expand_dims(target, axis=0) exp_term = C.exp(C.negate(C.square(C.reduce_l2(t - mu, axis=-1)) / (2 * C.square(sigma)))) factor = C.reciprocal((2 * pi) ** (ndim / 2) * C.pow(sigma, ndim)) return factor * exp_term
def test_ReduceL2(tmpdir): data = np.array([[[1, 2], [3, 4]], [[5, 6], [7, 8]], [[9, 10], [11, 12]]], dtype=np.float32) model = C.reduce_l2(data, 0) verify_no_input(model, tmpdir, 'ReduceL2_0')
def test_ReduceL2(tmpdir, dtype): with C.default_options(dtype = dtype): data = np.array([[[1,2], [3,4]],[[5,6], [7,8]],[[9,10], [11,12]]], dtype=dtype) model = C.reduce_l2(data, 0) verify_no_input(model, tmpdir, 'ReduceL2_0')
C_step_loss = 0 for _ in range(n_critic): z_data = np.random.normal(size=(minibatch_size, z_dim)).astype("float32") x_data = train_reader.next_minibatch(minibatch_size, input_map=input_map) batch_input = {x: x_data[x].data, z: z_data} # # compute gradient penalty # gradient = C_real.grad( {C_real.arguments[0]: interpolate.eval(batch_input)}) gp_norm = np.square( C.reduce_l2(gradient, axis=(1, 2, 3)).eval() - 1) batch_input[gradient_penalty] = gp_norm C_trainer.train_minibatch(batch_input) C_step_loss += C_trainer.previous_minibatch_loss_average C_step_loss /= n_critic # # generator # z_data = np.random.normal(size=(minibatch_size, z_dim)).astype("float32") batch_input = {z: z_data} output = G_trainer.train_minibatch(batch_input, outputs=[G_fake])
def main(): show_image = False if show_image: bs = 1 ci = 3 co = 3 cg = co * (ci + 1) gd = 8 gh = 64 gw = 64 h = 256 w = 256 else: bs = 1 ci = 3 co = 3 cg = co * (ci + 1) gd = 8 gh = 64 gw = 64 h = 1024 w = 1024 im = C.input_variable([bs, ci, h, w], needs_gradient=True, dynamic_axes=[]) guide = C.input_variable([bs, h, w], needs_gradient=True, dynamic_axes=[]) guide_no_grad = C.input_variable([bs, h, w], needs_gradient=False, dynamic_axes=[]) grid = C.input_variable([bs, cg, gd, gh, gw], needs_gradient=True, dynamic_axes=[]) # Create indices xx = np.arange(0, w).reshape(1, -1).repeat(h, 0).astype(np.float32) yy = np.arange(0, h).reshape(-1, 1).repeat(w, 1).astype(np.float32) xx = C.Constant(xx, xx.shape) yy = C.Constant(yy, yy.shape) gx = ((xx + 0.5) / w) * gw gy = ((yy + 0.5) / h) * gh gz = C.clip(guide, 0.0, 1.0) * gd gz_no_grad = C.clip(guide_no_grad, 0.0, 1.0) * gd fx = C.element_max(C.floor(gx - 0.5), 0.0) fy = C.element_max(C.floor(gy - 0.5), 0.0) fz = C.element_max(C.floor(gz - 0.5), 0.0) fz_no_grad = C.element_max(C.floor(gz_no_grad - 0.5), 0.0) wx = gx - 0.5 - fx wy = gy - 0.5 - fy wx = C.expand_dims(C.expand_dims(wx, -1 - len(wx.shape)), -1 - len(wx.shape)) wy = C.expand_dims(C.expand_dims(wy, -1 - len(wy.shape)), -1 - len(wy.shape)) wz = C.abs(gz - 0.5 - fz) wz = C.expand_dims(wz, 0) fx = C.expand_dims(C.expand_dims(fx, -1 - len(fx.shape)), -1 - len(fx.shape)) fy = C.expand_dims(C.expand_dims(fy, -1 - len(fy.shape)), -1 - len(fy.shape)) cx = C.element_min(fx + 1, gw - 1) cy = C.element_min(fy + 1, gh - 1) cz = C.element_min(fz_no_grad + 1, gd - 1) batch_idx = np.arange(bs).reshape(bs, 1, 1, 1).astype(np.float32) batch_idx = C.Constant(batch_idx, batch_idx.shape) out = [] flat_grid = C.reshape(grid, [-1]) for c_ in range(co): c_idx = np.arange((ci + 1) * c_, (ci + 1) * (c_ + 1)).reshape(1, ci + 1, 1, 1).astype(np.float32) c_idx = C.Constant(c_idx, c_idx.shape) def flatten_and_gather(x, y, z): linear_idx = x + gw * y + gw * gh * z + c_idx * gw * gh * gd + batch_idx * gw * gh * gd * cg flat_linear_idx = C.reshape(linear_idx, [-1]) return C.reshape(C.gather(flat_grid, flat_linear_idx), linear_idx.shape) gather_fff = flatten_and_gather(fx, fy, fz_no_grad) gather_ffc = flatten_and_gather(fx, fy, cz) gather_fcf = flatten_and_gather(fx, cy, fz_no_grad) gather_fcc = flatten_and_gather(fx, cy, cz) gather_cff = flatten_and_gather(cx, fy, fz_no_grad) gather_cfc = flatten_and_gather(cx, fy, cz) gather_ccf = flatten_and_gather(cx, cy, fz_no_grad) gather_ccc = flatten_and_gather(cx, cy, cz) a = gather_fff*(1-wx)*(1-wy)*(1-wz) + \ gather_ffc*(1-wx)*(1-wy)*( wz) + \ gather_fcf*(1-wx)*( wy)*(1-wz) + \ gather_fcc*(1-wx)*( wy)*( wz) + \ gather_cff*( wx)*(1-wy)*(1-wz) + \ gather_cfc*( wx)*(1-wy)*( wz) + \ gather_ccf*( wx)*( wy)*(1-wz) + \ gather_ccc*( wx)*( wy)*( wz) o = C.reduce_sum(a[:, :-1, ...] * im, 1) + a[:, -1, ...] print(o.shape) out.append(C.expand_dims(o, 0)) out = C.splice(*out, axis=1) loss = C.reduce_l2(out) grid_val = np.random.rand(bs, cg, gd, gh, gw).astype(np.float32) if show_image: guide_val = skio.imread("/data/rgb.png").mean(2)[:h, :w].astype( np.float32) guide_val = np.expand_dims(guide_val / 255.0, 0) im_val = np.tile(np.expand_dims(guide_val, 1), [1, 3, 1, 1]) out_val = out.eval({ im: im_val, guide: guide_val, guide_no_grad: guide_val, grid: grid_val }) out_val = np.clip(np.transpose(np.squeeze(out_val), [1, 2, 0]), 0, 1) skio.imsave("/output/imout.png", out_val) else: im_val = np.random.randn(bs, ci, h, w) guide_val = np.random.rand(bs, h, w).astype(np.float32) # burning iteration for it in range(5): print('burning (', it, ')') g = loss.grad({ im: im_val, guide: guide_val, guide_no_grad: guide_val, grid: grid_val }) # actual iterations start = time.time() for it in range(50): print('profiling (', it, ')') g = loss.grad({ im: im_val, guide: guide_val, guide_no_grad: guide_val, grid: grid_val }) end = time.time() runtime = (end - start) * 1000.0 / 50.0 print('Runtime:', runtime)
def main(): bs = 4 c = 64 h = 512 w = 512 im = C.input_variable([bs, c, h, w], needs_gradient=True, dynamic_axes=[]) warp = C.input_variable([bs, 2, h, w], needs_gradient=True, dynamic_axes=[]) warp_ng = C.input_variable([bs, 2, h, w], needs_gradient=False, dynamic_axes=[]) # Create indices dx = 0.5 * (warp[:, 0, :, :] + 1.0) dy = 0.5 * (warp[:, 1, :, :] + 1.0) new_x = C.clip(dx * w, 0, w) new_y = C.clip(dy * h, 0, h) fx = C.clip(C.floor(new_x), 0, w - 2) fy = C.clip(C.floor(new_y), 0, h - 2) wx = new_x - fx wy = new_y - fy dx_ng = 0.5 * (warp_ng[:, 0, :, :] + 1.0) dy_ng = 0.5 * (warp_ng[:, 1, :, :] + 1.0) new_x_ng = C.clip(dx_ng * w, 0, w) new_y_ng = C.clip(dy_ng * h, 0, h) fx_ng = C.clip(C.floor(new_x_ng), 0, w - 2) fy_ng = C.clip(C.floor(new_y_ng), 0, h - 2) chan_idx = np.arange(c).reshape(1, c, 1, 1) chan_idx = C.Constant(chan_idx, chan_idx.shape) batch_idx = np.arange(bs).reshape(bs, 1, 1, 1) batch_idx = C.Constant(batch_idx, batch_idx.shape) flat_im = C.reshape(im, [-1]) def flatten_and_gather(x, y): linear_idx = x + w * y + w * h * chan_idx + w * h * c * batch_idx flat_linear_idx = C.reshape(linear_idx, [-1]) return C.reshape(C.gather(flat_im, flat_linear_idx), linear_idx.shape) gather_ff = flatten_and_gather(fx_ng, fy_ng) gather_fc = flatten_and_gather(fx_ng, fy_ng + 1) gather_cf = flatten_and_gather(fx_ng + 1, fy_ng) gather_cc = flatten_and_gather(fx_ng + 1, fy_ng + 1) out = gather_ff*(1-wx)*(1-wy) + \ gather_fc*(1-wx)*( wy) + \ gather_cf*( wx)*(1-wy) + \ gather_cc*( wx)*( wy) loss = C.reduce_l2(out) im_val = np.random.randn(bs, c, h, w).astype(np.float32) warp_val = np.random.rand(bs, 2, h, w).astype(np.float32) # burning iteration for it in range(5): print('burning (', it, ')') g = loss.grad({im: im_val, warp: warp_val, warp_ng: warp_val}) # actual iterations start = time.time() for it in range(50): print('profiling (', it, ')') g = loss.grad({im: im_val, warp: warp_val, warp_ng: warp_val}) end = time.time() runtime = (end - start) * 1000.0 / 50.0 print('Runtime:', runtime)
def _build_model(self): hidden_size = self.hidden_size output_size = self.output_size num_layers = self.num_layers keep_prob = self.keep_prob inputs = cntk.sequence.input_variable((output_size), name='inputs') target = cntk.input_variable((output_size), name='target') def lstm_cell(): _cell_creator = cntk.layers.Recurrence(cntk.layers.LSTM( hidden_size, use_peepholes=self.params.use_peephole), name='basic_lstm') if self.params.use_dropout: print(" ** using dropout for LSTM ** ") _cell_creator = cntk.layers.Dropout( keep_prob=keep_prob)(_cell_creator) return _cell_creator def gru_cell(): _cell_creator = cntk.layers.Recurrence( cntk.layers.GRU(hidden_size), name='gru') if self.params.use_dropout: print(" ** using dropout for LSTM ** ") _cell_creator = cntk.layers.Dropout( keep_prob=keep_prob)(_cell_creator) return _cell_creator def cifg_cell(): _cell_creator = cntk.layers.Recurrence(CIFG_LSTM( hidden_size, use_peepholes=self.params.use_peephole), name='cifg_lstm') if self.params.use_dropout: print(" ** using dropout for LSTM ** ") _cell_creator = cntk.layers.Dropout( keep_prob=keep_prob)(_cell_creator) return _cell_creator if self.config.cell == 'gru': _cell_creator = gru_cell elif self.config.cell == 'lstm': _cell_creator = lstm_cell elif self.config.cell == 'cifg_lstm': _cell_creator = cifg_cell else: raise ValueError( "Unsupported cell type, choose from {'lstm', 'gru', 'cifg_lstm'}." ) if self.params.use_residual: print(" ** using residual ** ") _output = inputs for _ in range(num_layers): _output = self.params.resWeight * _cell_creator()( _output) + _output # _output = _cell_creator()(_output) + _output else: cell = cntk.layers.For(range(num_layers), lambda: _cell_creator()) _output = cell(inputs) _output = cntk.sequence.last(_output) output = cntk.layers.Dense(output_size)(_output) self.output = output self.loss = cntk.squared_error(output, target) cost_mape = cntk.reduce_mean(cntk.abs(output - target) / target, axis=cntk.Axis.all_axes(), name='mape') cost_mae = cntk.reduce_mean(cntk.abs(output - target), axis=cntk.Axis.all_axes(), name='mae') cost_rmse = cntk.reduce_l2((output - target), axis=cntk.Axis.all_axes(), name='rmse') self.cost = cntk.combine([cost_mape, cost_mae, cost_rmse]) self.criterion = cntk.combine([loss, cost_mape])
def main(): bs = 4 c = 16 h = 512 w = 512 im = C.input_variable([bs, c, h, w], needs_gradient=True, dynamic_axes=[]) affine_mtx = C.input_variable([bs, 2, 3], needs_gradient=True, dynamic_axes=[]) affine_mtx_ng = C.input_variable([bs, 2, 3], needs_gradient=False, dynamic_axes=[]) xx = np.arange(0, w).reshape(1, -1).repeat(h, 0).astype(np.float32) yy = np.arange(0, h).reshape(-1, 1).repeat(w, 1).astype(np.float32) xx = C.Constant(xx, xx.shape) yy = C.Constant(yy, yy.shape) nrm_x = 2.0 * (xx / w) - 1.0 nrm_y = 2.0 * (yy / h) - 1.0 nrm_x = C.expand_dims(nrm_x, -1 - len(nrm_x.shape)) nrm_y = C.expand_dims(nrm_y, -1 - len(nrm_y.shape)) xformed_x = affine_mtx[:, 0, 0] * nrm_x + \ affine_mtx[:, 0, 1] * nrm_y + \ affine_mtx[:, 0, 2] xformed_y = affine_mtx[:, 1, 0] * nrm_x + \ affine_mtx[:, 1, 1] * nrm_y + \ affine_mtx[:, 1, 2] xformed_x = 0.5 * xformed_x + 1.0 xformed_y = 0.5 * xformed_y + 1.0 xformed_x = C.expand_dims(xformed_x, 0) xformed_y = C.expand_dims(xformed_y, 0) xformed_x_ng = affine_mtx_ng[:, 0, 0] * nrm_x + \ affine_mtx_ng[:, 0, 1] * nrm_y + \ affine_mtx_ng[:, 0, 2] xformed_y_ng = affine_mtx_ng[:, 1, 0] * nrm_x + \ affine_mtx_ng[:, 1, 1] * nrm_y + \ affine_mtx_ng[:, 1, 2] xformed_x_ng = C.expand_dims(xformed_x_ng, 0) xformed_y_ng = C.expand_dims(xformed_y_ng, 0) fx = C.clip(w * xformed_x, 0, w-2) fy = C.clip(h * xformed_y, 0, h-2) wx = xformed_x - fx wy = xformed_y - fy fx_ng = C.clip(w * xformed_x_ng, 0, w-2) fy_ng = C.clip(h * xformed_y_ng, 0, h-2) chan_idx = np.arange(c).reshape(1, c, 1, 1) chan_idx = C.Constant(chan_idx, chan_idx.shape) batch_idx = np.arange(bs).reshape(bs, 1, 1, 1) batch_idx = C.Constant(batch_idx, batch_idx.shape) flat_im = C.reshape(im, [-1]) def flatten_and_gather(x, y): linear_idx = x + w*y linear_idx = linear_idx + w*h*chan_idx + w*h*c*batch_idx flat_linear_idx = C.reshape(linear_idx, [-1]) return C.reshape(C.gather(flat_im, flat_linear_idx),linear_idx.shape) gather_ff = flatten_and_gather(fx_ng , fy_ng ) gather_fc = flatten_and_gather(fx_ng , fy_ng + 1) gather_cf = flatten_and_gather(fx_ng + 1, fy_ng ) gather_cc = flatten_and_gather(fx_ng + 1, fy_ng + 1) out = gather_ff*(1-wx)*(1-wy) + \ gather_fc*(1-wx)*( wy) + \ gather_cf*( wx)*(1-wy) + \ gather_cc*( wx)*( wy) loss = C.reduce_l2(out) im_val = np.random.randn(bs, c, h, w).astype(np.float32) affine_mtx_val = np.zeros([bs, 2, 3], dtype=np.float32) affine_mtx_val[:, 0, 1] = 1.0 affine_mtx_val[:, 1, 0] = 1.0 # burning iteration for it in range(5): print('burning (', it, ')') g = loss.grad({im : im_val, affine_mtx : affine_mtx_val, affine_mtx_ng : affine_mtx_val}) # actual iterations start = time.time() for it in range(50): print('profiling (', it, ')') g = loss.grad({im : im_val, affine_mtx : affine_mtx_val, affine_mtx_ng : affine_mtx_val}) end = time.time() runtime = (end-start)*1000.0/50.0 print('Runtime:', runtime)