def test_conv_zeros(backend_default, zeros_convargs, deltas_buffer): fshape, nofm, batch_size = zeros_convargs NervanaObject.be.bsz = batch_size # basic sanity check with 0 weights random inputs init_unif = Uniform(low=0.0, high=0.0) inshape = (32, 32, 32) insize = np.prod(inshape) neon_layer = Convolution(fshape=(fshape, fshape, nofm), strides=1, padding=0, init=init_unif) inp = neon_layer.be.array(np.random.random((insize, batch_size))) inp.lshape = inshape neon_layer.configure(inshape) neon_layer.prev_layer = True neon_layer.allocate() neon_layer.allocate_deltas(deltas_buffer) deltas_buffer.allocate_buffers() neon_layer.set_deltas(deltas_buffer) out = neon_layer.fprop(inp).get() assert np.min(out) == 0.0 and np.max(out) == 0.0 err = np.zeros(out.shape) deltas = neon_layer.bprop(neon_layer.be.array(err)).get() assert np.min(deltas) == 0.0 and np.max(deltas) == 0.0 dw = neon_layer.dW.get() assert np.min(dw) == 0.0 and np.max(dw) == 0.0 return
def test_conv_zeros(backend_default, zeros_convargs, deltas_buffer): fshape, nofm, batch_size = zeros_convargs NervanaObject.be.bsz = batch_size # basic sanity check with 0 weights random inputs init_unif = Uniform(low=0.0, high=0.0) inshape = (32, 32, 32) insize = np.prod(inshape) neon_layer = Convolution(fshape=(fshape, fshape, nofm), strides=1, padding=0, init=init_unif) inp = neon_layer.be.array(np.random.random((insize, batch_size))) inp.lshape = inshape neon_layer.configure(inshape) neon_layer.prev_layer = True neon_layer.allocate() neon_layer.allocate_deltas(deltas_buffer) deltas_buffer.allocate_buffers() neon_layer.set_deltas(deltas_buffer) out = neon_layer.fprop(inp).get() assert np.min(out) == 0.0 and np.max(out) == 0.0 err = np.zeros(out.shape) deltas = neon_layer.bprop(neon_layer.be.array(err)).get() assert np.min(deltas) == 0.0 and np.max(deltas) == 0.0 dw = neon_layer.dW.get() assert np.min(dw) == 0.0 and np.max(dw) == 0.0 return
def test_conv_rand(backend_default, rand_convargs, deltas_buffer): indim, nifm, fshape, nofm, batch_size, stride, rng_max, w_rng, pad = rand_convargs if isinstance(NervanaObject.be, NervanaGPU) and NervanaObject.be.compute_capability < (5, 0): if nifm % 4 != 0: pytest.skip(msg="C dim must be a multiple of 4 for Kepler bprop kernel") NervanaObject.be.bsz = batch_size inp_rng = [0.0, rng_max] dtypeu = np.float32 init_unif = Uniform(low=w_rng[0], high=w_rng[1]) inshape = (nifm, indim, indim) insize = np.prod(inshape) # generate neon conv layer neon_layer = Convolution(fshape=(fshape, fshape, nofm), strides=stride, padding=pad, init=init_unif) # generate the reference layer ref_layer = ConvLayerRef(1, batch_size, identity, inshape[0], inshape[1:3], (fshape, fshape), nofm, stride, dtypeu, padding=pad) # setup input in range inp_rng inpa = np.random.random((insize, batch_size)) inpa *= inp_rng[1] - inp_rng[0] inpa += inp_rng[0] inpa = inpa.astype(dtypeu) inp = neon_layer.be.array(inpa) inp.lshape = inshape # run fprop on neon neon_layer.configure(inshape) neon_layer.prev_layer = True neon_layer.allocate() neon_layer.allocate_deltas(deltas_buffer) deltas_buffer.allocate_buffers() neon_layer.set_deltas(deltas_buffer) neon_out = neon_layer.fprop(inp).get() # pull neon weights into ref layer weights ref_layer.weights = neon_layer.W.get().T ref_layer.fprop(inpa.T) ref_out = np.copy(ref_layer.y) # estimate the numerical precision by # permuting order of ops in ref layer # fprop calculation ref_layer.fprop(inpa.T, permute=True) ref_out_perm = ref_layer.y atol = 4 * np.max(np.abs(ref_out - ref_out_perm)) # compare ref and neon layer fprop outputs # using the empirically determined atol assert allclose_with_out(ref_out.T, neon_out, atol=atol, rtol=1.e-4) # generate random deltas array erra = np.random.random(neon_out.shape) erra *= (inp_rng[1] - inp_rng[0]) erra += inp_rng[0] erra = erra.astype(dtypeu) err = neon_layer.be.array(erra) # run neon bprop neon_deltas = neon_layer.bprop(err).get() neon_dW = neon_layer.dW.get() # run ref code bprop ref_layer.bprop(erra.T, 1.0) ref_deltas = np.copy(ref_layer.berror_nopad.T) ref_dW = np.copy(ref_layer.updates) # estimate precision using permutation # of operation order on ref layer code ref_layer.bprop(erra.T, 1.0, permute=True) ref_deltas_perm = ref_layer.berror_nopad.T ref_dW_perm = ref_layer.updates atol = 4 * np.max(np.abs(ref_deltas - ref_deltas_perm)) assert allclose_with_out(ref_deltas, neon_deltas, atol=atol, rtol=1.e-4) atol = 4 * np.max(np.abs(ref_dW - ref_dW_perm)) assert allclose_with_out(ref_dW.T, neon_dW, atol=atol, rtol=1.e-4) return
def test_conv_ones(backend_default, ones_convargs, deltas_buffer): dtypeu = np.float32 indim, nifm, fshape, nofm, batch_size, stride, pad = ones_convargs if isinstance(NervanaObject.be, NervanaGPU) and NervanaObject.be.compute_capability < (5, 0): if nifm % 4 != 0: pytest.skip(msg="C dim must be a multiple of 4 for Kepler bprop kernel") NervanaObject.be.bsz = batch_size # weights set to one init_unif = Uniform(low=1.0, high=1.0) inshape = (nifm, indim, indim) insize = np.prod(inshape) neon_layer = Convolution(fshape=(fshape, fshape, nofm), strides=stride, padding=pad, init=init_unif) inp = neon_layer.be.array(np.ones((insize, batch_size))) inp.lshape = inshape neon_layer.configure(inshape) neon_layer.prev_layer = True neon_layer.allocate() neon_layer.allocate_deltas(deltas_buffer) deltas_buffer.allocate_buffers() neon_layer.set_deltas(deltas_buffer) # run fprop out = neon_layer.fprop(inp).get() # generate the reference layer ref_layer = ConvLayerRef(1, batch_size, identity, inshape[0], inshape[1:3], (fshape, fshape), nofm, stride, dtypeu, padding=pad) # init weights to ones ref_layer.weights = np.ones(neon_layer.W.shape).T.astype(dtypeu) ref_layer.fprop(inp.get().T) out_exp = ref_layer.y.copy() assert allclose_with_out(out_exp.T, out, atol=0.0, rtol=0.0) # generate err array err = np.ones(out.shape).astype(np.float32) # run bprop neon_layer.bprop(neon_layer.be.array(err)) dw = neon_layer.dW.get() # run bprop ref_layer.bprop(err.T.astype(dtypeu), 1.0) # expected output for updates is uniform matrix with # all elements == ofmsize*batch_size updates_exp = ref_layer.updates.T # check dw from neon layer assert allclose_with_out(dw, updates_exp, atol=0.0, rtol=0.0) # the deltas are more complicated since the matricies are not # uniform, going to use the reference code directly here # no tolerance here should be exact dd = np.abs(ref_layer.berror_nopad.T - neon_layer.deltas.get()) try: assert np.max(dd) == 0.0 except AssertionError: if ones_convargs in ((32, 32, 3, 32, 64, 2, 0), (32, 32, 3, 16, 64, 2, 0), (32, 32, 3, 64, 64, 2, 0)): pytest.xfail(reason="xfail before mkl update. issue: #1020") else: assert np.max(dd) == 0.0 return
def test_conv_rand(backend_default, rand_convargs, deltas_buffer): indim, nifm, fshape, nofm, batch_size, stride, rng_max, w_rng, pad = rand_convargs if isinstance(NervanaObject.be, NervanaGPU) and NervanaObject.be.compute_capability < (5, 0): if nifm % 4 != 0: pytest.skip(msg="C dim must be a multiple of 4 for Kepler bprop kernel") NervanaObject.be.bsz = batch_size inp_rng = [0.0, rng_max] dtypeu = np.float32 init_unif = Uniform(low=w_rng[0], high=w_rng[1]) inshape = (nifm, indim, indim) insize = np.prod(inshape) # generate neon conv layer neon_layer = Convolution(fshape=(fshape, fshape, nofm), strides=stride, padding=pad, init=init_unif) # generate the reference layer ref_layer = ConvLayerRef(1, batch_size, identity, inshape[0], inshape[1:3], (fshape, fshape), nofm, stride, dtypeu, padding=pad) # setup input in range inp_rng inpa = np.random.random((insize, batch_size)) inpa *= inp_rng[1] - inp_rng[0] inpa += inp_rng[0] inpa = inpa.astype(dtypeu) inp = neon_layer.be.array(inpa) inp.lshape = inshape # run fprop on neon neon_layer.configure(inshape) neon_layer.prev_layer = True neon_layer.allocate() neon_layer.allocate_deltas(deltas_buffer) deltas_buffer.allocate_buffers() neon_layer.set_deltas(deltas_buffer) neon_out = neon_layer.fprop(inp).get() # pull neon weights into ref layer weights ref_layer.weights = neon_layer.W.get().T ref_layer.fprop(inpa.T) ref_out = np.copy(ref_layer.y) # estimate the numerical precision by # permuting order of ops in ref layer # fprop calculation ref_layer.fprop(inpa.T, permute=True) ref_out_perm = ref_layer.y atol = 4 * np.max(np.abs(ref_out - ref_out_perm)) # compare ref and neon layer fprop outputs # using the empirically determined atol assert allclose_with_out(ref_out.T, neon_out, atol=atol, rtol=1.e-4) # generate random deltas array erra = np.random.random(neon_out.shape) erra *= (inp_rng[1] - inp_rng[0]) erra += inp_rng[0] erra = erra.astype(dtypeu) err = neon_layer.be.array(erra) # run neon bprop neon_deltas = neon_layer.bprop(err).get() neon_dW = neon_layer.dW.get() # run ref code bprop ref_layer.bprop(erra.T, 1.0) ref_deltas = np.copy(ref_layer.berror_nopad.T) ref_dW = np.copy(ref_layer.updates) # estimate precision using permutation # of operation order on ref layer code ref_layer.bprop(erra.T, 1.0, permute=True) ref_deltas_perm = ref_layer.berror_nopad.T ref_dW_perm = ref_layer.updates atol = 4 * np.max(np.abs(ref_deltas - ref_deltas_perm)) assert allclose_with_out(ref_deltas, neon_deltas, atol=atol, rtol=1.e-4) atol = 4 * np.max(np.abs(ref_dW - ref_dW_perm)) assert allclose_with_out(ref_dW.T, neon_dW, atol=atol, rtol=1.e-4) return
def test_conv_ones(backend_default, ones_convargs, deltas_buffer): dtypeu = np.float32 indim, nifm, fshape, nofm, batch_size, stride, pad = ones_convargs if isinstance(NervanaObject.be, NervanaGPU) and NervanaObject.be.compute_capability < (5, 0): if nifm % 4 != 0: pytest.skip(msg="C dim must be a multiple of 4 for Kepler bprop kernel") NervanaObject.be.bsz = batch_size # weights set to one init_unif = Uniform(low=1.0, high=1.0) inshape = (nifm, indim, indim) insize = np.prod(inshape) neon_layer = Convolution(fshape=(fshape, fshape, nofm), strides=stride, padding=pad, init=init_unif) inp = neon_layer.be.array(np.ones((insize, batch_size))) inp.lshape = inshape neon_layer.configure(inshape) neon_layer.prev_layer = True neon_layer.allocate() neon_layer.allocate_deltas(deltas_buffer) deltas_buffer.allocate_buffers() neon_layer.set_deltas(deltas_buffer) # run fprop out = neon_layer.fprop(inp).get() # generate the reference layer ref_layer = ConvLayerRef(1, batch_size, identity, inshape[0], inshape[1:3], (fshape, fshape), nofm, stride, dtypeu, padding=pad) # init weights to ones ref_layer.weights = np.ones(neon_layer.W.shape).T.astype(dtypeu) ref_layer.fprop(inp.get().T) out_exp = ref_layer.y.copy() assert np.allclose(out_exp.T, out, atol=0.0, rtol=0.0) # generate err array err = np.ones(out.shape).astype(np.float32) # run bprop neon_layer.bprop(neon_layer.be.array(err)) dw = neon_layer.dW.get() # run bprop ref_layer.bprop(err.T.astype(dtypeu), 1.0) # expected output for updates is uniform matrix with # all elements == ofmsize*batch_size updates_exp = ref_layer.updates.T # check dw from neon layer assert np.allclose(dw, updates_exp, atol=0.0, rtol=0.0) # the deltas are more complicated since the matricies are not # uniform, going to use the reference code directly here # no tolerance here should be exact dd = np.abs(ref_layer.berror_nopad.T - neon_layer.deltas.get()) assert np.max(dd) == 0.0 return