def test_select_proportional_to_weight(self): # Tests that multinomial_wo_replacement selects elements, on average, # proportional to the their probabilities th_rng = RandomStream(12345) p = tensor.fmatrix() n = tensor.iscalar() m = th_rng.multinomial_wo_replacement(pvals=p, n=n) f = function([p, n], m, allow_input_downcast=True) n_elements = 100 n_selected = 10 mean_rtol = 0.0005 np.random.seed(12345) pvals = np.random.randint(1, 100, (1, n_elements)).astype(config.floatX) pvals /= pvals.sum(1) avg_pvals = np.zeros((n_elements, ), dtype=config.floatX) for rep in range(10000): res = f(pvals, n_selected) res = np.squeeze(res) avg_pvals[res] += 1 avg_pvals /= avg_pvals.sum() avg_diff = np.mean(abs(avg_pvals - pvals)) assert avg_diff < mean_rtol
def test_get_substream_rstates(): with config.change_flags(compute_test_value="raise"): n_streams = 100 dtype = "float32" rng = MRG_RandomStream(np.random.randint(2147462579)) rng.get_substream_rstates(n_streams, dtype)
def test_f16_nonzero(mode=None, op_to_check=rng_mrg.mrg_uniform): srng = MRG_RandomStream(seed=utt.fetch_seed()) m = srng.uniform(size=(1000, 1000), dtype="float16") assert m.dtype == "float16", m.type f = theano.function([], m, mode=mode) assert any( isinstance(n.op, op_to_check) for n in f.maker.fgraph.apply_nodes) m_val = f() assert np.all((0 < m_val) & (m_val < 1))
def test_multiple_rng_aliasing(): # Test that when we have multiple random number generators, we do not alias # the state_updates member. `state_updates` can be useful when attempting to # copy the (random) state between two similar theano graphs. The test is # meant to detect a previous bug where state_updates was initialized as a # class-attribute, instead of the __init__ function. rng1 = MRG_RandomStream(1234) rng2 = MRG_RandomStream(2392) assert rng1.state_updates is not rng2.state_updates
def test_undefined_grad_opt(): # Make sure that undefined grad get removed in optimized graph. random = MRG_RandomStream(np.random.randint(1, 2147462579)) pvals = theano.shared(np.random.rand(10, 20).astype(theano.config.floatX)) pvals = pvals / pvals.sum(axis=1) pvals = theano.gradient.zero_grad(pvals) samples = random.multinomial(pvals=pvals, n=1) samples = theano.tensor.cast(samples, pvals.dtype) samples = theano.gradient.zero_grad(samples) cost = theano.tensor.sum(samples + pvals) grad = theano.tensor.grad(cost, samples) f = theano.function([], grad) assert not any([ isinstance(node.op, theano.gradient.UndefinedGrad) for node in f.maker.fgraph.apply_nodes ])
def test_dump_load_mrg(self): rng = MRG_RandomStream() with open("test", "wb") as f: dump(rng, f) with open("test", "rb") as f: rng = load(f) assert type(rng) == MRG_RandomStream
def test_fail_select_alot(self): # Tests that multinomial_wo_replacement fails when asked to sample more # elements than the actual number of elements th_rng = RandomStream(12345) p = tensor.fmatrix() n = tensor.iscalar() m = th_rng.multinomial_wo_replacement(pvals=p, n=n) f = function([p, n], m, allow_input_downcast=True) n_elements = 100 n_selected = 200 np.random.seed(12345) pvals = np.random.randint(1, 100, (1, n_elements)).astype(config.floatX) pvals /= pvals.sum(1) with pytest.raises(ValueError): f(pvals, n_selected)
def test_simple_shared_mrg_random(): theano_rng = MRG_RandomStream(10) values, updates = theano.scan( lambda: theano_rng.uniform((2, ), -1, 1), [], [], [], n_steps=5, truncate_gradient=-1, go_backwards=False, ) my_f = theano.function([], values, updates=updates, allow_input_downcast=True) # Just check for run-time errors my_f() my_f()
def test_cpu_target_with_shared_variable(): srng = MRG_RandomStream() s = np.random.rand(2, 3).astype("float32") x = gpuarray_shared_constructor(s, name="x") try: # To have theano.shared(x) try to move on the GPU theano.compile.shared_constructor(gpuarray_shared_constructor) y = srng.uniform(x.shape, target="cpu") y.name = "y" z = (x * y).sum() z.name = "z" fz = theano.function([], z, mode=mode) nodes = fz.maker.fgraph.toposort() assert not any( [isinstance(node.op, GPUA_mrg_uniform) for node in nodes]) finally: theano.compile.shared_constructor(gpuarray_shared_constructor, remove=True)
def test_select_distinct(self): # Tests that multinomial_wo_replacement always selects distinct elements th_rng = RandomStream(12345) p = tensor.fmatrix() n = tensor.iscalar() m = th_rng.multinomial_wo_replacement(pvals=p, n=n) f = function([p, n], m, allow_input_downcast=True) n_elements = 1000 all_indices = range(n_elements) np.random.seed(12345) for i in [5, 10, 50, 100, 500, n_elements]: pvals = np.random.randint(1, 100, (1, n_elements)).astype(config.floatX) pvals /= pvals.sum(1) res = f(pvals, i) res = np.squeeze(res) assert len(res) == i assert np.all(np.in1d(np.unique(res), all_indices)), res
def test_GPUA_full_fill(): # Make sure the whole sample buffer is filled. Also make sure # large samples are consistent with CPU results. # This needs to be large to trigger the problem on GPU size = (10, 1000) R = MRG_RandomStream(234) uni = R.uniform(size, nstreams=60 * 256) f_cpu = theano.function([], uni) rstate_gpu = gpuarray_shared_constructor( R.state_updates[-1][0].get_value()) new_rstate, sample = GPUA_mrg_uniform.new(rstate_gpu, ndim=None, dtype="float32", size=size) rstate_gpu.default_update = new_rstate f_gpu = theano.function([], sample, mode=mode) utt.assert_allclose(f_cpu(), f_gpu())
def test_multinomial(): steps = 100 if (config.mode in ["DEBUG_MODE", "DebugMode", "FAST_COMPILE"] or config.mode == "Mode" and config.linker in ["py"]): sample_size = (49, 5) else: sample_size = (450, 6) pvals = np.asarray(np.random.uniform(size=sample_size)) pvals = np.apply_along_axis(lambda row: row / np.sum(row), 1, pvals) R = MRG_RandomStream(234) # Note: we specify `nstreams` to avoid a warning. m = R.multinomial(pvals=pvals, dtype=config.floatX, nstreams=30 * 256) f = theano.function([], m) f() basic_multinomialtest(f, steps, sample_size, pvals, n_samples=1, prefix="mrg ")
def test_target_parameter(): srng = MRG_RandomStream() pvals = np.array([[0.98, 0.01, 0.01], [0.01, 0.49, 0.50]]) def basic_target_parameter_test(x): f = theano.function([], x) assert isinstance(f(), np.ndarray) basic_target_parameter_test(srng.uniform((3, 2), target="cpu")) basic_target_parameter_test(srng.normal((3, 2), target="cpu")) basic_target_parameter_test(srng.truncated_normal((3, 2), target="cpu")) basic_target_parameter_test(srng.binomial((3, 2), target="cpu")) basic_target_parameter_test( srng.multinomial(pvals=pvals.astype("float32"), target="cpu")) basic_target_parameter_test( srng.choice(p=pvals.astype("float32"), replace=False, target="cpu")) basic_target_parameter_test( srng.multinomial_wo_replacement(pvals=pvals.astype("float32"), target="cpu"))
def test_gradient_scan(): # Test for a crash when using MRG inside scan and taking the gradient # See https://groups.google.com/d/msg/theano-dev/UbcYyU5m-M8/UO9UgXqnQP0J theano_rng = MRG_RandomStream(10) w = theano.shared(np.ones(1, dtype="float32")) def one_step(x): return x + theano_rng.uniform((1, ), dtype="float32") * w x = tensor.vector(dtype="float32") values, updates = theano.scan(one_step, outputs_info=x, n_steps=10) gw = theano.grad(tensor.sum(values[-1]), w) f = theano.function([x], gw) f(np.arange(1, dtype="float32"))
def test_consistency_randomstreams(): # Verify that the random numbers generated by MRG_RandomStream # are the same as the reference (Java) implementation by L'Ecuyer et al. seed = 12345 n_samples = 5 n_streams = 12 n_substreams = 7 samples = [] rng = MRG_RandomStream(seed=seed) for i in range(n_streams): stream_samples = [] u = rng.uniform(size=(n_substreams, ), nstreams=n_substreams) f = theano.function([], u) for j in range(n_samples): s = f() stream_samples.append(s) stream_samples = np.array(stream_samples) stream_samples = stream_samples.T.flatten() samples.append(stream_samples) samples = np.array(samples).flatten() assert np.allclose(samples, java_samples)
def check_binomial(mean, size, const_size, var_input, input, steps, rtol): R = MRG_RandomStream(234) u = R.binomial(size=size, p=mean) f = theano.function(var_input, u) f(*input) # Increase the number of steps if sizes implies only a few samples if np.prod(const_size) < 10: steps_ = steps * 100 else: steps_ = steps check_basics( f, steps_, const_size, prefix="mrg cpu", inputs=input, allow_01=True, target_avg=mean, mean_rtol=rtol, ) RR = theano.tensor.random.utils.RandomStream(234) uu = RR.binomial(1, mean, size=size) ff = theano.function(var_input, uu) # It's not our problem if numpy generates 0 or 1 check_basics( ff, steps_, const_size, prefix="numpy", allow_01=True, inputs=input, target_avg=mean, mean_rtol=rtol, )
def test_broadcastable(): R = MRG_RandomStream(234) x = tensor.matrix() size1 = (10, 1) size2 = (x.shape[0], 1) pvals_1 = np.random.uniform(0, 1, size=size1) pvals_1 = pvals_1 / sum(pvals_1) pvals_2 = R.uniform(size=size2) pvals_2 = pvals_2 / tensor.sum(pvals_2) for distribution in [ R.uniform, R.normal, R.truncated_normal, R.binomial, R.multinomial, R.multinomial_wo_replacement, ]: # multinomial or multinomial_wo_replacement does not support "size" argument, # the sizes of them are implicitly defined with "pvals" argument. if distribution in [R.multinomial, R.multinomial_wo_replacement]: # check when all dimensions are constant uu = distribution(pvals=pvals_1) assert uu.broadcastable == (False, True) # check when some dimensions are theano variables uu = distribution(pvals=pvals_2) assert uu.broadcastable == (False, True) else: # check when all dimensions are constant uu = distribution(size=size1) assert uu.broadcastable == (False, True) # check when some dimensions are theano variables uu = distribution(size=size2) assert uu.broadcastable == (False, True)
def test_multinomial_n_samples(): if (config.mode in ["DEBUG_MODE", "DebugMode", "FAST_COMPILE"] or config.mode == "Mode" and config.linker in ["py"]): sample_size = (49, 5) else: sample_size = (450, 6) pvals = np.asarray(np.random.uniform(size=sample_size)) pvals = np.apply_along_axis(lambda row: row / np.sum(row), 1, pvals) R = MRG_RandomStream(234) for n_samples, steps in zip([5, 10, 100, 1000], [20, 10, 1, 1]): m = R.multinomial(pvals=pvals, n=n_samples, dtype=config.floatX, nstreams=30 * 256) f = theano.function([], m) basic_multinomialtest(f, steps, sample_size, pvals, n_samples, prefix="mrg ") sys.stdout.flush()
def test_seed_fn(): idx = tensor.ivector() for new_seed, same in [(234, True), (None, True), (23, False)]: random = MRG_RandomStream(234) fn1 = theano.function([], random.uniform((2, 2), dtype="float32")) fn2 = theano.function([], random.uniform((3, 3), nstreams=2, dtype="float32")) fn3 = theano.function([idx], random.uniform(idx, nstreams=3, ndim=1, dtype="float32")) fn1_val0 = fn1() fn1_val1 = fn1() assert not np.allclose(fn1_val0, fn1_val1) fn2_val0 = fn2() fn2_val1 = fn2() assert not np.allclose(fn2_val0, fn2_val1) fn3_val0 = fn3([4]) fn3_val1 = fn3([4]) assert not np.allclose(fn3_val0, fn3_val1) assert fn1_val0.size == 4 assert fn2_val0.size == 9 random.seed(new_seed) fn1_val2 = fn1() fn1_val3 = fn1() fn2_val2 = fn2() fn2_val3 = fn2() fn3_val2 = fn3([4]) fn3_val3 = fn3([4]) assert np.allclose(fn1_val0, fn1_val2) == same assert np.allclose(fn1_val1, fn1_val3) == same assert np.allclose(fn2_val0, fn2_val2) == same assert np.allclose(fn2_val1, fn2_val3) == same assert np.allclose(fn3_val0, fn3_val2) == same assert np.allclose(fn3_val1, fn3_val3) == same
def test_overflow_cpu(): # run with THEANO_FLAGS=mode=FAST_RUN,device=cpu,floatX=float32 rng = MRG_RandomStream(np.random.randint(1234)) fct = rng.uniform with config.change_flags(compute_test_value="off"): # should raise error as the size overflows sizes = [ (2**31, ), (2**32, ), ( 2**15, 2**16, ), (2, 2**15, 2**15), ] rng_mrg_overflow(sizes, fct, config.mode, should_raise_error=True) # should not raise error sizes = [(2**5, ), (2**5, 2**5), (2**5, 2**5, 2**5)] rng_mrg_overflow(sizes, fct, config.mode, should_raise_error=False) # should support int32 sizes sizes = [(np.int32(2**10), ), (np.int32(2), np.int32(2**10), np.int32(2**10))] rng_mrg_overflow(sizes, fct, config.mode, should_raise_error=False)
def test_deterministic(): seed = utt.fetch_seed() sample_size = (10, 20) R = MRG_RandomStream(seed=seed) u = R.uniform(size=sample_size) f = theano.function([], u) fsample1 = f() fsample2 = f() assert not np.allclose(fsample1, fsample2) R2 = MRG_RandomStream(seed=seed) u2 = R2.uniform(size=sample_size) g = theano.function([], u2) gsample1 = g() gsample2 = g() assert np.allclose(fsample1, gsample1) assert np.allclose(fsample2, gsample2)
def test_undefined_grad(): srng = MRG_RandomStream(seed=1234) # checking uniform distribution low = tensor.scalar() out = srng.uniform((), low=low) with pytest.raises(theano.gradient.NullTypeGradError): theano.grad(out, low) high = tensor.scalar() out = srng.uniform((), low=0, high=high) with pytest.raises(theano.gradient.NullTypeGradError): theano.grad(out, high) out = srng.uniform((), low=low, high=high) with pytest.raises(theano.gradient.NullTypeGradError): theano.grad(out, (low, high)) # checking binomial distribution prob = tensor.scalar() out = srng.binomial((), p=prob) with pytest.raises(theano.gradient.NullTypeGradError): theano.grad(out, prob) # checking multinomial distribution prob1 = tensor.scalar() prob2 = tensor.scalar() p = [theano.tensor.as_tensor_variable([prob1, 0.5, 0.25])] out = srng.multinomial(size=None, pvals=p, n=4)[0] with pytest.raises(theano.gradient.NullTypeGradError): theano.grad(theano.tensor.sum(out), prob1) p = [theano.tensor.as_tensor_variable([prob1, prob2])] out = srng.multinomial(size=None, pvals=p, n=4)[0] with pytest.raises(theano.gradient.NullTypeGradError): theano.grad(theano.tensor.sum(out), (prob1, prob2)) # checking choice p = [theano.tensor.as_tensor_variable([prob1, prob2, 0.1, 0.2])] out = srng.choice(a=None, size=1, p=p, replace=False)[0] with pytest.raises(theano.gradient.NullTypeGradError): theano.grad(out[0], (prob1, prob2)) p = [theano.tensor.as_tensor_variable([prob1, prob2])] out = srng.choice(a=None, size=1, p=p, replace=False)[0] with pytest.raises(theano.gradient.NullTypeGradError): theano.grad(out[0], (prob1, prob2)) p = [theano.tensor.as_tensor_variable([prob1, 0.2, 0.3])] out = srng.choice(a=None, size=1, p=p, replace=False)[0] with pytest.raises(theano.gradient.NullTypeGradError): theano.grad(out[0], prob1) # checking normal distribution avg = tensor.scalar() out = srng.normal((), avg=avg) with pytest.raises(theano.gradient.NullTypeGradError): theano.grad(out, avg) std = tensor.scalar() out = srng.normal((), avg=0, std=std) with pytest.raises(theano.gradient.NullTypeGradError): theano.grad(out, std) out = srng.normal((), avg=avg, std=std) with pytest.raises(theano.gradient.NullTypeGradError): theano.grad(out, (avg, std)) # checking truncated normal distribution avg = tensor.scalar() out = srng.truncated_normal((), avg=avg) with pytest.raises(theano.gradient.NullTypeGradError): theano.grad(out, avg) std = tensor.scalar() out = srng.truncated_normal((), avg=0, std=std) with pytest.raises(theano.gradient.NullTypeGradError): theano.grad(out, std) out = srng.truncated_normal((), avg=avg, std=std) with pytest.raises(theano.gradient.NullTypeGradError): theano.grad(out, (avg, std))
def test_uniform(): # TODO: test param low, high # TODO: test size=None # TODO: test ndim!=size.ndim # TODO: test bad seed # TODO: test size=Var, with shape that change from call to call if (config.mode in ["DEBUG_MODE", "DebugMode", "FAST_COMPILE"] or config.mode == "Mode" and config.linker in ["py"]): sample_size = (10, 100) steps = 50 else: sample_size = (500, 50) steps = int(1e3) x = tensor.matrix() for size, const_size, var_input, input in [ (sample_size, sample_size, [], []), (x.shape, sample_size, [x], [np.zeros(sample_size, dtype=config.floatX)]), ( (x.shape[0], sample_size[1]), sample_size, [x], [np.zeros(sample_size, dtype=config.floatX)], ), # test empty size (scalar) ((), (), [], []), ]: # TEST CPU IMPLEMENTATION # The python and C implementation are tested with DebugMode x = tensor.matrix() R = MRG_RandomStream(234) # Note: we specify `nstreams` to avoid a warning. # TODO Look for all occurrences of `guess_n_streams` and `30 * 256` # for such situations: it would be better to instead filter the # warning using the warning module. u = R.uniform(size=size, nstreams=rng_mrg.guess_n_streams(size, warn=False)) f = theano.function(var_input, u) assert any([ isinstance(node.op, theano.sandbox.rng_mrg.mrg_uniform) for node in f.maker.fgraph.toposort() ]) f(*input) # Increase the number of steps if sizes implies only a few samples if np.prod(const_size) < 10: steps_ = steps * 100 else: steps_ = steps check_basics(f, steps_, const_size, prefix="mrg cpu", inputs=input) RR = theano.tensor.random.utils.RandomStream(234) uu = RR.uniform(size=size) ff = theano.function(var_input, uu) # It's not our problem if numpy generates 0 or 1 check_basics(ff, steps_, const_size, prefix="numpy", allow_01=True, inputs=input)
def test_bad_size(self): R = MRG_RandomStream(234) for size in [ (0, 100), (-1, 100), (1, 0), ]: with pytest.raises(ValueError): R.uniform(size) with pytest.raises(ValueError): R.binomial(size) with pytest.raises(ValueError): R.multinomial(size, 1, []) with pytest.raises(ValueError): R.normal(size) with pytest.raises(ValueError): R.truncated_normal(size)
def test_truncated_normal(): # just a copy of test_normal0 for truncated normal steps = 50 std = 2.0 if (config.mode in ["DEBUG_MODE", "DebugMode", "FAST_COMPILE"] or config.mode == "Mode" and config.linker in ["py"]): sample_size = (25, 30) default_rtol = 0.02 else: sample_size = (999, 50) default_rtol = 0.01 sample_size_odd = (sample_size[0], sample_size[1] - 1) x = tensor.matrix() test_cases = [ (sample_size, sample_size, [], [], -5.0, default_rtol, default_rtol), ( x.shape, sample_size, [x], [np.zeros(sample_size, dtype=config.floatX)], -5.0, default_rtol, default_rtol, ), # test odd value ( x.shape, sample_size_odd, [x], [np.zeros(sample_size_odd, dtype=config.floatX)], -5.0, default_rtol, default_rtol, ), ( sample_size, sample_size, [], [], np.arange(np.prod(sample_size), dtype="float32").reshape(sample_size), 10.0 * std / np.sqrt(steps), default_rtol, ), # test empty size (scalar) ((), (), [], [], -5.0, default_rtol, 0.02), # test with few samples at the same time ((1, ), (1, ), [], [], -5.0, default_rtol, 0.02), ((3, ), (3, ), [], [], -5.0, default_rtol, 0.02), ] for size, const_size, var_input, input, avg, rtol, std_tol in test_cases: R = MRG_RandomStream(234) # Note: we specify `nstreams` to avoid a warning. n = R.truncated_normal( size=size, avg=avg, std=std, nstreams=rng_mrg.guess_n_streams(size, warn=False), ) f = theano.function(var_input, n) # Increase the number of steps if size implies only a few samples if np.prod(const_size) < 10: steps_ = steps * 60 else: steps_ = steps check_basics( f, steps_, const_size, target_avg=avg, target_std=std, prefix="mrg ", allow_01=True, inputs=input, mean_rtol=rtol, std_tol=std_tol, ) sys.stdout.flush()
def __init__(self, seed=123): self.rng = MRG_RandomStream(seed) self.y = self.rng.uniform(size=(1, ))
def test_normal_truncation(): # just a copy of test_normal0 with extra bound check steps = 50 std = 2.0 # standard deviation is slightly less than for a regular Gaussian # constant taken from scipy.stats.truncnorm.std(a=-2, b=2, loc=0., scale=1.) target_std = 0.87962566103423978 * std if (config.mode in ["DEBUG_MODE", "DebugMode", "FAST_COMPILE"] or config.mode == "Mode" and config.linker in ["py"]): sample_size = (25, 30) default_rtol = 0.02 else: sample_size = (999, 50) default_rtol = 0.01 sample_size_odd = (sample_size[0], sample_size[1] - 1) x = tensor.matrix() test_cases = [ (sample_size, sample_size, [], [], -5.0, default_rtol, default_rtol), ( x.shape, sample_size, [x], [np.zeros(sample_size, dtype=config.floatX)], -5.0, default_rtol, default_rtol, ), # test odd value ( x.shape, sample_size_odd, [x], [np.zeros(sample_size_odd, dtype=config.floatX)], -5.0, default_rtol, default_rtol, ), ( sample_size, sample_size, [], [], np.arange(np.prod(sample_size), dtype="float32").reshape(sample_size), 10.0 * std / np.sqrt(steps), default_rtol, ), # test empty size (scalar) ((), (), [], [], -5.0, default_rtol, 0.02), # test with few samples at the same time ((1, ), (1, ), [], [], -5.0, default_rtol, 0.02), ((3, ), (3, ), [], [], -5.0, default_rtol, 0.02), ] for size, const_size, var_input, input, avg, rtol, std_tol in test_cases: R = MRG_RandomStream(234) # Note: we specify `nstreams` to avoid a warning. n = R.normal( size=size, avg=avg, std=std, truncate=True, nstreams=rng_mrg.guess_n_streams(size, warn=False), ) f = theano.function(var_input, n) # check if truncated at 2*std samples = f(*input) assert np.all( avg + 2 * std - samples >= 0), "bad upper bound? {} {}".format( samples, avg + 2 * std, ) assert np.all(samples - (avg - 2 * std) >= 0), "bad lower bound? {} {}".format( samples, avg - 2 * std, ) # Increase the number of steps if size implies only a few samples if np.prod(const_size) < 10: steps_ = steps * 50 else: steps_ = steps check_basics( f, steps_, const_size, target_avg=avg, target_std=target_std, prefix="mrg ", allow_01=True, inputs=input, mean_rtol=rtol, std_tol=std_tol, ) sys.stdout.flush()