def test_fdrnn_initialize_stds(): m = SupervisedFastDropoutRnn( 50, [50], 50, ['identity'], 'identity', 'squared') inits = dict(par_std=1, par_std_affine=2, par_std_rec=3, par_std_in=4, sparsify_affine=None, sparsify_rec=None, spectral_radius=None) p = m.parameters p.data[:] = float('nan') m.initialize(**inits) assert np.isfinite(p.data).all() tolerance = 1e-1 def works(key, tensor): std = p[tensor].std() target = inits[key] success = target - tolerance < std < target + tolerance print '%g %g' % (std, target) assert success, '%s did not work: %g instead of %g' % ( key, std, target) works('par_std_in', m.rnn.affine_layers[0].weights) for l in m.rnn.affine_layers[1:]: works('par_std_affine', l.weights) for l in m.rnn.recurrent_layers: works('par_std_rec', l.weights)
def test_fd_srnn_iter_fit(): X = np.random.standard_normal((10, 5, 2)).astype(theano.config.floatX) Z = np.random.standard_normal((10, 5, 3)).astype(theano.config.floatX) rnn = SupervisedFastDropoutRnn(2, [10], 3, hidden_transfer='rectifier', max_iter=10) for i, info in enumerate(rnn.iter_fit(X, Z)): if i >= 10: break
def test_fdrnn_initialize_stds(): m = SupervisedFastDropoutRnn(50, [50], 50, ['identity'], 'identity', 'squared') inits = dict(par_std=1, par_std_affine=2, par_std_rec=3, par_std_in=4, sparsify_affine=None, sparsify_rec=None, spectral_radius=None) p = m.parameters p.data[:] = float('nan') m.initialize(**inits) assert np.isfinite(p.data).all() tolerance = 1e-1 def works(key, tensor): std = p[tensor].std() target = inits[key] success = target - tolerance < std < target + tolerance print '%g %g' % (std, target) assert success, '%s did not work: %g instead of %g' % (key, std, target) works('par_std_in', m.rnn.affine_layers[0].weights) for l in m.rnn.affine_layers[1:]: works('par_std_affine', l.weights) for l in m.rnn.recurrent_layers: works('par_std_rec', l.weights)
def test_fd_srnn_fit(): X = np.random.standard_normal((10, 5, 2)).astype(theano.config.floatX) Z = np.random.standard_normal((10, 5, 3)).astype(theano.config.floatX) rnn = SupervisedFastDropoutRnn(2, [10], 3, hidden_transfer='rectifier', max_iter=10) rnn.fit(X, Z)
def test_fd_srnn_predict(): X = np.random.standard_normal((10, 5, 2)).astype(theano.config.floatX) rnn = SupervisedFastDropoutRnn(2, [10], 3, hidden_transfer='rectifier', max_iter=10) print rnn.exprs rnn.predict(X)
def test_fdsrnn_lstm_fit(): X = np.random.standard_normal((13, 5, 4)).astype(theano.config.floatX) Z = np.random.standard_normal((13, 5, 3)).astype(theano.config.floatX) W = np.random.standard_normal((13, 5, 3)).astype(theano.config.floatX) X, Z, W = theano_floatx(X, Z, W) rnn = SupervisedFastDropoutRnn(4, [10], 3, hidden_transfers=['lstm'], max_iter=2) rnn.mode = 'FAST_COMPILE' rnn.fit(X, Z)
def test_fd_srnn_compile(): X = np.random.standard_normal((10, 5, 2)).astype(theano.config.floatX) Z = np.random.standard_normal((10, 5, 3)).astype(theano.config.floatX) W = np.random.standard_normal((10, 5, 3)).astype(theano.config.floatX) X, Z, W = theano_floatx(X, Z, W) rnn = SupervisedFastDropoutRnn(2, [10], 3, hidden_transfers=['rectifier'], max_iter=10) f_loss, f_dloss = rnn._make_loss_functions() f_loss(rnn.parameters.data, X, Z) f_dloss(rnn.parameters.data, X, Z)
def test_fdrnn_pickle(): X = np.random.standard_normal((10, 5, 2)).astype(theano.config.floatX) Z = np.random.standard_normal((10, 5, 3)).astype(theano.config.floatX) W = np.random.standard_normal((10, 5, 3)).astype(theano.config.floatX) X, Z, W = theano_floatx(X, Z, W) rnn = SupervisedFastDropoutRnn(2, [10], 3, hidden_transfers=['rectifier'], max_iter=2) rnn.fit(X, Z) rnn.predict(X) pickle.dumps(rnn)
def test_fd_srnn_predict(): X = np.random.standard_normal((10, 5, 2)).astype(theano.config.floatX) X, = theano_floatx(X) rnn = SupervisedFastDropoutRnn(2, [10], 3, hidden_transfers=['rectifier'], max_iter=10) Y = rnn.predict(X) assert Y.shape[2] == 6 rnn = SupervisedFastDropoutRnn( 2, [10, 20], 3, hidden_transfers=['rectifier', 'tanh'], max_iter=2) Y = rnn.predict(X) assert Y.shape[2] == 6
def test_fd_srnn_iter_fit(): X = np.random.standard_normal((10, 5, 2)).astype(theano.config.floatX) Z = np.random.standard_normal((10, 5, 3)).astype(theano.config.floatX) X, Z = theano_floatx(X, Z) rnn = SupervisedFastDropoutRnn(2, [10], 3, hidden_transfers=['rectifier'], max_iter=10) for i, info in enumerate(rnn.iter_fit(X, Z)): if i >= 10: break rnn = SupervisedFastDropoutRnn( 2, [10, 20], 3, hidden_transfers=['rectifier', 'tanh'], skip_to_out=True, max_iter=10) for i, info in enumerate(rnn.iter_fit(X, Z)): if i >= 10: break
def test_fd_srnn_predict(): X = np.random.standard_normal((10, 5, 2)).astype(theano.config.floatX) X, = theano_floatx(X) rnn = SupervisedFastDropoutRnn(2, [10], 3, hidden_transfers=['rectifier'], max_iter=10) rnn.predict(X) rnn = SupervisedFastDropoutRnn( 2, [10, 20], 3, hidden_transfers=['rectifier', 'tanh'], skip_to_out=True, max_iter=10) rnn.predict(X)
def test_fd_srnn_fit(): raise SkipTest() X = np.random.standard_normal((10, 5, 2)).astype(theano.config.floatX) Z = np.random.standard_normal((10, 5, 3)).astype(theano.config.floatX) W = np.random.standard_normal((10, 5, 3)).astype(theano.config.floatX) X, Z, W = theano_floatx(X, Z, W) rnn = SupervisedFastDropoutRnn(2, [10], 3, hidden_transfers=["rectifier"], max_iter=10) rnn.fit(X, Z) rnn = SupervisedFastDropoutRnn( 2, [10, 20], 3, hidden_transfers=["rectifier", "tanh"], skip_to_out=True, max_iter=10 ) rnn.fit(X, Z) rnn = SupervisedFastDropoutRnn( 2, [10, 20], 3, hidden_transfers=["rectifier", "tanh"], skip_to_out=True, max_iter=10, imp_weight=True ) rnn.fit(X, Z, W)
def test_fd_srnn_fit(): X = np.random.standard_normal((10, 5, 2)).astype(theano.config.floatX) Z = np.random.standard_normal((10, 5, 3)).astype(theano.config.floatX) W = np.random.standard_normal((10, 5, 3)).astype(theano.config.floatX) X, Z, W = theano_floatx(X, Z, W) rnn = SupervisedFastDropoutRnn(2, [10], 3, hidden_transfers=['rectifier'], max_iter=10) rnn.fit(X, Z) rnn = SupervisedFastDropoutRnn( 2, [10, 20], 3, hidden_transfers=['rectifier', 'tanh'], max_iter=2) rnn.fit(X, Z) rnn = SupervisedFastDropoutRnn( 2, [10, 20], 3, hidden_transfers=['rectifier', 'tanh'], max_iter=2, imp_weight=True) rnn.fit(X, Z, W)
def test_fdrnn_initialize_spectral_radius(): m = SupervisedFastDropoutRnn(50, [50], 50, ['identity'], 'identity', 'squared') inits = dict(par_std=1, spectral_radius=2.5) p = m.parameters p.data[:] = float('nan') m.initialize(**inits) assert np.isfinite(p.data).all() rec_layers = [i for i in m.rnn.layers if isinstance(i, FDRecurrent)] tol = .3 for l in rec_layers: val, vec = np.linalg.eig(p[l.weights]) sr = abs(sorted(val)[0]) print abs(sr) isr = inits['spectral_radius'] cond = isr - tol < sr < isr + tol assert cond, 'spectral radius in it did not work for %s: %g' % (l, sr)
def test_fd_srnn_deep(): X = np.random.standard_normal((10, 5, 2)).astype(theano.config.floatX) Z = np.random.standard_normal((10, 5, 3)).astype(theano.config.floatX) W = np.random.standard_normal((10, 5, 3)).astype(theano.config.floatX) X, Z, W = theano_floatx(X, Z, W) p_dropout_hiddens = .2 rnn = SupervisedFastDropoutRnn(2, [10] * 2, 3, hidden_transfers=['rectifier'] * 2, p_dropout_hiddens=p_dropout_hiddens, max_iter=10) n_layers = len([i for i in rnn.rnn.layers if isinstance(i, FDRecurrent)]) assert n_layers == 2
def test_fdrnn_initialize_sparsify(): m = SupervisedFastDropoutRnn(50, [50], 50, ['identity'], 'identity', 'squared') inits = dict(par_std=1, sparsify_affine=20, sparsify_rec=35) p = m.parameters p.data[:] = float('nan') m.initialize(**inits) assert np.isfinite(p.data).all() aff_layers = [i for i in m.rnn.layers if isinstance(i, AffineNonlinear)] rec_layers = [i for i in m.rnn.layers if isinstance(i, FDRecurrent)] for l in aff_layers: w = p[l.weights] cond = ((w != 0).sum(axis=0) == inits['sparsify_affine']).all() assert cond, 'sparsify affine did not work for %s' % l for l in rec_layers: w = p[l.weights] cond = ((w != 0).sum(axis=0) == inits['sparsify_rec']).all() assert cond, 'sparsify recurrent did not work for %s' % l
def test_fdrnn_initialize_spectral_radius(): m = SupervisedFastDropoutRnn( 50, [50], 50, ['identity'], 'identity', 'squared') inits = dict(par_std=1, spectral_radius=2.5) p = m.parameters p.data[:] = float('nan') m.initialize(**inits) assert np.isfinite(p.data).all() rec_layers = [i for i in m.rnn.layers if isinstance(i, FDRecurrent)] tol = .3 for l in rec_layers: val, vec = np.linalg.eig(p[l.weights]) sr = abs(sorted(val)[0]) print abs(sr) isr = inits['spectral_radius'] cond = isr - tol < sr < isr + tol assert cond, 'spectral radius in it did not work for %s: %g' % ( l, sr)
def test_fdrnn_initialize_sparsify(): m = SupervisedFastDropoutRnn( 50, [50], 50, ['identity'], 'identity', 'squared') inits = dict(par_std=1, sparsify_affine=20, sparsify_rec=35) p = m.parameters p.data[:] = float('nan') m.initialize(**inits) assert np.isfinite(p.data).all() aff_layers = [i for i in m.rnn.layers if isinstance(i, AffineNonlinear)] rec_layers = [i for i in m.rnn.layers if isinstance(i, FDRecurrent)] for l in aff_layers: w = p[l.weights] cond = ((w != 0).sum(axis=0) == inits['sparsify_affine']).all() assert cond, 'sparsify affine did not work for %s' % l for l in rec_layers: w = p[l.weights] cond = ((w != 0).sum(axis=0) == inits['sparsify_rec']).all() assert cond, 'sparsify recurrent did not work for %s' % l
def test_fd_srnn_predict(): X = np.random.standard_normal((10, 5, 2)).astype(theano.config.floatX) X, = theano_floatx(X) rnn = SupervisedFastDropoutRnn(2, [10], 3, hidden_transfers=["rectifier"], max_iter=10) rnn.predict(X) rnn = SupervisedFastDropoutRnn( 2, [10, 20], 3, hidden_transfers=["rectifier", "tanh"], skip_to_out=True, max_iter=10 ) rnn.predict(X)
def test_fd_srnn_iter_fit(): raise SkipTest() X = np.random.standard_normal((10, 5, 2)).astype(theano.config.floatX) Z = np.random.standard_normal((10, 5, 3)).astype(theano.config.floatX) X, Z = theano_floatx(X, Z) rnn = SupervisedFastDropoutRnn(2, [10], 3, hidden_transfers=['rectifier'], max_iter=10) for i, info in enumerate(rnn.iter_fit(X, Z)): if i >= 10: break rnn = SupervisedFastDropoutRnn( 2, [10, 20], 3, hidden_transfers=['rectifier', 'tanh'], skip_to_out=True, max_iter=10) for i, info in enumerate(rnn.iter_fit(X, Z)): if i >= 10: break
def test_fd_srnn_fit(): X = np.random.standard_normal((10, 5, 2)).astype(theano.config.floatX) Z = np.random.standard_normal((10, 5, 3)).astype(theano.config.floatX) W = np.random.standard_normal((10, 5, 3)).astype(theano.config.floatX) X, Z, W = theano_floatx(X, Z, W) rnn = SupervisedFastDropoutRnn(2, [10], 3, hidden_transfers=['rectifier'], max_iter=10) rnn.fit(X, Z) rnn = SupervisedFastDropoutRnn(2, [10, 20], 3, hidden_transfers=['rectifier', 'tanh'], max_iter=2) rnn.fit(X, Z) rnn = SupervisedFastDropoutRnn(2, [10, 20], 3, hidden_transfers=['rectifier', 'tanh'], max_iter=2, imp_weight=True) rnn.fit(X, Z, W)