def test_fdrnn_initialize_stds():
m = SupervisedFastDropoutRnn(
50, [50], 50,
['identity'], 'identity', 'squared')
inits = dict(par_std=1, par_std_affine=2, par_std_rec=3, par_std_in=4,
sparsify_affine=None, sparsify_rec=None, spectral_radius=None)
p = m.parameters
p.data[:] = float('nan')
m.initialize(**inits)
assert np.isfinite(p.data).all()
tolerance = 1e-1
def works(key, tensor):
std = p[tensor].std()
target = inits[key]
success = target - tolerance < std < target + tolerance
print '%g %g' % (std, target)
assert success, '%s did not work: %g instead of %g' % (
key, std, target)
works('par_std_in', m.rnn.affine_layers[0].weights)
for l in m.rnn.affine_layers[1:]:
works('par_std_affine', l.weights)
for l in m.rnn.recurrent_layers:
works('par_std_rec', l.weights)
def test_fd_srnn_iter_fit():
X = np.random.standard_normal((10, 5, 2)).astype(theano.config.floatX)
Z = np.random.standard_normal((10, 5, 3)).astype(theano.config.floatX)
rnn = SupervisedFastDropoutRnn(2, [10], 3, hidden_transfer='rectifier', max_iter=10)
for i, info in enumerate(rnn.iter_fit(X, Z)):
if i >= 10:
break
def test_fdrnn_initialize_stds():
m = SupervisedFastDropoutRnn(50, [50], 50, ['identity'], 'identity',
'squared')
inits = dict(par_std=1,
par_std_affine=2,
par_std_rec=3,
par_std_in=4,
sparsify_affine=None,
sparsify_rec=None,
spectral_radius=None)
p = m.parameters
p.data[:] = float('nan')
m.initialize(**inits)
assert np.isfinite(p.data).all()
tolerance = 1e-1
def works(key, tensor):
std = p[tensor].std()
target = inits[key]
success = target - tolerance < std < target + tolerance
print '%g %g' % (std, target)
assert success, '%s did not work: %g instead of %g' % (key, std,
target)
works('par_std_in', m.rnn.affine_layers[0].weights)
for l in m.rnn.affine_layers[1:]:
works('par_std_affine', l.weights)
for l in m.rnn.recurrent_layers:
works('par_std_rec', l.weights)
def test_fd_srnn_fit():
X = np.random.standard_normal((10, 5, 2)).astype(theano.config.floatX)
Z = np.random.standard_normal((10, 5, 3)).astype(theano.config.floatX)
rnn = SupervisedFastDropoutRnn(2, [10],
3,
hidden_transfer='rectifier',
max_iter=10)
rnn.fit(X, Z)
def test_fd_srnn_predict():
X = np.random.standard_normal((10, 5, 2)).astype(theano.config.floatX)
rnn = SupervisedFastDropoutRnn(2, [10],
3,
hidden_transfer='rectifier',
max_iter=10)
print rnn.exprs
rnn.predict(X)
def test_fd_srnn_iter_fit():
X = np.random.standard_normal((10, 5, 2)).astype(theano.config.floatX)
Z = np.random.standard_normal((10, 5, 3)).astype(theano.config.floatX)
rnn = SupervisedFastDropoutRnn(2, [10],
3,
hidden_transfer='rectifier',
max_iter=10)
for i, info in enumerate(rnn.iter_fit(X, Z)):
if i >= 10:
break
def test_fdsrnn_lstm_fit():
X = np.random.standard_normal((13, 5, 4)).astype(theano.config.floatX)
Z = np.random.standard_normal((13, 5, 3)).astype(theano.config.floatX)
W = np.random.standard_normal((13, 5, 3)).astype(theano.config.floatX)
X, Z, W = theano_floatx(X, Z, W)
rnn = SupervisedFastDropoutRnn(4, [10], 3, hidden_transfers=['lstm'],
max_iter=2)
rnn.mode = 'FAST_COMPILE'
rnn.fit(X, Z)
def test_fd_srnn_compile():
X = np.random.standard_normal((10, 5, 2)).astype(theano.config.floatX)
Z = np.random.standard_normal((10, 5, 3)).astype(theano.config.floatX)
W = np.random.standard_normal((10, 5, 3)).astype(theano.config.floatX)
X, Z, W = theano_floatx(X, Z, W)
rnn = SupervisedFastDropoutRnn(2, [10], 3, hidden_transfers=['rectifier'],
max_iter=10)
f_loss, f_dloss = rnn._make_loss_functions()
f_loss(rnn.parameters.data, X, Z)
f_dloss(rnn.parameters.data, X, Z)
def test_fdrnn_pickle():
X = np.random.standard_normal((10, 5, 2)).astype(theano.config.floatX)
Z = np.random.standard_normal((10, 5, 3)).astype(theano.config.floatX)
W = np.random.standard_normal((10, 5, 3)).astype(theano.config.floatX)
X, Z, W = theano_floatx(X, Z, W)
rnn = SupervisedFastDropoutRnn(2, [10], 3, hidden_transfers=['rectifier'],
max_iter=2)
rnn.fit(X, Z)
rnn.predict(X)
pickle.dumps(rnn)
def test_fdsrnn_lstm_fit():
X = np.random.standard_normal((13, 5, 4)).astype(theano.config.floatX)
Z = np.random.standard_normal((13, 5, 3)).astype(theano.config.floatX)
W = np.random.standard_normal((13, 5, 3)).astype(theano.config.floatX)
X, Z, W = theano_floatx(X, Z, W)
rnn = SupervisedFastDropoutRnn(4, [10],
3,
hidden_transfers=['lstm'],
max_iter=2)
rnn.mode = 'FAST_COMPILE'
rnn.fit(X, Z)
def test_fd_srnn_predict():
X = np.random.standard_normal((10, 5, 2)).astype(theano.config.floatX)
X, = theano_floatx(X)
rnn = SupervisedFastDropoutRnn(2, [10], 3, hidden_transfers=['rectifier'],
max_iter=10)
Y = rnn.predict(X)
assert Y.shape[2] == 6
rnn = SupervisedFastDropoutRnn(
2, [10, 20], 3, hidden_transfers=['rectifier', 'tanh'],
max_iter=2)
Y = rnn.predict(X)
assert Y.shape[2] == 6
def test_fd_srnn_compile():
X = np.random.standard_normal((10, 5, 2)).astype(theano.config.floatX)
Z = np.random.standard_normal((10, 5, 3)).astype(theano.config.floatX)
W = np.random.standard_normal((10, 5, 3)).astype(theano.config.floatX)
X, Z, W = theano_floatx(X, Z, W)
rnn = SupervisedFastDropoutRnn(2, [10],
3,
hidden_transfers=['rectifier'],
max_iter=10)
f_loss, f_dloss = rnn._make_loss_functions()
f_loss(rnn.parameters.data, X, Z)
f_dloss(rnn.parameters.data, X, Z)
def test_fd_srnn_iter_fit():
X = np.random.standard_normal((10, 5, 2)).astype(theano.config.floatX)
Z = np.random.standard_normal((10, 5, 3)).astype(theano.config.floatX)
X, Z = theano_floatx(X, Z)
rnn = SupervisedFastDropoutRnn(2, [10], 3, hidden_transfers=['rectifier'], max_iter=10)
for i, info in enumerate(rnn.iter_fit(X, Z)):
if i >= 10:
break
rnn = SupervisedFastDropoutRnn(
2, [10, 20], 3, hidden_transfers=['rectifier', 'tanh'],
skip_to_out=True, max_iter=10)
for i, info in enumerate(rnn.iter_fit(X, Z)):
if i >= 10:
break
def test_fd_srnn_predict():
X = np.random.standard_normal((10, 5, 2)).astype(theano.config.floatX)
X, = theano_floatx(X)
rnn = SupervisedFastDropoutRnn(2, [10], 3, hidden_transfers=['rectifier'], max_iter=10)
rnn.predict(X)
rnn = SupervisedFastDropoutRnn(
2, [10, 20], 3, hidden_transfers=['rectifier', 'tanh'],
skip_to_out=True, max_iter=10)
rnn.predict(X)
def test_fd_srnn_fit():
raise SkipTest()
X = np.random.standard_normal((10, 5, 2)).astype(theano.config.floatX)
Z = np.random.standard_normal((10, 5, 3)).astype(theano.config.floatX)
W = np.random.standard_normal((10, 5, 3)).astype(theano.config.floatX)
X, Z, W = theano_floatx(X, Z, W)
rnn = SupervisedFastDropoutRnn(2, [10], 3, hidden_transfers=["rectifier"], max_iter=10)
rnn.fit(X, Z)
rnn = SupervisedFastDropoutRnn(
2, [10, 20], 3, hidden_transfers=["rectifier", "tanh"], skip_to_out=True, max_iter=10
)
rnn.fit(X, Z)
rnn = SupervisedFastDropoutRnn(
2, [10, 20], 3, hidden_transfers=["rectifier", "tanh"], skip_to_out=True, max_iter=10, imp_weight=True
)
rnn.fit(X, Z, W)
def test_fd_srnn_fit():
X = np.random.standard_normal((10, 5, 2)).astype(theano.config.floatX)
Z = np.random.standard_normal((10, 5, 3)).astype(theano.config.floatX)
W = np.random.standard_normal((10, 5, 3)).astype(theano.config.floatX)
X, Z, W = theano_floatx(X, Z, W)
rnn = SupervisedFastDropoutRnn(2, [10], 3, hidden_transfers=['rectifier'],
max_iter=10)
rnn.fit(X, Z)
rnn = SupervisedFastDropoutRnn(
2, [10, 20], 3, hidden_transfers=['rectifier', 'tanh'], max_iter=2)
rnn.fit(X, Z)
rnn = SupervisedFastDropoutRnn(
2, [10, 20], 3, hidden_transfers=['rectifier', 'tanh'],
max_iter=2, imp_weight=True)
rnn.fit(X, Z, W)
def test_fdrnn_initialize_spectral_radius():
m = SupervisedFastDropoutRnn(50, [50], 50, ['identity'], 'identity',
'squared')
inits = dict(par_std=1, spectral_radius=2.5)
p = m.parameters
p.data[:] = float('nan')
m.initialize(**inits)
assert np.isfinite(p.data).all()
rec_layers = [i for i in m.rnn.layers if isinstance(i, FDRecurrent)]
tol = .3
for l in rec_layers:
val, vec = np.linalg.eig(p[l.weights])
sr = abs(sorted(val)[0])
print abs(sr)
isr = inits['spectral_radius']
cond = isr - tol < sr < isr + tol
assert cond, 'spectral radius in it did not work for %s: %g' % (l, sr)
def test_fd_srnn_deep():
X = np.random.standard_normal((10, 5, 2)).astype(theano.config.floatX)
Z = np.random.standard_normal((10, 5, 3)).astype(theano.config.floatX)
W = np.random.standard_normal((10, 5, 3)).astype(theano.config.floatX)
X, Z, W = theano_floatx(X, Z, W)
p_dropout_hiddens = .2
rnn = SupervisedFastDropoutRnn(2, [10] * 2,
3,
hidden_transfers=['rectifier'] * 2,
p_dropout_hiddens=p_dropout_hiddens,
max_iter=10)
n_layers = len([i for i in rnn.rnn.layers if isinstance(i, FDRecurrent)])
assert n_layers == 2
def test_fdrnn_initialize_sparsify():
m = SupervisedFastDropoutRnn(50, [50], 50, ['identity'], 'identity',
'squared')
inits = dict(par_std=1, sparsify_affine=20, sparsify_rec=35)
p = m.parameters
p.data[:] = float('nan')
m.initialize(**inits)
assert np.isfinite(p.data).all()
aff_layers = [i for i in m.rnn.layers if isinstance(i, AffineNonlinear)]
rec_layers = [i for i in m.rnn.layers if isinstance(i, FDRecurrent)]
for l in aff_layers:
w = p[l.weights]
cond = ((w != 0).sum(axis=0) == inits['sparsify_affine']).all()
assert cond, 'sparsify affine did not work for %s' % l
for l in rec_layers:
w = p[l.weights]
cond = ((w != 0).sum(axis=0) == inits['sparsify_rec']).all()
assert cond, 'sparsify recurrent did not work for %s' % l
def test_fdrnn_initialize_spectral_radius():
m = SupervisedFastDropoutRnn(
50, [50], 50,
['identity'], 'identity', 'squared')
inits = dict(par_std=1, spectral_radius=2.5)
p = m.parameters
p.data[:] = float('nan')
m.initialize(**inits)
assert np.isfinite(p.data).all()
rec_layers = [i for i in m.rnn.layers if isinstance(i, FDRecurrent)]
tol = .3
for l in rec_layers:
val, vec = np.linalg.eig(p[l.weights])
sr = abs(sorted(val)[0])
print abs(sr)
isr = inits['spectral_radius']
cond = isr - tol < sr < isr + tol
assert cond, 'spectral radius in it did not work for %s: %g' % (
l, sr)
def test_fdrnn_initialize_sparsify():
m = SupervisedFastDropoutRnn(
50, [50], 50,
['identity'], 'identity', 'squared')
inits = dict(par_std=1, sparsify_affine=20, sparsify_rec=35)
p = m.parameters
p.data[:] = float('nan')
m.initialize(**inits)
assert np.isfinite(p.data).all()
aff_layers = [i for i in m.rnn.layers if isinstance(i, AffineNonlinear)]
rec_layers = [i for i in m.rnn.layers if isinstance(i, FDRecurrent)]
for l in aff_layers:
w = p[l.weights]
cond = ((w != 0).sum(axis=0) == inits['sparsify_affine']).all()
assert cond, 'sparsify affine did not work for %s' % l
for l in rec_layers:
w = p[l.weights]
cond = ((w != 0).sum(axis=0) == inits['sparsify_rec']).all()
assert cond, 'sparsify recurrent did not work for %s' % l
def test_fd_srnn_predict():
X = np.random.standard_normal((10, 5, 2)).astype(theano.config.floatX)
X, = theano_floatx(X)
rnn = SupervisedFastDropoutRnn(2, [10], 3, hidden_transfers=["rectifier"], max_iter=10)
rnn.predict(X)
rnn = SupervisedFastDropoutRnn(
2, [10, 20], 3, hidden_transfers=["rectifier", "tanh"], skip_to_out=True, max_iter=10
)
rnn.predict(X)
def test_fdrnn_pickle():
X = np.random.standard_normal((10, 5, 2)).astype(theano.config.floatX)
Z = np.random.standard_normal((10, 5, 3)).astype(theano.config.floatX)
W = np.random.standard_normal((10, 5, 3)).astype(theano.config.floatX)
X, Z, W = theano_floatx(X, Z, W)
rnn = SupervisedFastDropoutRnn(2, [10],
3,
hidden_transfers=['rectifier'],
max_iter=2)
rnn.fit(X, Z)
rnn.predict(X)
pickle.dumps(rnn)
def test_fd_srnn_iter_fit():
raise SkipTest()
X = np.random.standard_normal((10, 5, 2)).astype(theano.config.floatX)
Z = np.random.standard_normal((10, 5, 3)).astype(theano.config.floatX)
X, Z = theano_floatx(X, Z)
rnn = SupervisedFastDropoutRnn(2, [10], 3, hidden_transfers=['rectifier'], max_iter=10)
for i, info in enumerate(rnn.iter_fit(X, Z)):
if i >= 10:
break
rnn = SupervisedFastDropoutRnn(
2, [10, 20], 3, hidden_transfers=['rectifier', 'tanh'],
skip_to_out=True, max_iter=10)
for i, info in enumerate(rnn.iter_fit(X, Z)):
if i >= 10:
break
def test_fd_srnn_fit():
X = np.random.standard_normal((10, 5, 2)).astype(theano.config.floatX)
Z = np.random.standard_normal((10, 5, 3)).astype(theano.config.floatX)
rnn = SupervisedFastDropoutRnn(2, [10], 3, hidden_transfer='rectifier', max_iter=10)
rnn.fit(X, Z)
def test_fd_srnn_predict():
X = np.random.standard_normal((10, 5, 2)).astype(theano.config.floatX)
rnn = SupervisedFastDropoutRnn(2, [10], 3, hidden_transfer='rectifier', max_iter=10)
print rnn.exprs
rnn.predict(X)
def test_fd_srnn_fit():
X = np.random.standard_normal((10, 5, 2)).astype(theano.config.floatX)
Z = np.random.standard_normal((10, 5, 3)).astype(theano.config.floatX)
W = np.random.standard_normal((10, 5, 3)).astype(theano.config.floatX)
X, Z, W = theano_floatx(X, Z, W)
rnn = SupervisedFastDropoutRnn(2, [10],
3,
hidden_transfers=['rectifier'],
max_iter=10)
rnn.fit(X, Z)
rnn = SupervisedFastDropoutRnn(2, [10, 20],
3,
hidden_transfers=['rectifier', 'tanh'],
max_iter=2)
rnn.fit(X, Z)
rnn = SupervisedFastDropoutRnn(2, [10, 20],
3,
hidden_transfers=['rectifier', 'tanh'],
max_iter=2,
imp_weight=True)
rnn.fit(X, Z, W)
