def output(self, x):
d_0 = global_theano_rand.binomial(x.shape,
p=1 - self.d_p_0,
dtype=FLOATX)
d_1 = global_theano_rand.binomial((x.shape[0], self.projection_dim),
p=1 - self.d_p_1,
dtype=FLOATX)
tl_raw = T.dot(x * d_0, self.W_tl)
hl_raw = T.dot(x * d_0, self.W_hl)
tl_mean = T.mean(tl_raw, axis=0)
hl_mean = T.mean(hl_raw, axis=0)
tl_std = T.std(tl_raw, axis=0)
hl_std = T.std(hl_raw, axis=0)
tl = (tl_raw - tl_mean) / (tl_std + self.epsilon)
hl = (hl_raw - hl_mean) / (hl_std + self.epsilon)
new_Mean_tl = self.tau * tl_mean + (1.0 - self.tau) * self.Mean_tl
new_Mean_hl = self.tau * hl_mean + (1.0 - self.tau) * self.Mean_hl
new_Std_tl = self.tau * tl_std + (1.0 - self.tau) * self.Std_tl
new_Std_hl = self.tau * hl_std + (1.0 - self.tau) * self.Std_hl
tr_raw = (tl * d_1).dot(self.W_tr)
hr_raw = (hl * d_1).dot(self.W_hr)
tr_mean = T.mean(tr_raw, axis=0)
hr_mean = T.mean(hr_raw, axis=0)
tr_std = T.std(tr_raw, axis=0)
hr_std = T.std(hr_raw, axis=0)
tr = (tr_raw - tr_mean) / (tr_std + self.epsilon)
hr = (hr_raw - hr_mean) / (hr_std + self.epsilon)
new_Mean_tr = self.tau * tr_mean + (1.0 - self.tau) * self.Mean_tr
new_Mean_hr = self.tau * hr_mean + (1.0 - self.tau) * self.Mean_hr
new_Std_tr = self.tau * tr_std + (1.0 - self.tau) * self.Std_tr
new_Std_hr = self.tau * hr_std + (1.0 - self.tau) * self.Std_hr
t = T.nnet.sigmoid(tr * self.S_t + self.B_t)
h = self._act(hr * self.S_h + self.B_h)
rv = h * t + x * (1 - t)
self.register_training_updates(
(self.Mean_tl, new_Mean_tl), (self.Mean_hl, new_Mean_hl),
(self.Mean_tr, new_Mean_tr), (self.Mean_hr, new_Mean_hr),
(self.Std_tl, new_Std_tl), (self.Std_hl, new_Std_hl),
(self.Std_tr, new_Std_tr), (self.Std_hr, new_Std_hr))
return rv
def output(self, x):
d_0 = global_theano_rand.binomial(x.shape, p=1-self.d_p_0, dtype=FLOATX)
d_1 = global_theano_rand.binomial((x.shape[0], self.projection_dim), p=1-self.d_p_1, dtype=FLOATX)
tl_raw = T.dot(x * d_0, self.W_tl)
hl_raw = T.dot(x * d_0, self.W_hl)
tl_mean = T.mean(tl_raw, axis=0)
hl_mean = T.mean(hl_raw, axis=0)
tl_std = T.std(tl_raw, axis=0)
hl_std = T.std(hl_raw, axis=0)
tl = (tl_raw - tl_mean) / (tl_std + self.epsilon)
hl = (hl_raw - hl_mean) / (hl_std + self.epsilon)
new_Mean_tl = self.tau * tl_mean + (1.0 - self.tau) * self.Mean_tl
new_Mean_hl = self.tau * hl_mean + (1.0 - self.tau) * self.Mean_hl
new_Std_tl = self.tau * tl_std + (1.0 - self.tau) * self.Std_tl
new_Std_hl = self.tau * hl_std + (1.0 - self.tau) * self.Std_hl
tr_raw = (tl * d_1).dot(self.W_tr)
hr_raw = (hl * d_1).dot(self.W_hr)
tr_mean = T.mean(tr_raw, axis=0)
hr_mean = T.mean(hr_raw, axis=0)
tr_std = T.std(tr_raw, axis=0)
hr_std = T.std(hr_raw, axis=0)
tr = (tr_raw - tr_mean) / (tr_std + self.epsilon)
hr = (hr_raw - hr_mean) / (hr_std + self.epsilon)
new_Mean_tr = self.tau * tr_mean + (1.0 - self.tau) * self.Mean_tr
new_Mean_hr = self.tau * hr_mean + (1.0 - self.tau) * self.Mean_hr
new_Std_tr = self.tau * tr_std + (1.0 - self.tau) * self.Std_tr
new_Std_hr = self.tau * hr_std + (1.0 - self.tau) * self.Std_hr
t = T.nnet.sigmoid(tr * self.S_t + self.B_t)
h = self._act(hr * self.S_h + self.B_h)
rv = h * t + x * (1 - t)
self.register_training_updates((self.Mean_tl, new_Mean_tl),
(self.Mean_hl, new_Mean_hl),
(self.Mean_tr, new_Mean_tr),
(self.Mean_hr, new_Mean_hr),
(self.Std_tl, new_Std_tl),
(self.Std_hl, new_Std_hl),
(self.Std_tr, new_Std_tr),
(self.Std_hr, new_Std_hr))
return rv
def compute_tensor(self, x):
if self.p > 0:
# deal with the problem of test_value
backup_test_value_setting = theano.config.compute_test_value
theano.config.compute_test_value = 'ignore'
binomial_mask = global_theano_rand.binomial(x.shape, p=1-self.p, dtype=FLOATX)
theano.config.compute_test_value = backup_test_value_setting
# apply dropout
x *= binomial_mask
return x
def compute_tensor(self, x):
if self.p > 0:
# deal with the problem of test_value
backup_test_value_setting = theano.config.compute_test_value
theano.config.compute_test_value = 'ignore'
binomial_mask = global_theano_rand.binomial(x.shape,
p=1 - self.p,
dtype=FLOATX)
theano.config.compute_test_value = backup_test_value_setting
# apply dropout
x *= binomial_mask
return x
def output(self, x):
if self.p > 0:
x *= global_theano_rand.binomial(x.shape, p=1-self.p, dtype=FLOATX)
return x
