def _init_params(self):
shape_xh = (self.n_in, self.n_hids)
shape_hh = (self.n_hids, self.n_hids)
self.W_xz = norm_weight(shape=shape_xh, name=_p(self.pname, 'W_xz'))
self.W_xr = norm_weight(shape=shape_xh, name=_p(self.pname, 'W_xr'))
self.W_xh = norm_weight(shape=shape_xh, name=_p(self.pname, 'W_xh'))
self.b_z = constant_weight(shape=(self.n_hids, ),
name=_p(self.pname, 'b_z'))
self.b_r = constant_weight(shape=(self.n_hids, ),
name=_p(self.pname, 'b_r'))
self.b_h = constant_weight(shape=(self.n_hids, ),
name=_p(self.pname, 'b_h'))
self.W_hz = ortho_weight(shape=shape_hh, name=_p(self.pname, 'W_hz'))
self.W_hr = ortho_weight(shape=shape_hh, name=_p(self.pname, 'W_hr'))
self.W_hh = ortho_weight(shape=shape_hh, name=_p(self.pname, 'W_hh'))
self.params = [
self.W_xz, self.W_xr, self.W_xh, self.W_hz, self.W_hr, self.W_hh,
self.b_z, self.b_r, self.b_h
]
if self.with_context:
shape_ch = (self.c_hids, self.n_hids)
self.W_cz = norm_weight(shape=shape_ch,
name=_p(self.pname, 'W_cz'))
self.W_cr = norm_weight(shape=shape_ch,
name=_p(self.pname, 'W_cr'))
self.W_ch = norm_weight(shape=shape_ch,
name=_p(self.pname, 'W_ch'))
self.W_c_init = norm_weight(shape=shape_ch,
name=_p(self.pname, 'W_c_init'))
self.params += [self.W_cz, self.W_cr, self.W_ch, self.W_c_init]
def _init_params(self):
shape_xh = (self.n_in, self.n_hids)
shape_hh = (self.n_hids, self.n_hids)
self.W_xz = norm_weight(rng=self.rng, shape=shape_xh, name=_p(self.pname, 'W_xz'))
self.W_xr = norm_weight(rng=self.rng, shape=shape_xh, name=_p(self.pname, 'W_xr'))
self.W_xh = norm_weight(rng=self.rng, shape=shape_xh, name=_p(self.pname, 'W_xh'))
self.b_z = constant_weight(shape=(self.n_hids, ), name=_p(self.pname, 'b_z'))
self.b_r = constant_weight(shape=(self.n_hids, ), name=_p(self.pname, 'b_r'))
self.b_h = constant_weight(shape=(self.n_hids, ), name=_p(self.pname, 'b_h'))
self.W_hz = ortho_weight(rng=self.rng, shape=shape_hh, name=_p(self.pname, 'W_hz'))
self.W_hr = ortho_weight(rng=self.rng, shape=shape_hh, name=_p(self.pname, 'W_hr'))
self.W_hh = ortho_weight(rng=self.rng, shape=shape_hh, name=_p(self.pname, 'W_hh'))
self.params = [self.W_xz, self.W_xr, self.W_xh,
self.W_hz, self.W_hr, self.W_hh,
self.b_z, self.b_r, self.b_h]
if self.with_layernorm:
self.W_xz_lnb = constant_weight(shape=(self.n_hids), value=scale_add, name=_p(self.pname, 'xz_lnb'))
self.W_xz_lns = constant_weight(shape=(self.n_hids), value=scale_mul, name=_p(self.pname, 'xz_lns'))
self.W_xr_lnb = constant_weight(shape=(self.n_hids), value=scale_add, name=_p(self.pname, 'xr_lnb'))
self.W_xr_lns = constant_weight(shape=(self.n_hids), value=scale_mul, name=_p(self.pname, 'xr_lns'))
self.W_xh_lnb = constant_weight(shape=(self.n_hids), value=scale_add, name=_p(self.pname, 'xh_lnb'))
self.W_xh_lns = constant_weight(shape=(self.n_hids), value=scale_mul, name=_p(self.pname, 'xh_lns'))
self.W_z_lnb = constant_weight(shape=(self.n_hids), value=scale_add, name=_p(self.pname, 'z_lnb'))
self.W_z_lns = constant_weight(shape=(self.n_hids), value=scale_mul, name=_p(self.pname, 'z_lns'))
self.W_r_lnb = constant_weight(shape=(self.n_hids), value=scale_add, name=_p(self.pname, 'r_lnb'))
self.W_r_lns = constant_weight(shape=(self.n_hids), value=scale_mul, name=_p(self.pname, 'r_lns'))
self.W_h_lnb = constant_weight(shape=(self.n_hids), value=scale_add, name=_p(self.pname, 'h_lnb'))
self.W_h_lns = constant_weight(shape=(self.n_hids), value=scale_mul, name=_p(self.pname, 'h_lns'))
self.params += [self.W_xz_lnb, self.W_xz_lns, self.W_xr_lnb, self.W_xr_lns, self.W_xh_lnb, self.W_xh_lns, \
self.W_z_lnb, self.W_z_lns, self.W_r_lnb, self.W_r_lns, self.W_h_lnb, self.W_h_lns]
if self.with_context:
shape_ch = (self.c_hids, self.n_hids)
self.W_cz = norm_weight(rng=self.rng, shape=shape_ch, name=_p(self.pname, 'W_cz'))
self.W_cr = norm_weight(rng=self.rng, shape=shape_ch, name=_p(self.pname, 'W_cr'))
self.W_ch = norm_weight(rng=self.rng, shape=shape_ch, name=_p(self.pname, 'W_ch'))
self.W_c_init = norm_weight(rng=self.rng, shape=shape_ch, name=_p(self.pname, 'W_c_init'))
self.params += [self.W_cz, self.W_cr, self.W_ch, self.W_c_init]
if self.with_layernorm:
self.W_cz_lnb = constant_weight(shape=(self.n_hids), value=scale_add, name=_p(self.pname, 'cz_lnb'))
self.W_cz_lns = constant_weight(shape=(self.n_hids), value=scale_mul, name=_p(self.pname, 'cz_lns'))
self.W_cr_lnb = constant_weight(shape=(self.n_hids), value=scale_add, name=_p(self.pname, 'cr_lnb'))
self.W_cr_lns = constant_weight(shape=(self.n_hids), value=scale_mul, name=_p(self.pname, 'cr_lns'))
self.W_ch_lnb = constant_weight(shape=(self.n_hids), value=scale_add, name=_p(self.pname, 'ch_lnb'))
self.W_ch_lns = constant_weight(shape=(self.n_hids), value=scale_mul, name=_p(self.pname, 'ch_lns'))
self.params += [self.W_cz_lnb, self.W_cz_lns, self.W_cr_lnb, self.W_cr_lns, self.W_ch_lnb, self.W_ch_lns]
def _init_params(self):
shape_i0o = (self.n_in_0, self.n_out)
shape_i1o = (self.n_in_1, self.n_out)
if self.orth:
if self.n_in_0 != self.n_out or self.n_in_1 != self.n_out:
raise ValueError('n_in != n_out when require orth in FeedForward')
self.W0 = ortho_weight(rng=self.rng, shape=shape_i0o, name=_p(self.pname, 'W0'))
self.W1 = ortho_weight(rng=self.rng, shape=shape_i1o, name=_p(self.pname, 'W1'))
else:
self.W0 = norm_weight(rng=self.rng, shape=shape_i0o, name=_p(self.pname, 'W0'))
self.W1 = norm_weight(rng=self.rng, shape=shape_i1o, name=_p(self.pname, 'W1'))
self.b = constant_weight(shape=(self.n_out, ), name=_p(self.pname, 'b'))
self.params = [self.W0, self.W1, self.b]
def _init_params(self):
shape_xh = (self.n_in*3, self.n_hids)
shape_hh = (self.n_hids*3, self.n_hids)
self.W_x = norm_weight(shape=shape_xh, name=_p(self.pname, 'W_x'))
self.b = constant_weight(shape=(self.n_hids*3,), name=_p(self.pname, 'b'))
self.W_h = ortho_weight(shape=shape_hh, name=_p(self.pname, 'W_h'))
self.params = [self.W_x, self.W_h, self.b]
self.GRU_op = mkl_gru.GRU(hid=self.n_hids, return_sequences=True)
self.h_init_state = numpy.zeros((80, 1000), numpy.float64)
def _init_params(self):
shape_io = (self.n_in, self.n_out)
if self.orth:
if self.n_in != self.n_out :
raise ValueError('n_in != n_out when require orth in FeedForward')
self.W = ortho_weight(rng=self.rng, shape=shape_io, name=_p(self.pname, 'W'))
else:
self.W = norm_weight(rng=self.rng, shape=shape_io, name=_p(self.pname, 'W'))
self.b = constant_weight(shape=(self.n_out, ), name=_p(self.pname, 'b'))
self.params = [self.W, self.b]
def _init_params(self):
shape_io = (self.n_in_0, self.n_out)
if self.orth:
if self.n_in_0 != self.n_out:
raise ValueError('n_in != n_out when require orth in FeedForward')
self.W = ortho_weight(rng=self.rng, shape=shape_io, name=_p(self.pname, 'W'))
else:
self.W = norm_weight(rng=self.rng, shape=shape_io, name=_p(self.pname, 'W'))
self.params = [self.W]
self.ff = FeedForward(self.n_in_1, self.n_out, orth=self.orth, rng=self.rng, name=_p(self.pname, 'FF_W') )
self.params.extend(self.ff.params)
def _init_params(self):
shape_xh = (self.n_in, self.n_hids)
shape_xh2 = (self.n_in, 2*self.n_hids)
shape_hh = (self.n_hids, self.n_hids)
shape_hh2 = (self.n_hids, 2*self.n_hids)
self.W_xzr = norm_weight(rng=self.rng, shape=shape_xh2, name=_p(self.pname, 'W_xzr'))
self.W_xh = norm_weight(rng=self.rng, shape=shape_xh, name=_p(self.pname, 'W_xh'))
self.b_zr = constant_weight(shape=(2*self.n_hids, ), name=_p(self.pname, 'b_zr'))
self.b_h = constant_weight(shape=(self.n_hids, ), name=_p(self.pname, 'b_h'))
self.W_hzr = multi_orth(rng=self.rng, size=shape_hh2, name=_p(self.pname, 'W_hzr'))
self.W_hh = ortho_weight(rng=self.rng, shape=shape_hh, name=_p(self.pname, 'W_hh'))
self.params += [self.W_xzr, self.W_xh,
self.W_hzr, self.W_hh,
self.b_zr, self.b_h]
if self.with_context:
shape_ch = (self.c_hids, self.n_hids)
self.W_cz = norm_weight(rng=self.rng, shape=shape_ch, name=_p(self.pname, 'W_cz'))
self.W_cr = norm_weight(rng=self.rng, shape=shape_ch, name=_p(self.pname, 'W_cr'))
self.W_ch = norm_weight(rng=self.rng, shape=shape_ch, name=_p(self.pname, 'W_ch'))
self.W_c_init = norm_weight(rng=self.rng, shape=shape_ch, name=_p(self.pname, 'W_c_init'))
self.params += [self.W_cz, self.W_cr, self.W_ch, self.W_c_init]
if self.with_begin_tag:
self.struct_begin_tag = constant_weight(shape=(self.n_hids,), value=0., name=_p(self.pname, 'struct_begin_tag'))
self.params += [self.struct_begin_tag]
if self.with_end_tag:
self.struct_end_tag = constant_weight(shape=(self.n_in,), value=0., name=_p(self.pname, 'struct_end_tag'))
self.params += [self.struct_end_tag]
if self.n_att_ctx:
self.gru_combine_ctx_h = GRU(self.n_att_ctx, self.n_hids, rng=self.rng, name=_p(self.pname, 'gru_combine_ctx_h'))
self.params.extend(self.gru_combine_ctx_h.params)
self.attention = ATTENTION(self.n_hids, self.rng, name=_p(self.pname, 'att_ctx'))
self.params.extend(self.attention.params)
def _init_params(self):
shape_xh = (self.n_in, self.n_hids)
shape_hh = (self.n_hids, self.n_hids)
self.W_xz = norm_weight(shape=shape_xh, name=_p(self.pname, 'W_xz'))
self.W_xr = norm_weight(shape=shape_xh, name=_p(self.pname, 'W_xr'))
self.W_xh = norm_weight(shape=shape_xh, name=_p(self.pname, 'W_xh'))
self.b_z = constant_weight(shape=(self.n_hids, ),
name=_p(self.pname, 'b_z'))
self.b_r = constant_weight(shape=(self.n_hids, ),
name=_p(self.pname, 'b_r'))
self.b_h = constant_weight(shape=(self.n_hids, ),
name=_p(self.pname, 'b_h'))
self.W_hz = ortho_weight(shape=shape_hh, name=_p(self.pname, 'W_hz'))
self.W_hr = ortho_weight(shape=shape_hh, name=_p(self.pname, 'W_hr'))
self.W_hh = ortho_weight(shape=shape_hh, name=_p(self.pname, 'W_hh'))
self.params = [
self.W_xz, self.W_xr, self.W_xh, self.W_hz, self.W_hr, self.W_hh,
self.b_z, self.b_r, self.b_h
]
shape_ch = (self.n_cdim, self.n_hids)
self.W_cz = norm_weight(shape=shape_ch, name=_p(self.pname, 'W_cz'))
self.W_cr = norm_weight(shape=shape_ch, name=_p(self.pname, 'W_cr'))
self.W_ch = norm_weight(shape=shape_ch, name=_p(self.pname, 'W_ch'))
self.W_c_init = norm_weight(shape=(self.n_cdim, self.n_hids),
name=_p(self.pname, 'W_c_init'))
self.b_c_init = constant_weight(shape=(self.n_hids, ),
name=_p(self.pname, 'b_c_init'))
self.params += [
self.W_cz, self.W_cr, self.W_ch, self.W_c_init, self.b_c_init
]
# we moved the parameters below here, to make it works for both with_context and with_attention modes
# modification in this version
# in the paper, e_{i,j} = a(s_{i-1}, h_j)
# here, e_{i,j} = a(GRU(s_{i-1}, y_{i-1}), h_j), which considers the lastly generated target word
# all the following parameters are for the introduced GRU
# it is reasonable
self.W_n1_h = ortho_weight(shape=shape_hh,
name=_p(self.pname, 'W_n1_h'))
self.W_n1_r = ortho_weight(shape=shape_hh,
name=_p(self.pname, 'W_n1_r'))
self.W_n1_z = ortho_weight(shape=shape_hh,
name=_p(self.pname, 'W_n1_z'))
self.b_n1_h = constant_weight(shape=(self.n_hids, ),
name=_p(self.pname, 'b_n1_h'))
self.b_n1_r = constant_weight(shape=(self.n_hids, ),
name=_p(self.pname, 'b_n1_r'))
self.b_n1_z = constant_weight(shape=(self.n_hids, ),
name=_p(self.pname, 'b_n1_z'))
self.params += [
self.W_n1_h, self.W_n1_r, self.W_n1_z, self.b_n1_h, self.b_n1_r,
self.b_n1_z
]
if self.with_attention:
self.A_cp = norm_weight(shape=shape_ch,
name=_p(self.pname, 'A_cp'))
self.B_hp = norm_weight(shape=shape_hh,
name=_p(self.pname, 'B_hp'))
self.b_tt = constant_weight(shape=(self.n_hids, ),
name=_p(self.pname, 'b_tt'))
self.D_pe = norm_weight(shape=(self.n_hids, 1),
name=_p(self.pname, 'D_pe'))
self.params += [self.A_cp, self.B_hp, self.b_tt, self.D_pe]
# coverage only works for attention model
if self.with_coverage:
shape_covh = (self.coverage_dim, self.n_hids)
self.C_covp = norm_weight(shape=shape_covh,
name=_p(self.pname, 'Cov_covp'))
if self.coverage_type is 'linguistic':
# for linguistic coverage, fertility model is necessary since it yields better translation and alignment quality
self.W_cov_fertility = norm_weight(shape=(self.n_cdim, 1),
name=_p(
self.pname,
'W_cov_fertility'))
self.b_cov_fertility = constant_weight(
shape=(1, ), name=_p(self.pname, 'b_cov_fertility'))
self.params += [self.W_cov_fertility, self.b_cov_fertility]
else:
# for neural network based coverage, gating is necessary
shape_covcov = (self.coverage_dim, self.coverage_dim)
self.W_cov_h = ortho_weight(shape=shape_covcov,
name=_p(self.pname, 'W_cov_h'))
self.W_cov_r = ortho_weight(shape=shape_covcov,
name=_p(self.pname, 'W_cov_r'))
self.W_cov_z = ortho_weight(shape=shape_covcov,
name=_p(self.pname, 'W_cov_z'))
self.b_cov_h = constant_weight(shape=(self.coverage_dim, ),
name=_p(
self.pname, 'b_cov_h'))
self.b_cov_r = constant_weight(shape=(self.coverage_dim, ),
name=_p(
self.pname, 'b_cov_r'))
self.b_cov_z = constant_weight(shape=(self.coverage_dim, ),
name=_p(
self.pname, 'b_cov_z'))
self.params += [
self.W_cov_h, self.W_cov_r, self.W_cov_z, self.b_cov_h,
self.b_cov_r, self.b_cov_z
]
# parameters for coverage inputs
# attention probablity
self.W_cov_ph = norm_weight(shape=(1, self.coverage_dim),
name=_p(
self.pname, 'W_cov_ph'))
self.W_cov_pr = norm_weight(shape=(1, self.coverage_dim),
name=_p(
self.pname, 'W_cov_pr'))
self.W_cov_pz = norm_weight(shape=(1, self.coverage_dim),
name=_p(
self.pname, 'W_cov_pz'))
# source annotations
self.W_cov_ch = norm_weight(
shape=(self.n_cdim, self.coverage_dim),
name=_p(self.pname, 'W_cov_ch'))
self.W_cov_cr = norm_weight(
shape=(self.n_cdim, self.coverage_dim),
name=_p(self.pname, 'W_cov_cr'))
self.W_cov_cz = norm_weight(
shape=(self.n_cdim, self.coverage_dim),
name=_p(self.pname, 'W_cov_cz'))
# previous decoding states
self.W_cov_hh = norm_weight(
shape=(self.n_hids, self.coverage_dim),
name=_p(self.pname, 'W_cov_hh'))
self.W_cov_hr = norm_weight(
shape=(self.n_hids, self.coverage_dim),
name=_p(self.pname, 'W_cov_hr'))
self.W_cov_hz = norm_weight(
shape=(self.n_hids, self.coverage_dim),
name=_p(self.pname, 'W_cov_hz'))
self.params += [
self.W_cov_ph, self.W_cov_pr, self.W_cov_pz,
self.W_cov_ch, self.W_cov_cr, self.W_cov_cz,
self.W_cov_hh, self.W_cov_hr, self.W_cov_hz
]
# for context gate, which works for both with_attention and with_context modes
if self.with_context_gate:
# parameters for coverage inputs
# input form target context
self.W_ctx_h = norm_weight(shape=(self.n_hids, self.n_hids),
name=_p(self.pname, 'W_ctx_h'))
self.W_ctx_c = norm_weight(shape=(self.n_cdim, self.n_hids),
name=_p(self.pname, 'W_ctx_c'))
self.b_ctx = constant_weight(shape=(self.n_hids, ),
name=_p(self.pname, 'b_ctx'))
self.params += [self.W_ctx_h, self.W_ctx_c]
# for readout
n_out = self.n_in * self.maxout_part
self.W_o_c = norm_weight(shape=(self.n_cdim, n_out),
name=_p(self.pname, 'W_out_c'))
self.W_o_h = norm_weight(shape=(self.n_hids, n_out),
name=_p(self.pname, 'W_out_h'))
self.W_o_e = norm_weight(shape=(self.n_in, n_out),
name=_p(self.pname, 'W_out_e'))
self.b_o = constant_weight(shape=(n_out, ),
name=_p(self.pname, 'b_out_o'))
self.params += [self.W_o_c, self.W_o_h, self.W_o_e, self.b_o]
def _init_params(self):
# generally, parameters with shape shape_ch = (self.c_ndim, self.n_hids) can be applied with tied weights
# this for combining lastly generated words and decoder state,
# and thus cannot be applied with tied weights
shape_xh = (self.n_in, self.n_hids)
shape_hh = (self.n_hids, self.n_hids)
self.W_xz = norm_weight(shape=shape_xh, name=_p(self.pname, 'W_xz'))
self.W_xr = norm_weight(shape=shape_xh, name=_p(self.pname, 'W_xr'))
self.W_xh = norm_weight(shape=shape_xh, name=_p(self.pname, 'W_xh'))
self.b_z = constant_weight(shape=(self.n_hids, ),
name=_p(self.pname, 'b_z'))
self.b_r = constant_weight(shape=(self.n_hids, ),
name=_p(self.pname, 'b_r'))
self.b_h = constant_weight(shape=(self.n_hids, ),
name=_p(self.pname, 'b_h'))
self.W_hz = ortho_weight(shape=shape_hh, name=_p(self.pname, 'W_hz'))
self.W_hr = ortho_weight(shape=shape_hh, name=_p(self.pname, 'W_hr'))
self.W_hh = ortho_weight(shape=shape_hh, name=_p(self.pname, 'W_hh'))
self.params = [
self.W_xz, self.W_xr, self.W_xh, self.W_hz, self.W_hr, self.W_hh,
self.b_z, self.b_r, self.b_h
]
shape_ch = (self.n_cdim, self.n_hids)
self.W_cz = norm_weight(shape=shape_ch, name=_p(self.pname, 'W_cz'))
self.W_cr = norm_weight(shape=shape_ch, name=_p(self.pname, 'W_cr'))
self.W_ch = norm_weight(shape=shape_ch, name=_p(self.pname, 'W_ch'))
self.W_c_init = norm_weight(shape=shape_ch,
name=_p(self.pname, 'W_c_init'))
# we don't add the new params if we use tied_weights, since we reuse the weights in decoder
self.params += [self.W_cz, self.W_cr, self.W_ch, self.W_c_init]
self.b_c_init = constant_weight(shape=(self.n_hids, ),
name=_p(self.pname, 'b_c_init'))
self.params += [self.b_c_init]
# we moved the parameters below here, to make it works for both with_context and with_attention modes
# modification in this version
# in the paper, e_{i,j} = a(s_{i-1}, h_j)
# here, e_{i,j} = a(GRU(s_{i-1}, y_{i-1}), h_j), which considers the lastly generated target word
# all the following parameters are for the introduced GRU
# it is reasonable
self.W_n1_h = ortho_weight(shape=shape_hh,
name=_p(self.pname, 'W_n1_h'))
self.W_n1_r = ortho_weight(shape=shape_hh,
name=_p(self.pname, 'W_n1_r'))
self.W_n1_z = ortho_weight(shape=shape_hh,
name=_p(self.pname, 'W_n1_z'))
self.b_n1_h = constant_weight(shape=(self.n_hids, ),
name=_p(self.pname, 'b_n1_h'))
self.b_n1_r = constant_weight(shape=(self.n_hids, ),
name=_p(self.pname, 'b_n1_r'))
self.b_n1_z = constant_weight(shape=(self.n_hids, ),
name=_p(self.pname, 'b_n1_z'))
self.params += [
self.W_n1_h, self.W_n1_r, self.W_n1_z, self.b_n1_h, self.b_n1_r,
self.b_n1_z
]
###############################################
if self.with_attention:
self.A_cp = norm_weight(shape=shape_ch,
name=_p(self.pname, 'A_cp'))
self.params += [self.A_cp]
self.B_hp = norm_weight(shape=shape_hh,
name=_p(self.pname, 'B_hp'))
self.b_tt = constant_weight(shape=(self.n_hids, ),
name=_p(self.pname, 'b_tt'))
self.D_pe = norm_weight(shape=(self.n_hids, 1),
name=_p(self.pname, 'D_pe'))
# self.c_tt = constant_weight(shape=(1,), name=_p(self.pname, 'c_tt'))
self.params += [self.B_hp, self.b_tt, self.D_pe]
# for error on encoder states, we don't need the probability
# thus no need for readout, which costs a large number of parameters
# for readout
n_out = self.n_in * self.maxout_part
self.W_o_c = norm_weight(shape=(self.n_cdim, n_out),
name=_p(self.pname, 'W_out_c'))
self.W_o_h = norm_weight(shape=(self.n_hids, n_out),
name=_p(self.pname, 'W_out_h'))
self.W_o_e = norm_weight(shape=(self.n_in, n_out),
name=_p(self.pname, 'W_out_e'))
self.b_o = constant_weight(shape=(n_out, ),
name=_p(self.pname, 'b_out_o'))
self.params += [self.W_o_c, self.W_o_h, self.W_o_e, self.b_o]
def _init_params(self):
# added by Zhaopeng Tu, 2016-07-12
# this for combining lastly generated words and decoder state,
shape_xh = (self.n_in, self.n_hids)
shape_hh = (self.n_hids, self.n_hids)
self.W_xz = norm_weight(rng=self.rng, shape=shape_xh, name=_p(self.pname, 'W_xz'))
self.W_xr = norm_weight(rng=self.rng, shape=shape_xh, name=_p(self.pname, 'W_xr'))
self.W_xh = norm_weight(rng=self.rng, shape=shape_xh, name=_p(self.pname, 'W_xh'))
self.b_z = constant_weight(shape=(self.n_hids, ), name=_p(self.pname, 'b_z'))
self.b_r = constant_weight(shape=(self.n_hids, ), name=_p(self.pname, 'b_r'))
self.b_h = constant_weight(shape=(self.n_hids, ), name=_p(self.pname, 'b_h'))
self.W_hz = ortho_weight(rng=self.rng, shape=shape_hh, name=_p(self.pname, 'W_hz'))
self.W_hr = ortho_weight(rng=self.rng, shape=shape_hh, name=_p(self.pname, 'W_hr'))
self.W_hh = ortho_weight(rng=self.rng, shape=shape_hh, name=_p(self.pname, 'W_hh'))
self.params = [self.W_xz, self.W_xr, self.W_xh,
self.W_hz, self.W_hr, self.W_hh,
self.b_z, self.b_r, self.b_h]
shape_ch = (self.n_cdim, self.n_hids)
self.W_cz = norm_weight(rng=self.rng, shape=shape_ch, name=_p(self.pname, 'W_cz'))
self.W_cr = norm_weight(rng=self.rng, shape=shape_ch, name=_p(self.pname, 'W_cr'))
self.W_ch = norm_weight(rng=self.rng, shape=shape_ch, name=_p(self.pname, 'W_ch'))
self.W_c_init = norm_weight(rng=self.rng, shape=shape_ch, name=_p(self.pname, 'W_c_init'))
# modified by Zhaopeng Tu, 2016-07-29
# we don't add the new params if we use tied_weights, since we reuse the weights in decoder
self.params += [self.W_cz, self.W_cr, self.W_ch, self.W_c_init]
self.b_c_init = constant_weight(shape=(self.n_hids, ), name=_p(self.pname, 'b_c_init'))
self.params += [self.b_c_init]
# modified by Zhaopeng Tu, 2016-06-08
# we moved the parameters below here, to make it works for both with_context and with_attention modes
# commented by Zhaopeng Tu, 2016-04-29
# modification in this version
# in the paper, e_{i,j} = a(s_{i-1}, h_j)
# here, e_{i,j} = a(GRU(s_{i-1}, y_{i-1}), h_j), which considers the lastly generated target word
# all the following parameters are for the introduced GRU
# it is reasonable
self.W_n1_h = ortho_weight(rng=self.rng, shape=shape_hh, name=_p(self.pname, 'W_n1_h'))
self.W_n1_r = ortho_weight(rng=self.rng, shape=shape_hh, name=_p(self.pname, 'W_n1_r'))
self.W_n1_z = ortho_weight(rng=self.rng, shape=shape_hh, name=_p(self.pname, 'W_n1_z'))
self.b_n1_h = constant_weight(shape=(self.n_hids, ), name=_p(self.pname, 'b_n1_h'))
self.b_n1_r = constant_weight(shape=(self.n_hids, ), name=_p(self.pname, 'b_n1_r'))
self.b_n1_z = constant_weight(shape=(self.n_hids, ), name=_p(self.pname, 'b_n1_z'))
self.params += [self.W_n1_h, self.W_n1_r, self.W_n1_z, self.b_n1_h, self.b_n1_r, self.b_n1_z]
###############################################
if self.with_attention:
self.A_cp = norm_weight(rng=self.rng, shape=shape_ch, name=_p(self.pname, 'A_cp'))
self.B_hp = norm_weight(rng=self.rng, shape=shape_hh, name=_p(self.pname, 'B_hp'))
self.b_tt = constant_weight(shape=(self.n_hids, ), name=_p(self.pname, 'b_tt'))
self.D_pe = norm_weight(rng=self.rng, shape=(self.n_hids, 1), name=_p(self.pname, 'D_pe'))
self.c_tt = constant_weight(shape=(1, ), name=_p(self.pname, 'c_tt'))
self.params += [self.A_cp, self.B_hp, self.b_tt, self.D_pe, self.c_tt]
# for readout
n_out = self.n_in * self.maxout_part
self.W_o_c = norm_weight(rng=self.rng, shape=(self.n_cdim, n_out), name=_p(self.pname, 'W_out_c'))
self.W_o_h = norm_weight(rng=self.rng, shape=(self.n_hids, n_out), name=_p(self.pname, 'W_out_h'))
self.W_o_e = norm_weight(rng=self.rng, shape=(self.n_in, n_out), name=_p(self.pname, 'W_out_e'))
self.b_o = constant_weight(shape=(n_out, ), name=_p(self.pname, 'b_out_o'))
self.params += [self.W_o_c, self.W_o_h, self.W_o_e, self.b_o]