def _create_weights(self):
"""Create weights for the model."""
with tf.variable_scope("MLP") as scope:
# for P(C|F_prev)
self.a_ = tf.get_variable(
name="a_", initializer=tf.zeros_initializer(),
shape=[self.emb_dim, 1])
# for P(F|F_prev)
self.W_tF = tf.get_variable(
name="W_tF", initializer=glorot_uniform(),
shape=[self.emb_dim, self.y_vocabulary_size])
self.b_tF = tf.get_variable(
name="b_tF", initializer=tf.zeros_initializer(),
shape=[self.y_vocabulary_size])
# for P(F|E)
self.W_tE = tf.get_variable(
name="W_tE", initializer=glorot_uniform(),
shape=[self.emb_dim, self.y_vocabulary_size])
self.b_tE = tf.get_variable(
name="b_tE", initializer=tf.zeros_initializer(),
shape=[self.y_vocabulary_size])
# Let's create a (source) word embeddings matrix.
self.x_embeddings = tf.get_variable(
name="x_embeddings", initializer=tf.random_uniform_initializer(),
shape=[self.x_vocabulary_size, self.emb_dim])
# Let's create a (target) word embeddings matrix.
self.y_embeddings = tf.get_variable(
name="y_embeddings", initializer=tf.random_uniform_initializer(),
shape=[self.y_vocabulary_size, self.emb_dim])
def _create_weights(self):
"""Create weights for the model."""
with tf.variable_scope("MLP") as scope:
if self.gated:
self.mlp_b_ = tf.get_variable(
name="b_", initializer=tf.zeros_initializer,
shape=[self.emb_dim, 1])
self.mlp_W = tf.get_variable(
name="W", initializer=glorot_uniform(),
shape=[self.emb_dim, self.y_vocabulary_size])
else:
self.mlp_W_ = tf.get_variable(
name="W_", initializer=glorot_uniform(),
shape=[2 * self.emb_dim, self.mlp_dim])
self.mlp_b_ = tf.get_variable(
name="b_", initializer=tf.zeros_initializer,
shape=[self.mlp_dim])
self.mlp_W = tf.get_variable(
name="W", initializer=glorot_uniform(),
shape=[self.mlp_dim, self.y_vocabulary_size])
self.mlp_b = tf.get_variable(
name="b", initializer=tf.zeros_initializer,
shape=[self.y_vocabulary_size])
# Let's create a (source) word embeddings matrix.
self.x_embeddings = tf.get_variable(
name="x_embeddings", initializer=tf.random_uniform_initializer(),
shape=[self.x_vocabulary_size, self.emb_dim])
# Let's create a (target) word embeddings matrix.
self.y_embeddings = tf.get_variable(
name="y_embeddings", initializer=tf.random_uniform_initializer(),
shape=[self.y_vocabulary_size, self.emb_dim])
def _create_weights(self):
"""Create weights for the model."""
with tf.variable_scope("MLP") as scope:
# s(f_(j-1)) for Gate FFNN
self.mlp_Ws_ = tf.get_variable(name="Ws_",
initializer=glorot_uniform(),
shape=[self.emb_dim, self.mlp_dim])
self.mlp_bs_ = tf.get_variable(
name="bs_",
initializer=tf.constant_initializer(0.0),
shape=[self.mlp_dim])
self.mlp_Ws = tf.get_variable(name="Ws",
initializer=glorot_uniform(),
shape=[self.mlp_dim, 1])
self.mlp_bs = tf.get_variable(
name="bs", initializer=tf.constant_initializer(0.0), shape=[1])
# latent gatent IBM1
self.mlp_Wy_ = tf.get_variable(name="Wy_",
initializer=glorot_uniform(),
shape=[self.emb_dim, self.mlp_dim])
self.mlp_by_ = tf.get_variable(
name="by_",
initializer=tf.constant_initializer(0.0),
shape=[self.mlp_dim])
self.mlp_Wx_ = tf.get_variable(name="Wx_",
initializer=glorot_uniform(),
shape=[self.emb_dim, self.mlp_dim])
self.mlp_bx_ = tf.get_variable(
name="bx_",
initializer=tf.constant_initializer(0.0),
shape=[self.mlp_dim])
self.mlp_W = tf.get_variable(
name="W",
initializer=glorot_uniform(),
shape=[self.mlp_dim, self.y_vocabulary_size])
self.mlp_b = tf.get_variable(
name="b",
initializer=tf.constant_initializer(0.0),
shape=[self.y_vocabulary_size])
def _create_weights(self):
"""Create weights for the model."""
with tf.variable_scope("MLP") as scope:
self.mlp_W_ = tf.get_variable(
name="W_", initializer=glorot_uniform(),
shape=[(self.emb_dim * 2), self.mlp_dim])
self.mlp_b_ = tf.get_variable(
name="b_", initializer=tf.constant_initializer(0.0),
shape=[self.mlp_dim])
self.mlp_W = tf.get_variable(
name="W", initializer=glorot_uniform(),
shape=[self.mlp_dim, self.y_vocabulary_size])
self.mlp_b = tf.get_variable(
name="b", initializer=tf.constant_initializer(0.0),
shape=[self.y_vocabulary_size])
def _create_weights(self): """Create weights for the model.""" # self.W_eh = tf.Variable(tf.random_normal([self.emb_dim, self.mlp_dim])) # (emb_dim, mlp_dim) # self.W_ho = tf.Variable(tf.random_normal([self.mlp_dim, self.y_vocabulary_size])) # (mlp_dim, y_vocabulary_size) initializer = glorot_uniform() self.W_eh = tf.Variable(initializer(shape=[self.emb_dim, self.mlp_dim])) # (emb_dim, mlp_dim) self.W_ho = tf.Variable(initializer(shape=[self.mlp_dim, self.y_vocabulary_size])) # (mlp_dim, y_vocabulary_size) self.b_h = tf.Variable(tf.zeros([self.mlp_dim])) self.b_o = tf.Variable(tf.zeros([self.y_vocabulary_size]))
def _create_weights(self):
"""Create weights for the model."""
with tf.variable_scope("MLP") as scope:
self.W1 = tf.get_variable(name="W1",
initializer=glorot_uniform(),
shape=[self.emb_dim, self.mlp_dim])
self.b1 = tf.get_variable(name="b1",
initializer=tf.zeros_initializer(),
shape=[self.mlp_dim])
self.W2 = tf.get_variable(
name="W2",
initializer=glorot_uniform(),
shape=[self.mlp_dim, self.y_vocabulary_size])
self.b2 = tf.get_variable(name="b2",
initializer=tf.zeros_initializer(),
shape=[self.y_vocabulary_size])
def build(self, input_shape):
self.kernel = self.add_weight(name=KERNEL,
shape=(self.input_dim, self.output_dim),
initializer=initializers.glorot_uniform(),
trainable=True)
self.bias = self.add_weight(name=BIAS,
shape=(self.output_dim,),
initializer=initializers.Zeros,
trainable=True)
super(SparseFC, self).build(input_shape) # Be sure to call this somewhere!
def build(self, input_shape):
support_size = len(self.supports)
# There is no need to maintain this ds as I can always access the layers by calling self.l but I am keeping it
# for now as it just makes life easier.
self.support_kernels = []
for i in range(support_size):
self.support_kernels.append(
self.add_weight(name=SUPPORT_KERNEL + "_" + str(i),
shape=(self.input_dim, self.output_dim),
initializer=initializers.glorot_uniform(),
trainable=True))
self.bias = self.add_weight(name=BIAS,
shape=(self.output_dim, ),
initializer=initializers.Zeros,
trainable=True)
super(SparseGC,
self).build(input_shape) # Be sure to call this somewhere!
def _create_weights(self):
"""Create weights for the model."""
with tf.variable_scope("MLP") as scope:
# a(f_(j-1)) for Beta FFNN
self.mlp_Waf_ = tf.get_variable(name="Waf_",
initializer=glorot_uniform(),
shape=[self.emb_dim, self.mlp_dim])
self.mlp_baf_ = tf.get_variable(
name="baf_",
initializer=tf.constant_initializer(0.0),
shape=[self.mlp_dim])
self.mlp_Waf = tf.get_variable(name="Waf",
initializer=glorot_uniform(),
shape=[self.mlp_dim, 1])
self.mlp_baf = tf.get_variable(
name="baf",
initializer=tf.constant_initializer(0.0),
shape=[1])
# b(f_(j-1)) for Beta FFNN
self.mlp_Wbf_ = tf.get_variable(name="Wbf_",
initializer=glorot_uniform(),
shape=[self.emb_dim, self.mlp_dim])
self.mlp_bbf_ = tf.get_variable(
name="bbf_",
initializer=tf.constant_initializer(0.0),
shape=[self.mlp_dim])
self.mlp_Wbf = tf.get_variable(name="Wbf",
initializer=glorot_uniform(),
shape=[self.mlp_dim, 1])
self.mlp_bbf = tf.get_variable(
name="bbf",
initializer=tf.constant_initializer(0.0),
shape=[1])
# a(f_(j, j-1)) for Kuma FFNN
self.mlp_Waff_ = tf.get_variable(
name="Waff_",
initializer=glorot_uniform(),
shape=[self.emb_dim * 2, self.mlp_dim])
self.mlp_baff_ = tf.get_variable(
name="baff_",
initializer=tf.constant_initializer(0.0),
shape=[self.mlp_dim])
self.mlp_Waff = tf.get_variable(name="Waff",
initializer=glorot_uniform(),
shape=[self.mlp_dim, 1])
self.mlp_baff = tf.get_variable(
name="baff",
initializer=tf.constant_initializer(0.0),
shape=[1])
# b(f_(j, j-1)) for Kuma FFNN
self.mlp_Wbff_ = tf.get_variable(
name="Wbff_",
initializer=glorot_uniform(),
shape=[self.emb_dim * 2, self.mlp_dim])
self.mlp_bbff_ = tf.get_variable(
name="bbff_",
initializer=tf.constant_initializer(0.0),
shape=[self.mlp_dim])
self.mlp_Wbff = tf.get_variable(name="Wbff",
initializer=glorot_uniform(),
shape=[self.mlp_dim, 1])
self.mlp_bbff = tf.get_variable(
name="bbff",
initializer=tf.constant_initializer(0.0),
shape=[1])
# latent gatent IBM1
self.mlp_Wy_ = tf.get_variable(name="Wy_",
initializer=glorot_uniform(),
shape=[self.emb_dim, self.mlp_dim])
self.mlp_by_ = tf.get_variable(
name="by_",
initializer=tf.constant_initializer(0.0),
shape=[self.mlp_dim])
self.mlp_Wx_ = tf.get_variable(name="Wx_",
initializer=glorot_uniform(),
shape=[self.emb_dim, self.mlp_dim])
self.mlp_bx_ = tf.get_variable(
name="bx_",
initializer=tf.constant_initializer(0.0),
shape=[self.mlp_dim])
self.mlp_W = tf.get_variable(
name="W",
initializer=glorot_uniform(),
shape=[self.mlp_dim, self.y_vocabulary_size])
self.mlp_b = tf.get_variable(
name="b",
initializer=tf.constant_initializer(0.0),
shape=[self.y_vocabulary_size])
def _create_weights(self):
"""Create weights for the model."""
with tf.variable_scope("MLP") as scope:
# ============= phi weights ============
self.phi_W_ha = tf.get_variable(
name="phi_W_ha", initializer=glorot_uniform(),
shape=[self.emb_dim, self.mlp_dim])
self.phi_b_ha = tf.get_variable(
name="phi_b_ha", initializer=tf.zeros_initializer(),
shape=[self.mlp_dim])
self.phi_W_hb = tf.get_variable(
name="phi_W_hb", initializer=glorot_uniform(),
shape=[self.emb_dim, self.mlp_dim])
self.phi_b_hb = tf.get_variable(
name="phi_b_hb", initializer=tf.zeros_initializer(),
shape=[self.mlp_dim])
self.phi_W_a = tf.get_variable(
name="phi_W_a", initializer=glorot_uniform(),
shape=[self.mlp_dim, 1])
self.phi_b_a = tf.get_variable(
name="phi_b_a", initializer=tf.zeros_initializer(),
shape=[1])
self.phi_W_b = tf.get_variable(
name="phi_W_b", initializer=glorot_uniform(),
shape=[self.mlp_dim, 1])
self.phi_b_b = tf.get_variable(
name="phi_b_b", initializer=tf.zeros_initializer(),
shape=[1])
# ============= theta weights ============
self.mlp_W_ref = tf.get_variable(
name="W_ref", initializer=glorot_uniform(),
shape=[self.emb_dim, self.mlp_dim])
self.mlp_b_ref = tf.get_variable(
name="b_ref", initializer=tf.zeros_initializer(),
shape=[self.mlp_dim])
self.mlp_W_t = tf.get_variable(
name="W_t", initializer=glorot_uniform(),
shape=[self.mlp_dim, self.y_vocabulary_size])
self.mlp_b_t = tf.get_variable(
name="b_t", initializer=tf.zeros_initializer(),
shape=[self.y_vocabulary_size])
self.th_W_ha = tf.get_variable(
name="th_W_ha", initializer=glorot_uniform(),
shape=[self.emb_dim, self.mlp_dim])
self.th_b_ha = tf.get_variable(
name="th_b_ha", initializer=tf.zeros_initializer(),
shape=[self.mlp_dim])
self.th_W_hb = tf.get_variable(
name="th_W_hb", initializer=glorot_uniform(),
shape=[self.emb_dim, self.mlp_dim])
self.th_b_hb = tf.get_variable(
name="th_b_hb", initializer=tf.zeros_initializer(),
shape=[self.mlp_dim])
self.th_W_a = tf.get_variable(
name="th_W_a", initializer=glorot_uniform(),
shape=[self.mlp_dim, 1])
self.th_b_a = tf.get_variable(
name="th_b_a", initializer=tf.zeros_initializer(),
shape=[1])
self.th_W_b = tf.get_variable(
name="th_W_b", initializer=glorot_uniform(),
shape=[self.mlp_dim, 1])
self.th_b_b = tf.get_variable(
name="th_b_b", initializer=tf.zeros_initializer(),
shape=[1])
def _create_weights(self):
"""Create weights for the model."""
with tf.variable_scope("MLP") as scope:
#Weights for P(C|F_prev=f)
#input: d_F - embedding of f
#output: 1 - the gate C - between 0 and 1 (not discrete yet)
self.mlp_Wc_ = tf.get_variable(
name="Wc_", initializer=glorot_uniform(),
shape=[self.emb_dim, self.mlp_dim])
self.mlp_bc_ = tf.get_variable(
name="bc_", initializer=tf.constant_initializer(0.0),
shape=[self.mlp_dim])
self.mlp_Wc = tf.get_variable(
name="Wc", initializer=glorot_uniform(),
shape=[self.mlp_dim, 1])
self.mlp_bc = tf.get_variable(
name="bc", initializer=tf.constant_initializer(0.0),
shape=[1])
#Weights for P(F|F_prev=f)
#input: d_F - embedding of f
#output: V_F - softmax over all the french words
self.mlp_Wf_ = tf.get_variable(
name="Wf_", initializer=glorot_uniform(),
shape=[self.emb_dim, self.mlp_dim])
self.mlp_bf_ = tf.get_variable(
name="bf_", initializer=tf.constant_initializer(0.0),
shape=[self.mlp_dim])
self.mlp_Wf = tf.get_variable(
name="Wf", initializer=glorot_uniform(),
shape=[self.mlp_dim, self.y_vocabulary_size])
self.mlp_bf = tf.get_variable(
name="bf", initializer=tf.constant_initializer(0.0),
shape=[self.y_vocabulary_size])
#Weights for P(F|E=e)
#input: d_E - embedding of e
#output: V_F - softmax over all the french words
self.mlp_We_ = tf.get_variable(
name="We_", initializer=glorot_uniform(),
shape=[self.emb_dim, self.mlp_dim])
self.mlp_be_ = tf.get_variable(
name="be_", initializer=tf.constant_initializer(0.0),
shape=[self.mlp_dim])
self.mlp_We = tf.get_variable(
name="We", initializer=glorot_uniform(),
shape=[self.mlp_dim, self.y_vocabulary_size])
self.mlp_be = tf.get_variable(
name="be", initializer=tf.constant_initializer(0.0),
shape=[self.y_vocabulary_size])