def __init__(self, name, shape, init_fn=None):
super(ParamNode, self).__init__(name)
self.shape = shape
if init_fn is None:
self.params = rand(shape)
else:
self.params = init_fn(shape)
self.out = self.params
def __init__(self, name, shape, init_fn=None):
super(ParamNode, self).__init__(name)
self.shape = shape
if init_fn is None:
self.params = rand(shape)
else:
self.params = init_fn(shape)
self.out = self.params
def alloc_params(self):
rand_init = lambda shape: rand(shape, rand_range)
# PARAM Following Vaswani et al. EMNLP 2013
bias_init = lambda shape: zeros(shape) - np.log(self.vocab_size)
# NOTE IndexedParamNode allocates batch of values indexed from C
self.C = IndexedParamNode('x = C[:, ks]', self.dset, (embed_size, self.vocab_size), init_fn=rand_init)
self.H = ParamNode('H', (hidden_size, context_size*embed_size), init_fn=rand_init)
self.d = ParamNode('d', (hidden_size, 1), init_fn=bias_init)
self.U = ParamNode('U', (self.vocab_size, hidden_size), init_fn=rand_init)
self.b = ParamNode('b', (self.vocab_size, 1), init_fn=bias_init)
self.W = ParamNode('W', (self.vocab_size, context_size*embed_size), init_fn=rand_init)
self.param_nodes = [self.C, self.H, self.d, self.U, self.b, self.W]
logger.info('Allocated parameters')
def alloc_params(self):
rand_init = lambda shape: rand(shape, rand_range)
# PARAM Following Vaswani et al. EMNLP 2013
bias_init = lambda shape: zeros(shape) - np.log(self.vocab_size)
# NOTE IndexedParamNode allocates batch of values indexed from C
self.C = IndexedParamNode('x = C[:, ks]',
self.dset, (embed_size, self.vocab_size),
init_fn=rand_init)
self.H = ParamNode('H', (hidden_size, context_size * embed_size),
init_fn=rand_init)
self.d = ParamNode('d', (hidden_size, 1), init_fn=bias_init)
self.U = ParamNode('U', (self.vocab_size, hidden_size),
init_fn=rand_init)
self.b = ParamNode('b', (self.vocab_size, 1), init_fn=bias_init)
self.W = ParamNode('W', (self.vocab_size, context_size * embed_size),
init_fn=rand_init)
self.param_nodes = [self.C, self.H, self.d, self.U, self.b, self.W]
logger.info('Allocated parameters')
def vp_init(shape):
return rand(shape, (-0.01, 0.01))
def vp_init(shape):
return rand(shape, (-0.01, 0.01))