def __init__(self, in_size, out_size, act_func='tanh',
Wscale=1.0, Uscale=1.0, Vscale=1.0,
nobias=False, bias=0.0):
self.bias = np.float32(bias)
self.nobias = nobias
self.in_size = in_size
self.out_size = out_size
self.act_func_str = act_func.lower()
#initialize weight matrices that operates on the input
self.W = weight_initialization(in_size, out_size*3, Wscale)
self.gW = np.empty_like(self.W)
self.V = weight_initialization(out_size, out_size*3, Uscale)
self.gV = np.empty_like(self.V)
#initialize weight matrices that operates on the input
self.Wc = weight_initialization(in_size, out_size, Wscale)
self.gWc = np.empty_like(self.Wc)
self.Vhc = weight_initialization(out_size, out_size, Uscale)
self.gVhc = np.empty_like(self.Vhc)
self.Vcc = weight_initialization(out_size, out_size, Vscale)
self.gVcc = np.empty_like(self.Vcc)
if not self.nobias:
self.b = np.empty((1, out_size*3), dtype=np.float32)
self.b.fill(self.bias)
self.gb = np.empty_like(self.b)
self.bc = np.empty((1, out_size), dtype=np.float32)
self.bc.fill(self.bias)
self.gbc = np.empty_like(self.bc)
def __init__(self, in_size, out_size, act_func='tanh',
Wscale=1.0, Vscale=1.0,
nobias=False, bias=0.0,
initV2identity=False,
hot=False):
self.bias = bias
self.nobias=False
self.in_size = in_size
self.out_size = out_size
self.initV2identity = initV2identity
self.hot = hot
self.act_func_str = act_func.lower()
#initialize W weight matrix
self.W = utils.weight_initialization(in_size, out_size, Wscale)
self.gW = np.empty_like(self.W)
#initialize V square weight matrix
if initV2identity:
self.V = np.eye(out_size, dtype=np.float32)*Vscale
else:
self.V = utils.weight_initialization(out_size, out_size, Vscale)
self.gV = np.empty_like(self.V)
if not self.nobias:
self.b = np.empty((1, out_size), dtype=np.float32)
self.b.fill(self.bias)
self.gb = np.empty_like(self.b)
available_act_funcs = {
'sigmoid': (utils.Sigmoid, utils.dSigmoid),
'tanh': (utils.Tanh, utils.dTanh),
'relu': (utils.ReLU, utils.dReLU),
'leakyrelu': (utils.LeakyReLU, utils.dLeakyReLU),
}
available_cu_act_funcs = {
'sigmoid': (cuk.sigmoid, cuk.dsigmoid),
'tanh': (cuk.tanh, cuk.dtanh),
'relu': (cuk.reLU, cuk.dreLU),
'leakyrelu': (cuk.leakyReLU, cuk.dleakyReLU),
}
self.act_func = available_act_funcs[self.act_func_str][0]
self.dact_func = available_act_funcs[self.act_func_str][1]
self.cu_act_func = available_cu_act_funcs[self.act_func_str][0]
self.cu_dact_func = available_cu_act_funcs[self.act_func_str][1]
def __init__(self, in_size, out_size, act_func='tanh',
Wscale=1.0, Uscale=1.0,
nobias=False, bias=0.0):
self.bias = np.float32(bias)
self.nobias = nobias
self.in_size = in_size
self.out_size = out_size
self.act_func_str = act_func.lower()
#initialize weight matrices that operates on the input
self.W = weight_initialization(in_size, out_size*4, Wscale)
self.gW = np.empty_like(self.W)
self.Wi = self.W[:,]
self.gWi = np.empty_like(self.Wi)
self.Wf = weight_initialization(in_size, out_size, Wscale)
self.gWf = np.empty_like(self.Wf)
self.Wo = weight_initialization(in_size, out_size, Wscale)
self.gWo = np.empty_like(self.Wo)
self.Wc = weight_initialization(in_size, out_size, Wscale)
self.gWc = np.empty_like(self.Wc)
#initialize weight matrices that operates on the previous hidden state
self.Ui = weight_initialization(out_size, out_size, Uscale)
self.gUi = np.empty_like(self.Ui)
self.Uf = weight_initialization(out_size, out_size, Uscale)
self.gUf = np.empty_like(self.Wf)
self.Uo = weight_initialization(out_size, out_size, Uscale)
self.gUo = np.empty_like(self.Uo)
self.Uc = weight_initialization(out_size, out_size, Uscale)
self.gUc = np.empty_like(self.Uc)
if not self.nobias:
self.bi = np.empty((1, out_size), dtype=np.float32).fill(self.bias)
self.gbi = np.empty_like(self.bi)
self.bf = np.empty((1, out_size), dtype=np.float32).fill(self.bias)
self.gbf = np.empty_like(self.bf)
self.bo = np.empty((1, out_size), dtype=np.float32).fill(self.bias)
self.gbo = np.empty_like(self.bo)
self.bc = np.empty((1, out_size), dtype=np.float32).fill(self.bias)
self.gbc = np.empty_like(self.bc)
def __init__(self, in_size,
Wscale=1.0,
nobias=False,
bias=0.0,
compute_loss=True,
return_y=False,
use_cudnn=True):
self.bias = np.float32(bias)
self.nobias = nobias
self.in_size = in_size
self.compute_loss = compute_loss
self.use_cudnn = use_cudnn
self.return_y = return_y
#initialize W weight matrix
self.W = weight_initialization(in_size, 1, Wscale)
self.gW = np.empty_like(self.W)
if not self.nobias:
self.b = np.empty((1, 1), dtype=np.float32)
self.b.fill(self.bias)
self.gb = np.empty_like(self.b)
def __init__(self, in_size, out_size, act_func='tanh',
Wscale=1.0, Vscale=1.0,
nobias=False, bias=0.0):
self.bias = np.float32(bias)
self.nobias = nobias
self.in_size = in_size
self.out_size = out_size
self.act_func_str = act_func.lower()
#initialize weight matrices that operates on the input
self.Wu = utils.weight_initialization(in_size, out_size, Wscale)
self.gWu = np.empty_like(self.Wu)
self.Vu = utils.weight_initialization(out_size, out_size, Vscale)
self.gVu = np.empty_like(self.Vu)
#initialize weight matrices that operates on the input
self.Wr = utils.weight_initialization(in_size, out_size, Wscale)
self.gWr = np.empty_like(self.Wr)
self.Vr = utils.weight_initialization(out_size, out_size, Vscale)
self.gVr = np.empty_like(self.Vr)
#initialize weight matrices that operates on the input
self.Wh = utils.weight_initialization(in_size, out_size, Wscale)
self.gWh = np.empty_like(self.Wh)
self.Vh = utils.weight_initialization(out_size, out_size, Vscale)
self.gVh = np.empty_like(self.Vh)
if not self.nobias:
self.bu = np.empty((1, out_size), dtype=np.float32)
self.bu.fill(self.bias)
self.gbu = np.empty_like(self.bu)
self.br = np.empty((1, out_size), dtype=np.float32)
self.br.fill(self.bias)
self.gbr = np.empty_like(self.br)
self.bh = np.empty((1, out_size), dtype=np.float32)
self.bh.fill(self.bias)
self.gbh = np.empty_like(self.bh)
