def __init__(self, in_size, out_size, act_func='tanh', Wscale=1.0, Uscale=1.0, Vscale=1.0, nobias=False, bias=0.0): self.bias = np.float32(bias) self.nobias = nobias self.in_size = in_size self.out_size = out_size self.act_func_str = act_func.lower() #initialize weight matrices that operates on the input self.W = weight_initialization(in_size, out_size*3, Wscale) self.gW = np.empty_like(self.W) self.V = weight_initialization(out_size, out_size*3, Uscale) self.gV = np.empty_like(self.V) #initialize weight matrices that operates on the input self.Wc = weight_initialization(in_size, out_size, Wscale) self.gWc = np.empty_like(self.Wc) self.Vhc = weight_initialization(out_size, out_size, Uscale) self.gVhc = np.empty_like(self.Vhc) self.Vcc = weight_initialization(out_size, out_size, Vscale) self.gVcc = np.empty_like(self.Vcc) if not self.nobias: self.b = np.empty((1, out_size*3), dtype=np.float32) self.b.fill(self.bias) self.gb = np.empty_like(self.b) self.bc = np.empty((1, out_size), dtype=np.float32) self.bc.fill(self.bias) self.gbc = np.empty_like(self.bc)
def __init__(self, in_size, out_size, act_func='tanh', Wscale=1.0, Vscale=1.0, nobias=False, bias=0.0, initV2identity=False, hot=False): self.bias = bias self.nobias=False self.in_size = in_size self.out_size = out_size self.initV2identity = initV2identity self.hot = hot self.act_func_str = act_func.lower() #initialize W weight matrix self.W = utils.weight_initialization(in_size, out_size, Wscale) self.gW = np.empty_like(self.W) #initialize V square weight matrix if initV2identity: self.V = np.eye(out_size, dtype=np.float32)*Vscale else: self.V = utils.weight_initialization(out_size, out_size, Vscale) self.gV = np.empty_like(self.V) if not self.nobias: self.b = np.empty((1, out_size), dtype=np.float32) self.b.fill(self.bias) self.gb = np.empty_like(self.b) available_act_funcs = { 'sigmoid': (utils.Sigmoid, utils.dSigmoid), 'tanh': (utils.Tanh, utils.dTanh), 'relu': (utils.ReLU, utils.dReLU), 'leakyrelu': (utils.LeakyReLU, utils.dLeakyReLU), } available_cu_act_funcs = { 'sigmoid': (cuk.sigmoid, cuk.dsigmoid), 'tanh': (cuk.tanh, cuk.dtanh), 'relu': (cuk.reLU, cuk.dreLU), 'leakyrelu': (cuk.leakyReLU, cuk.dleakyReLU), } self.act_func = available_act_funcs[self.act_func_str][0] self.dact_func = available_act_funcs[self.act_func_str][1] self.cu_act_func = available_cu_act_funcs[self.act_func_str][0] self.cu_dact_func = available_cu_act_funcs[self.act_func_str][1]
def __init__(self, in_size, out_size, act_func='tanh', Wscale=1.0, Uscale=1.0, nobias=False, bias=0.0): self.bias = np.float32(bias) self.nobias = nobias self.in_size = in_size self.out_size = out_size self.act_func_str = act_func.lower() #initialize weight matrices that operates on the input self.W = weight_initialization(in_size, out_size*4, Wscale) self.gW = np.empty_like(self.W) self.Wi = self.W[:,] self.gWi = np.empty_like(self.Wi) self.Wf = weight_initialization(in_size, out_size, Wscale) self.gWf = np.empty_like(self.Wf) self.Wo = weight_initialization(in_size, out_size, Wscale) self.gWo = np.empty_like(self.Wo) self.Wc = weight_initialization(in_size, out_size, Wscale) self.gWc = np.empty_like(self.Wc) #initialize weight matrices that operates on the previous hidden state self.Ui = weight_initialization(out_size, out_size, Uscale) self.gUi = np.empty_like(self.Ui) self.Uf = weight_initialization(out_size, out_size, Uscale) self.gUf = np.empty_like(self.Wf) self.Uo = weight_initialization(out_size, out_size, Uscale) self.gUo = np.empty_like(self.Uo) self.Uc = weight_initialization(out_size, out_size, Uscale) self.gUc = np.empty_like(self.Uc) if not self.nobias: self.bi = np.empty((1, out_size), dtype=np.float32).fill(self.bias) self.gbi = np.empty_like(self.bi) self.bf = np.empty((1, out_size), dtype=np.float32).fill(self.bias) self.gbf = np.empty_like(self.bf) self.bo = np.empty((1, out_size), dtype=np.float32).fill(self.bias) self.gbo = np.empty_like(self.bo) self.bc = np.empty((1, out_size), dtype=np.float32).fill(self.bias) self.gbc = np.empty_like(self.bc)
def __init__(self, in_size, Wscale=1.0, nobias=False, bias=0.0, compute_loss=True, return_y=False, use_cudnn=True): self.bias = np.float32(bias) self.nobias = nobias self.in_size = in_size self.compute_loss = compute_loss self.use_cudnn = use_cudnn self.return_y = return_y #initialize W weight matrix self.W = weight_initialization(in_size, 1, Wscale) self.gW = np.empty_like(self.W) if not self.nobias: self.b = np.empty((1, 1), dtype=np.float32) self.b.fill(self.bias) self.gb = np.empty_like(self.b)
def __init__(self, in_size, out_size, act_func='tanh', Wscale=1.0, Vscale=1.0, nobias=False, bias=0.0): self.bias = np.float32(bias) self.nobias = nobias self.in_size = in_size self.out_size = out_size self.act_func_str = act_func.lower() #initialize weight matrices that operates on the input self.Wu = utils.weight_initialization(in_size, out_size, Wscale) self.gWu = np.empty_like(self.Wu) self.Vu = utils.weight_initialization(out_size, out_size, Vscale) self.gVu = np.empty_like(self.Vu) #initialize weight matrices that operates on the input self.Wr = utils.weight_initialization(in_size, out_size, Wscale) self.gWr = np.empty_like(self.Wr) self.Vr = utils.weight_initialization(out_size, out_size, Vscale) self.gVr = np.empty_like(self.Vr) #initialize weight matrices that operates on the input self.Wh = utils.weight_initialization(in_size, out_size, Wscale) self.gWh = np.empty_like(self.Wh) self.Vh = utils.weight_initialization(out_size, out_size, Vscale) self.gVh = np.empty_like(self.Vh) if not self.nobias: self.bu = np.empty((1, out_size), dtype=np.float32) self.bu.fill(self.bias) self.gbu = np.empty_like(self.bu) self.br = np.empty((1, out_size), dtype=np.float32) self.br.fill(self.bias) self.gbr = np.empty_like(self.br) self.bh = np.empty((1, out_size), dtype=np.float32) self.bh.fill(self.bias) self.gbh = np.empty_like(self.bh)