def _init_params(self): self._dv = {} self.conv_dilation = Conv1D( stride=1, pad="causal", init=self.init, kernel_width=2, dilation=self.dilation, out_ch=self.ch_dilation, optimizer=self.optimizer, act_fn=Affine(slope=1, intercept=0), ) self.tanh = Tanh() self.sigm = Sigmoid() self.multiply_gate = Multiply(act_fn=Affine(slope=1, intercept=0)) self.conv_1x1 = Conv1D( stride=1, pad="same", dilation=0, init=self.init, kernel_width=1, out_ch=self.ch_residual, optimizer=self.optimizer, act_fn=Affine(slope=1, intercept=0), ) self.add_residual = Add(act_fn=Affine(slope=1, intercept=0)) self.add_skip = Add(act_fn=Affine(slope=1, intercept=0))
def _build_basic_network(self, letters_number, input_dim=None, n_letter_embedding=32, use_bert=True, d=10, n_proj=256, n_layers=1, window=5, n_hidden=256, use_embedding_relu=False, use_batch_norm=True, dropout=0.1, n_before_dense=None): self.letters_number = letters_number self.n_letter_embedding = n_letter_embedding self.use_bert = use_bert self.d = d self.n_proj = n_proj self.n_layers = n_layers self.window = window self.n_hidden = n_hidden self.use_embedding_relu = use_embedding_relu self.use_batch_norm = use_batch_norm # layers self.dropout = torch.nn.Dropout(dropout) if self.use_bert: self.projection = torch.nn.Linear(d * input_dim, n_proj) if self.n_letter_embedding is not None: self.letter_embedding = torch.nn.Embedding(letters_number, self.n_letter_embedding) if self.use_embedding_relu: self.embedding_relu = torch.nn.ReLU() self.conv_layer = Conv1D(self.hidden_dim, n_layers, window, n_hidden, dropout=dropout, use_batch_norm=use_batch_norm)
import numpy as np import torch import torch.nn as nn from torch.autograd import Variable from layers import Conv1D ## initialize your layer and PyTorch layer net1 = Conv1D(8, 12, 3, 2) net2 = torch.nn.Conv1d(8, 12, 3, 2) ## initialize the inputs x1 = np.random.rand(3, 8, 20) x2 = Variable(torch.tensor(x1), requires_grad=True) ## Copy the parameters from the Conv1D class to PyTorch layer net2.weight = nn.Parameter(torch.tensor(net1.W)) net2.bias = nn.Parameter(torch.tensor(net1.b)) ## Your forward and backward y1 = net1(x1) b, c, w = y1.shape delta = np.random.randn(b, c, w) dx = net1.backward(delta) ## PyTorch forward and backward y2 = net2(x2) delta = torch.tensor(delta) y2.backward(delta) ## Compare def compare(x, y, var=""): y = y.detach().numpy() print("Diff of {}".format(var), abs(x - y).max())
def _build(self): """ Build model Input's form is BatchSize x Num_Time_Steps x Num_Channels Params: None Returns: None """ self.layers.append( Conv1D(num_in_channels=self.num_input_channels, num_out_channels=64, filter_size=3, strides=1, padding="SAME", dropout=0.0, bias=True, act=leak_relu)) self.layers.append( Conv1D(num_in_channels=64, num_out_channels=64, filter_size=3, strides=1, padding="SAME", dropout=0.0, bias=True, act=leak_relu)) self.layers.append(MaxPooling1D(ksize=2, strides=2, padding="VALID")) self.layers.append( Conv1D(num_in_channels=64, num_out_channels=128, filter_size=3, strides=1, padding="SAME", dropout=0.0, bias=True, act=leak_relu)) self.layers.append( Conv1D(num_in_channels=128, num_out_channels=128, filter_size=3, strides=1, padding="SAME", dropout=0.0, bias=True, act=leak_relu)) self.layers.append( Conv1D(num_in_channels=128, num_out_channels=128, filter_size=3, strides=1, padding="SAME", dropout=0.0, bias=True, act=leak_relu)) self.layers.append(MaxPooling1D(ksize=2, strides=2, padding="VALID")) self.layers.append(Flatten(num_dims=int(self.num_time_steps / 4) * 128)) self.layers.append( Dense(input_dim=int(self.num_time_steps / 4) * 128, output_dim=512, dropout=0.0, sparse_inputs=False, act=leak_relu, bias=True)) self.layers.append( Dense(input_dim=512, output_dim=256, dropout=0.0, sparse_inputs=False, act=leak_relu, bias=True)) self.layers.append( Dense(input_dim=256, output_dim=64, dropout=0.0, sparse_inputs=False, act=leak_relu, bias=True)) self.layers.append( CenterLoss(num_classes=self.num_classes, num_feas=64, learning_rate=0.1)) self.layers.append( Dense(input_dim=64, output_dim=self.num_classes, dropout=0.0, sparse_inputs=False, act=lambda x: x, bias=True))
def test_cnn_correctness_once(idx): from layers import Conv1D scores_dict = [0, 0, 0, 0] ############################################################################################ ############################# Initialize hyperparameters ############################## ############################################################################################ rint = np.random.randint norm = np.linalg.norm in_c, out_c = rint(5, 15), rint(5, 15) kernel, stride = rint(1, 10), rint(1, 10) batch, width = rint(1, 4), rint(20, 300) def info(): print('\nTesting model:') print(' in_channel: {}, out_channel: {},'.format(in_c, out_c)) print(' kernel size: {}, stride: {},'.format(kernel, stride)) print(' batch size: {}, input size: {}.'.format(batch, width)) ############################################################################################## ########## Initialize the CNN layer and copy parameters to a PyTorch CNN layer ########## ############################################################################################## try: net = Conv1D(in_c, out_c, kernel, stride) except: info() print('Failed to pass parameters to your Conv1D function!') return scores_dict model = nn.Conv1d(in_c, out_c, kernel, stride) model.weight = nn.Parameter(torch.tensor(net.W)) model.bias = nn.Parameter(torch.tensor(net.b)) ############################################################################################# ######################### Get the correct results from PyTorch ######################### ############################################################################################# x = np.random.randn(batch, in_c, width) x1 = Variable(torch.tensor(x), requires_grad=True) y1 = model(x1) b, c, w = y1.shape delta = np.random.randn(b, c, w) y1.backward(torch.tensor(delta)) ############################################################################################# ########################## Get your forward results and compare ########################## ############################################################################################# y = net(x) # print("net(x) =y, reference = y1", y.shape,y1.shape) # print(y,'================================\n',y1) # sys.exit(1) assert y.shape == y1.shape if not (y.shape == y1.shape): print("FAILURE") forward_res = y - y1.detach().numpy() forward_res_norm = abs(forward_res).max() if forward_res_norm < 1e-12: scores_dict[0] = 1 print("PASSED: Foward") else: info() print('Fail to return correct forward values') print("Reference: ", y1) print("Your answer:", y) return scores_dict ############################################################################################# ################## Get your backward results and check the tensor shape ################### ############################################################################################# dx = net.backward(delta) assert dx.shape == x.shape assert net.dW.shape == model.weight.grad.detach().numpy().shape assert net.db.shape == model.bias.grad.detach().numpy().shape ############################################################################################# ################ Check your dx, dW and db with PyTorch build-in functions ################# ############################################################################################# dx1 = x1.grad.detach().numpy() delta_res_norm = abs(dx - dx1).max() dW_res = net.dW - model.weight.grad.detach().numpy() dW_res_norm = abs(dW_res).max() db_res = net.db - model.bias.grad.detach().numpy() db_res_norm = abs(db_res).max() # # print("=============dx=================") # print("-----------Result------------") # print(dx[0][0]) # print("-----------Reference------------") # print(dx1[0][0]) # print("\n\n=============dW=================") # print("-----------Result------------") # print(net.dW[0][0]) # print("-----------Reference------------") # print(model.weight.grad.detach().numpy()[0][0]) # print("\n\n=============db=================") # print("-----------Result------------") # print(net.db) # print("-----------Reference------------") # print(model.bias.grad.detach().numpy()) if delta_res_norm < 1e-12: scores_dict[1] = 1 if dW_res_norm < 1e-12: scores_dict[2] = 1 if db_res_norm < 1e-12: scores_dict[3] = 1 if min(scores_dict) != 1: info() if scores_dict[1] == 0: print('Fail to return correct backward values dx') if scores_dict[2] == 0: print('Fail to return correct backward values dW') if scores_dict[3] == 0: print('Fail to return correct backward values db') return scores_dict
def _build(self): """ Build model Input's form is BatchSize x Num_Time_Steps x Num_Channels Params: None Returns: None """ self.layers.append( Conv1D(num_in_channels=self.num_input_channels, num_out_channels=16, filter_size=3, strides=1, padding="SAME", dropout=0.0, bias=True, act=leak_relu ) ) self.layers.append( Conv1D(num_in_channels=16, num_out_channels=32, filter_size=3, strides=1, padding="SAME", dropout=0.0, bias=True, act=leak_relu ) ) self.layers.append( LSTM(input_dim=32, num_units=128, length=self.num_time_steps, batch_size=32, return_sequece=False, bias=True ) ) self.layers.append( Dense(input_dim=128, output_dim=64, dropout=0.0, sparse_inputs=False, act=leak_relu, bias=True ) ) self.layers.append(CenterLoss(num_classes=self.num_classes, num_feas=64, learning_rate=0.5)) self.layers.append( Dense(input_dim=64, output_dim=self.num_classes, dropout=0.0, sparse_inputs=False, act=leak_relu, bias=True ) )