def _init_params(self):
self._dv = {}
self.conv_dilation = Conv1D(
stride=1,
pad="causal",
init=self.init,
kernel_width=2,
dilation=self.dilation,
out_ch=self.ch_dilation,
optimizer=self.optimizer,
act_fn=Affine(slope=1, intercept=0),
)
self.tanh = Tanh()
self.sigm = Sigmoid()
self.multiply_gate = Multiply(act_fn=Affine(slope=1, intercept=0))
self.conv_1x1 = Conv1D(
stride=1,
pad="same",
dilation=0,
init=self.init,
kernel_width=1,
out_ch=self.ch_residual,
optimizer=self.optimizer,
act_fn=Affine(slope=1, intercept=0),
)
self.add_residual = Add(act_fn=Affine(slope=1, intercept=0))
self.add_skip = Add(act_fn=Affine(slope=1, intercept=0))
def _build_basic_network(self,
letters_number,
input_dim=None,
n_letter_embedding=32,
use_bert=True,
d=10,
n_proj=256,
n_layers=1,
window=5,
n_hidden=256,
use_embedding_relu=False,
use_batch_norm=True,
dropout=0.1,
n_before_dense=None):
self.letters_number = letters_number
self.n_letter_embedding = n_letter_embedding
self.use_bert = use_bert
self.d = d
self.n_proj = n_proj
self.n_layers = n_layers
self.window = window
self.n_hidden = n_hidden
self.use_embedding_relu = use_embedding_relu
self.use_batch_norm = use_batch_norm
# layers
self.dropout = torch.nn.Dropout(dropout)
if self.use_bert:
self.projection = torch.nn.Linear(d * input_dim, n_proj)
if self.n_letter_embedding is not None:
self.letter_embedding = torch.nn.Embedding(letters_number,
self.n_letter_embedding)
if self.use_embedding_relu:
self.embedding_relu = torch.nn.ReLU()
self.conv_layer = Conv1D(self.hidden_dim,
n_layers,
window,
n_hidden,
dropout=dropout,
use_batch_norm=use_batch_norm)
import numpy as np
import torch
import torch.nn as nn
from torch.autograd import Variable
from layers import Conv1D
## initialize your layer and PyTorch layer
net1 = Conv1D(8, 12, 3, 2)
net2 = torch.nn.Conv1d(8, 12, 3, 2)
## initialize the inputs
x1 = np.random.rand(3, 8, 20)
x2 = Variable(torch.tensor(x1), requires_grad=True)
## Copy the parameters from the Conv1D class to PyTorch layer
net2.weight = nn.Parameter(torch.tensor(net1.W))
net2.bias = nn.Parameter(torch.tensor(net1.b))
## Your forward and backward
y1 = net1(x1)
b, c, w = y1.shape
delta = np.random.randn(b, c, w)
dx = net1.backward(delta)
## PyTorch forward and backward
y2 = net2(x2)
delta = torch.tensor(delta)
y2.backward(delta)
## Compare
def compare(x, y, var=""):
y = y.detach().numpy()
print("Diff of {}".format(var), abs(x - y).max())
def _build(self):
"""
Build model
Input's form is BatchSize x Num_Time_Steps x Num_Channels
Params:
None
Returns:
None
"""
self.layers.append(
Conv1D(num_in_channels=self.num_input_channels,
num_out_channels=64,
filter_size=3,
strides=1,
padding="SAME",
dropout=0.0,
bias=True,
act=leak_relu))
self.layers.append(
Conv1D(num_in_channels=64,
num_out_channels=64,
filter_size=3,
strides=1,
padding="SAME",
dropout=0.0,
bias=True,
act=leak_relu))
self.layers.append(MaxPooling1D(ksize=2, strides=2, padding="VALID"))
self.layers.append(
Conv1D(num_in_channels=64,
num_out_channels=128,
filter_size=3,
strides=1,
padding="SAME",
dropout=0.0,
bias=True,
act=leak_relu))
self.layers.append(
Conv1D(num_in_channels=128,
num_out_channels=128,
filter_size=3,
strides=1,
padding="SAME",
dropout=0.0,
bias=True,
act=leak_relu))
self.layers.append(
Conv1D(num_in_channels=128,
num_out_channels=128,
filter_size=3,
strides=1,
padding="SAME",
dropout=0.0,
bias=True,
act=leak_relu))
self.layers.append(MaxPooling1D(ksize=2, strides=2, padding="VALID"))
self.layers.append(Flatten(num_dims=int(self.num_time_steps / 4) *
128))
self.layers.append(
Dense(input_dim=int(self.num_time_steps / 4) * 128,
output_dim=512,
dropout=0.0,
sparse_inputs=False,
act=leak_relu,
bias=True))
self.layers.append(
Dense(input_dim=512,
output_dim=256,
dropout=0.0,
sparse_inputs=False,
act=leak_relu,
bias=True))
self.layers.append(
Dense(input_dim=256,
output_dim=64,
dropout=0.0,
sparse_inputs=False,
act=leak_relu,
bias=True))
self.layers.append(
CenterLoss(num_classes=self.num_classes,
num_feas=64,
learning_rate=0.1))
self.layers.append(
Dense(input_dim=64,
output_dim=self.num_classes,
dropout=0.0,
sparse_inputs=False,
act=lambda x: x,
bias=True))
def test_cnn_correctness_once(idx):
from layers import Conv1D
scores_dict = [0, 0, 0, 0]
############################################################################################
############################# Initialize hyperparameters ##############################
############################################################################################
rint = np.random.randint
norm = np.linalg.norm
in_c, out_c = rint(5, 15), rint(5, 15)
kernel, stride = rint(1, 10), rint(1, 10)
batch, width = rint(1, 4), rint(20, 300)
def info():
print('\nTesting model:')
print(' in_channel: {}, out_channel: {},'.format(in_c, out_c))
print(' kernel size: {}, stride: {},'.format(kernel, stride))
print(' batch size: {}, input size: {}.'.format(batch, width))
##############################################################################################
########## Initialize the CNN layer and copy parameters to a PyTorch CNN layer ##########
##############################################################################################
try:
net = Conv1D(in_c, out_c, kernel, stride)
except:
info()
print('Failed to pass parameters to your Conv1D function!')
return scores_dict
model = nn.Conv1d(in_c, out_c, kernel, stride)
model.weight = nn.Parameter(torch.tensor(net.W))
model.bias = nn.Parameter(torch.tensor(net.b))
#############################################################################################
######################### Get the correct results from PyTorch #########################
#############################################################################################
x = np.random.randn(batch, in_c, width)
x1 = Variable(torch.tensor(x), requires_grad=True)
y1 = model(x1)
b, c, w = y1.shape
delta = np.random.randn(b, c, w)
y1.backward(torch.tensor(delta))
#############################################################################################
########################## Get your forward results and compare ##########################
#############################################################################################
y = net(x)
# print("net(x) =y, reference = y1", y.shape,y1.shape)
# print(y,'================================\n',y1)
# sys.exit(1)
assert y.shape == y1.shape
if not (y.shape == y1.shape): print("FAILURE")
forward_res = y - y1.detach().numpy()
forward_res_norm = abs(forward_res).max()
if forward_res_norm < 1e-12:
scores_dict[0] = 1
print("PASSED: Foward")
else:
info()
print('Fail to return correct forward values')
print("Reference: ", y1)
print("Your answer:", y)
return scores_dict
#############################################################################################
################## Get your backward results and check the tensor shape ###################
#############################################################################################
dx = net.backward(delta)
assert dx.shape == x.shape
assert net.dW.shape == model.weight.grad.detach().numpy().shape
assert net.db.shape == model.bias.grad.detach().numpy().shape
#############################################################################################
################ Check your dx, dW and db with PyTorch build-in functions #################
#############################################################################################
dx1 = x1.grad.detach().numpy()
delta_res_norm = abs(dx - dx1).max()
dW_res = net.dW - model.weight.grad.detach().numpy()
dW_res_norm = abs(dW_res).max()
db_res = net.db - model.bias.grad.detach().numpy()
db_res_norm = abs(db_res).max()
#
# print("=============dx=================")
# print("-----------Result------------")
# print(dx[0][0])
# print("-----------Reference------------")
# print(dx1[0][0])
# print("\n\n=============dW=================")
# print("-----------Result------------")
# print(net.dW[0][0])
# print("-----------Reference------------")
# print(model.weight.grad.detach().numpy()[0][0])
# print("\n\n=============db=================")
# print("-----------Result------------")
# print(net.db)
# print("-----------Reference------------")
# print(model.bias.grad.detach().numpy())
if delta_res_norm < 1e-12:
scores_dict[1] = 1
if dW_res_norm < 1e-12:
scores_dict[2] = 1
if db_res_norm < 1e-12:
scores_dict[3] = 1
if min(scores_dict) != 1:
info()
if scores_dict[1] == 0:
print('Fail to return correct backward values dx')
if scores_dict[2] == 0:
print('Fail to return correct backward values dW')
if scores_dict[3] == 0:
print('Fail to return correct backward values db')
return scores_dict
def _build(self): """ Build model Input's form is BatchSize x Num_Time_Steps x Num_Channels Params: None Returns: None """ self.layers.append( Conv1D(num_in_channels=self.num_input_channels, num_out_channels=16, filter_size=3, strides=1, padding="SAME", dropout=0.0, bias=True, act=leak_relu ) ) self.layers.append( Conv1D(num_in_channels=16, num_out_channels=32, filter_size=3, strides=1, padding="SAME", dropout=0.0, bias=True, act=leak_relu ) ) self.layers.append( LSTM(input_dim=32, num_units=128, length=self.num_time_steps, batch_size=32, return_sequece=False, bias=True ) ) self.layers.append( Dense(input_dim=128, output_dim=64, dropout=0.0, sparse_inputs=False, act=leak_relu, bias=True ) ) self.layers.append(CenterLoss(num_classes=self.num_classes, num_feas=64, learning_rate=0.5)) self.layers.append( Dense(input_dim=64, output_dim=self.num_classes, dropout=0.0, sparse_inputs=False, act=leak_relu, bias=True ) )