def hybrid_forward(self, F, x):
residual = x
x = self.bn1(x)
x = F.Activation(x, act_type='relu')
if self.downsample:
residual = self.downsample(x)
x = self.conv1(x)
x = self.bn2(x)
x = F.Activation(x, act_type='relu')
x = self.conv2(x)
return x + residual
def GRU_Cell(input, state):
for x in input:
z_t = nd.Activation(nd.FullyConnected(data=x,weight=wxz,no_bias=True,num_hidden=num_hidden)+
nd.FullyConnected(data=state,weight=whz,no_bias=True,num_hidden=num_hidden)+bz,act_type="sigmoid")
r_t = nd.Activation(nd.FullyConnected(data=x,weight=wxr,no_bias=True,num_hidden=num_hidden)+
nd.FullyConnected(data=state,weight=whr,no_bias=True,num_hidden=num_hidden)+br,act_type="sigmoid")
g_t = nd.Activation(nd.FullyConnected(data=x,weight=wxh,no_bias=True,num_hidden=num_hidden)+
nd.FullyConnected(data=r_t*state,weight=whh,no_bias=True,num_hidden=num_hidden)+bh,act_type="tanh")
state = nd.multiply(z_t,state) + nd.multiply(1-z_t,g_t)
output = nd.FullyConnected(data=state, weight=why, bias=by, num_hidden=num_outputs)
output = nd.softmax(data=output)
return output, state
def network(X,dropout=0.0):
#encoder
H1 = nd.Activation(nd.FullyConnected(data=X , weight=W1 , bias=B1 , num_hidden=num_hidden1), act_type="sigmoid")
H1 = nd.Dropout(data=H1 , p=dropout) # apply dropout layer!!!
H2 = nd.Activation(nd.FullyConnected(data=H1 , weight=W2 , bias=B2 , num_hidden=num_hidden2), act_type="sigmoid")
H2 = nd.Dropout(data=H2 , p=dropout) # apply dropout layer!!!
#decoder
H3 = nd.Activation(nd.FullyConnected(data=H2 , weight=W3 , bias=B3 , num_hidden=num_hidden1_), act_type="sigmoid")
H3 = nd.Dropout(data=H3 , p=dropout) # apply dropout layer!!!
H4 = nd.Activation(nd.FullyConnected(data=H3 , weight=W4 , bias=B4 , num_hidden=num_hidden2_), act_type="sigmoid")
H4 = nd.Dropout(data=H4 , p=dropout) # apply dropout layer!!!
H5 = nd.Activation(nd.FullyConnected(data=H4 , weight=W5 , bias=B5 , num_hidden=num_outputs), act_type="sigmoid")
out = H5
return out
def forward(self, x):
inp = x.shape[1]
oup = self.width_opt[self.idx]
x = nd.Convolution(x, weight=self.conv_weight.data()[:oup,:inp,:,:], kernel=(3, 3), stride=self.stride, pad=(1, 1), num_filter=oup, no_bias=True)
x = nd.BatchNorm(x, self.gamma[self.idx].data(), self.beta[self.idx].data(), self.moving_mean[self.idx].data(), self.moving_var[self.idx].data())
x = nd.Activation(x, act_type='relu')
return x
def hybrid_forward(self, F, x):
residual = x
x = self.body(x)
if self.downsample:
residual = self.downsample(residual)
x = F.Activation(x + residual, act_type='relu')
return x
def LSTM_Cell(input, h_state, c_state):
for x in input:
f_t = nd.Activation(nd.FullyConnected(
data=x, weight=wxhf, no_bias=True, num_hidden=num_hidden) +
nd.FullyConnected(data=h_state,
weight=whhf,
no_bias=True,
num_hidden=num_hidden) + bhf,
act_type="sigmoid")
i_t = nd.Activation(nd.FullyConnected(
data=x, weight=wxhi, no_bias=True, num_hidden=num_hidden) +
nd.FullyConnected(data=h_state,
weight=whhi,
no_bias=True,
num_hidden=num_hidden) + bhi,
act_type="sigmoid")
o_t = nd.Activation(nd.FullyConnected(
data=x, weight=wxho, no_bias=True, num_hidden=num_hidden) +
nd.FullyConnected(data=h_state,
weight=whho,
no_bias=True,
num_hidden=num_hidden) + bho,
act_type="sigmoid")
g_t = nd.Activation(nd.FullyConnected(
data=x, weight=wxhg, no_bias=True, num_hidden=num_hidden) +
nd.FullyConnected(data=h_state,
weight=whhg,
no_bias=True,
num_hidden=num_hidden) + bhg,
act_type="tanh")
c_state = nd.multiply(f_t, c_state) + nd.multiply(i_t, g_t)
h_state = nd.multiply(o_t, nd.tanh(c_state))
output = nd.FullyConnected(data=h_state,
weight=why,
bias=by,
num_hidden=num_outputs)
output = nd.softmax(data=output)
return output, h_state, c_state
def network(X,drop_rate=0.0): # formula : output_size=((input−weights+2*Padding)/Stride)+1
#data size
# MNIST,FashionMNIST = (batch size , 1 , 28 , 28)
# CIFAR = (batch size , 3 , 32 , 32)
C_H1=nd.Activation(data= nd.Convolution(data=X , weight = W1 , bias = B1 , kernel=(3,3) , stride=(1,1) , num_filter=60) , act_type="relu") # MNIST : result = ( batch size , 60 , 26 , 26) , CIFAR10 : : result = ( batch size , 60 , 30 , 30)
P_H1=nd.Pooling(data = C_H1 , pool_type = "max" , kernel=(2,2), stride = (2,2)) # MNIST : result = (batch size , 60 , 13 , 13) , CIFAR10 : result = (batch size , 60 , 15 , 15)
C_H2=nd.Activation(data= nd.Convolution(data=P_H1 , weight = W2 , bias = B2 , kernel=(6,6) , stride=(1,1) , num_filter=30), act_type="relu") # MNIST : result = ( batch size , 30 , 8 , 8), CIFAR10 : result = ( batch size , 30 , 10 , 10)
P_H2=nd.Pooling(data = C_H2 , pool_type = "max" , kernel=(2,2), stride = (2,2)) # MNIST : result = (batch size , 30 , 4 , 4) , CIFAR10 : result = (batch size , 30 , 5 , 5)
P_H2 = nd.flatten(data=P_H2)
'''FullyConnected parameter
• data: (batch_size, input_dim)
• weight: (num_hidden, input_dim)
• bias: (num_hidden,)
• out: (batch_size, num_hidden)
'''
F_H1 =nd.Activation(nd.FullyConnected(data=P_H2 , weight=W3 , bias=B3 , num_hidden=120),act_type="sigmoid")
F_H1 =nd.Dropout(data=F_H1, p=drop_rate)
F_H2 =nd.Activation(nd.FullyConnected(data=F_H1 , weight=W4 , bias=B4 , num_hidden=64),act_type="sigmoid")
F_H2 =nd.Dropout(data=F_H2, p=drop_rate)
softmax_Y = nd.softmax(nd.FullyConnected(data=F_H2 ,weight=W5 , bias=B5 , num_hidden=10))
return softmax_Y
def forward(self, X):
h = F.Activation(self.conv1_1(X), act_type='relu')
h = F.Activation(self.conv1_2(h), act_type='relu')
relu1_2 = h
h = F.Pooling(h, pool_type='max', kernel=(2, 2), stride=(2, 2))
h = F.Activation(self.conv2_1(h), act_type='relu')
h = F.Activation(self.conv2_2(h), act_type='relu')
relu2_2 = h
h = F.Pooling(h, pool_type='max', kernel=(2, 2), stride=(2, 2))
h = F.Activation(self.conv3_1(h), act_type='relu')
h = F.Activation(self.conv3_2(h), act_type='relu')
h = F.Activation(self.conv3_3(h), act_type='relu')
relu3_3 = h
h = F.Pooling(h, pool_type='max', kernel=(2, 2), stride=(2, 2))
h = F.Activation(self.conv4_1(h), act_type='relu')
h = F.Activation(self.conv4_2(h), act_type='relu')
h = F.Activation(self.conv4_3(h), act_type='relu')
relu4_3 = h
return [relu1_2, relu2_2, relu3_3, relu4_3]
def margin_loss(self, pick_fc):
args = self.args
import math
m = args.margin_m
s = args.margin_s
assert s > 0.0
#assert m >= 0.1
assert m < (math.pi / 2)
# cos_t * s
cos_t = pick_fc / s
cos_m = math.cos(m)
sin_m = math.sin(m)
mm = math.sin(math.pi - m) * m #sin(pi-m)*m=sin(m)*m
# threadhold = 0.0
threshold = math.cos(math.pi - m) # threshold < -cos(m)
if args.easy_margin:
cond = nd.Activation(data=cos_t, act_type='relu')
else:
cond_v = cos_t - threshold
cond = nd.Activation(data=cond_v, act_type='relu')
body = cos_t * cos_t
body = 1.0 - body
sin_t = nd.sqrt(body) #mx.sym.sqrt(body)
new_zy = cos_t * cos_m # cos(t+m) = c*c - s*s
b = sin_t * sin_m
new_zy = new_zy - b
new_zy = new_zy * s
if args.easy_margin:
zy_keep = pick_fc
else:
zy_keep = pick_fc - s * mm # zy-s*sin(m)*m = s*cos(t) - s*m*sin(m)
new_zy = nd.where(
cond, new_zy, zy_keep
) # cond < 0, zy_keep= s*cos(theta) or s*cos(theta)-s*m*sin(m)
return new_zy
def forward(self, x):
x = F.Activation(self.conv1(x), act_type='relu')
x = F.Activation(self.conv2(x), act_type='relu')
x = F.Activation(self.conv3(x), act_type='relu')
return _rearrange(self.conv4(x), F, self.upscale_factor)
def forward(self, input, training=True):
if self.activation != 'linear':
if training:
if self.use_bias:
return F.Activation(
F.FullyConnected(input,
self.w_mu.data() +
self.w_sigma.data() * self.w_epsilon,
self.b_mu.data() +
self.b_sigma.data() * self.b_epsilon,
num_hidden=self.units),
self.activation)
else:
return F.Activation(
F.FullyConnected(input,
self.w_mu.data() +
self.w_sigma.data() * self.w_epsilon,
no_bias=True,
num_hidden=self.units),
self.activation)
else:
if self.use_bias:
return F.Activation(
F.FullyConnected(input,
self.w_mu.data(),
self.b_mu.data(),
num_hidden=self.units),
self.activation)
else:
return F.Activation(
F.FullyConnected(input,
self.w_mu.data(),
no_bias=True,
num_hidden=self.units),
self.activation)
else:
if training:
if self.use_bias:
return F.FullyConnected(
input,
self.w_mu.data() +
self.w_sigma.data() * self.w_epsilon,
self.b_mu.data() +
self.b_sigma.data() * self.b_epsilon,
num_hidden=self.units)
else:
return F.FullyConnected(
input,
self.w_mu.data() +
self.w_sigma.data() * self.w_epsilon,
no_bias=True,
num_hidden=self.units)
else:
if self.use_bias:
return F.FullyConnected(input,
self.w_mu.data(),
self.b_mu.data(),
num_hidden=self.units)
else:
return F.FullyConnected(input,
self.w_mu.data(),
no_bias=True,
num_hidden=self.units)
def network(
X,
drop_rate=0.0
): # formula : output_size=((input−weights+2*Padding)/Stride)+1
#data size
# MNIST,FashionMNIST = (batch size , 1 , 28 , 28)
# CIFAR = (batch size , 3 , 32 , 32)
# builtin The BatchNorm function moving_mean, moving_var does not work.
C_H1 = nd.Activation(
data=nd.BatchNorm(data=nd.Convolution(data=X,
weight=W1,
bias=B1,
kernel=(3, 3),
stride=(1, 1),
num_filter=60),
gamma=gamma1,
beta=beta1,
moving_mean=ma1,
moving_var=mv1,
momentum=0.9,
fix_gamma=False,
use_global_stats=True),
act_type="relu"
) # MNIST : result = ( batch size , 60 , 26 , 26) , CIFAR10 : : result = ( batch size , 60 , 30 , 30)
P_H1 = nd.Pooling(
data=C_H1, pool_type="avg", kernel=(2, 2), stride=(2, 2)
) # MNIST : result = (batch size , 60 , 13 , 13) , CIFAR10 : result = (batch size , 60 , 15 , 15)
C_H2 = nd.Activation(
data=nd.BatchNorm(data=nd.Convolution(data=P_H1,
weight=W2,
bias=B2,
kernel=(6, 6),
stride=(1, 1),
num_filter=30),
gamma=gamma2,
beta=beta2,
moving_mean=ma2,
moving_var=mv2,
momentum=0.9,
fix_gamma=False,
use_global_stats=True),
act_type="relu"
) # MNIST : result = ( batch size , 30 , 8 , 8), CIFAR10 : result = ( batch size , 30 , 10 , 10)
P_H2 = nd.Pooling(
data=C_H2, pool_type="avg", kernel=(2, 2), stride=(2, 2)
) # MNIST : result = (batch size , 30 , 4 , 4) , CIFAR10 : result = (batch size , 30 , 5 , 5)
P_H2 = nd.flatten(data=P_H2)
'''FullyConnected parameter
• data: (batch_size, input_dim)
• weight: (num_hidden, input_dim)
• bias: (num_hidden,)
• out: (batch_size, num_hidden)
'''
F_H1 = nd.Activation(nd.BatchNorm(data=nd.FullyConnected(
data=P_H2, weight=W3, bias=B3, num_hidden=120),
gamma=gamma3,
beta=beta3,
moving_mean=ma3,
moving_var=mv3,
momentum=0.9,
fix_gamma=False,
use_global_stats=True),
act_type="relu")
F_H1 = nd.Dropout(data=F_H1, p=drop_rate)
F_H2 = nd.Activation(nd.BatchNorm(data=nd.FullyConnected(
data=F_H1, weight=W4, bias=B4, num_hidden=64),
gamma=gamma4,
beta=beta4,
moving_mean=ma4,
moving_var=mv4,
momentum=0.9,
fix_gamma=False,
use_global_stats=True),
act_type="relu")
F_H2 = nd.Dropout(data=F_H2, p=drop_rate)
#softmax_Y = nd.softmax(nd.FullyConnected(data=F_H2 ,weight=W5 , bias=B5 , num_hidden=10))
out = nd.FullyConnected(data=F_H2, weight=W5, bias=B5, num_hidden=10)
return out
def decode(self, z):
h3 = nd.Activation(self.fc3(z), 'relu')
return nd.Activation(self.fc4(h3), 'sigmoid')
def encode(self, x):
h1 = nd.Activation(self.fc1(x), 'relu')
return self.fc21(h1), self.fc22(h1)
import numpy as np
import mxnet as mx
