def __init__(self,inputSize=168, outputSize=24 ,dilations=[1,2,4,8,16,20,32],nResidue=35, actType='relu' ,dropout=0.2, **kwargs):
super(TCN, self).__init__(**kwargs)
self.inputSize = inputSize
self.outputSize = outputSize
self.dilations = dilations
self.encoder = nn.Sequential()
self.outputLayer= nn.Sequential()
with self.name_scope():
# The embedding of auxiliary variables
self.stationEmbedding = nn.Embedding(963,18)
self.nYearEmbedding = nn.Embedding(3,2)
self.nMonthEmbedding = nn.Embedding(12,2)
self.mDayEmbedding = nn.Embedding(31,5)
self.wdayEmbedding = nn.Embedding(7,3)
self.nHourEmbedding = nn.Embedding(24,4)
for d in self.dilations:
self.encoder.add(ResidualTCN(d=d, n_residue=nResidue))
self.decoder = (futureResidual(xDim=64))
self.outputLayer.add(nn.Dense(64, flatten=False))
self.outputLayer.add(nn.BatchNorm(axis=2))
self.outputLayer.add(nn.Swish())
#self.outputLayer.add(nn.Activation(activation=actType))
self.outputLayer.add(nn.Dropout(dropout))
#self.outputLayer.add(nn.Dense(1,activation='relu',flatten=False))
self.mu = nn.Dense(1, flatten=False, activation='relu')
self.sigma = nn.Dense(1, flatten=False, activation='softrelu')
def get_activation_layer(activation):
"""
Create activation layer from string/function.
Parameters:
----------
activation : function, or str, or HybridBlock
Activation function or name of activation function.
Returns
-------
HybridBlock
Activation layer.
"""
assert (activation is not None)
if isfunction(activation):
return activation()
elif isinstance(activation, str):
if activation == "relu6":
return ReLU6()
elif activation == "swish":
return nn.Swish()
elif activation == "hswish":
return HSwish()
elif activation == "hsigmoid":
return HSigmoid()
else:
return nn.Activation(activation)
else:
assert (isinstance(activation, HybridBlock))
return activation
def __init__(self, act_func: str, **kwargs):
super(Activation, self).__init__()
with self.name_scope():
if act_func in ('relu', 'sigmoid', 'softrelu', 'softsign', 'tanh'):
self.act = nn.Activation(act_func)
elif act_func == 'leaky':
self.act = nn.LeakyReLU(**kwargs)
elif act_func == 'prelu':
self.act = nn.PReLU(**kwargs)
elif act_func == 'selu':
self.act = nn.SELU()
elif act_func == 'elu':
self.act = nn.ELU(**kwargs)
elif act_func == 'gelu':
self.act = nn.GELU()
elif act_func == 'relu6':
self.act = ReLU6()
elif act_func == 'hard_sigmoid':
self.act = HardSigmoid()
elif act_func == 'swish':
self.act = nn.Swish()
elif act_func == 'hard_swish':
self.act = HardSwish()
elif act_func == 'mish':
self.act = Mish()
else:
raise NotImplementedError(
f"Not implemented activation: {act_func}")
def _add_conv(out, channels=1, kernel=1, stride=1, pad=0,
num_group=1, active=True):
out.add(nn.Conv2D(channels, kernel, stride, pad, groups=num_group, use_bias=False))
print(nn.Conv2D(channels, kernel, stride, pad, groups=num_group, use_bias=False))
out.add(nn.BatchNorm(scale=True, momentum=0.99, epsilon=1e-3))
if active:
out.add(nn.Swish())
def get_activation(activation: str, **kwargs) -> nn.HybridBlock:
"""
Parameters
----------
activation
Activation type
Returns
-------
mxnet.gluon.HybridBlock
Activation object
"""
if activation in ["relu", "sigmoid", "softrelu", "softsign", "tanh"]:
return nn.Activation(activation=activation, **kwargs)
if activation == "lrelu":
return nn.LeakyReLU(alpha=0.2, **kwargs)
if activation == "elu":
return nn.ELU(**kwargs)
if activation == "swish":
return nn.Swish(**kwargs)
if activation == "lipswish":
return LipSwish(**kwargs)
raise NotImplementedError(activation)
def __init__(self, num_clothes, num_colors, ctx):
super(fashion_net_2_branches, self).__init__()
self._features = model_zoo.get_model('mobilenetv2_1.0',
pretrained=True,
ctx=ctx).features
for _, w in self._features.collect_params().items():
w.grad_req = 'null'
self._flatten = nn.Flatten()
self._relu = nn.Activation(activation='relu')
self._swish = nn.Swish()
self._clothes_fc_1 = nn.Dense(100)
self._clothes_bn = nn.BatchNorm(center=False, scale=True)
self._clothes_out = nn.Dense(num_clothes)
self._clothes_fc_1.initialize(init=init.Xavier(), ctx=ctx)
self._clothes_bn.initialize(init=init.Zero(), ctx=ctx)
self._clothes_out.initialize(init=init.Xavier(), ctx=ctx)
self._color_fc_1 = nn.Dense(100)
self._color_bn_1 = nn.BatchNorm(center=False, scale=True)
self._color_fc_2 = nn.Dense(50)
self._color_bn_2 = nn.BatchNorm(center=False, scale=True)
self._color_out = nn.Dense(num_colors)
self._color_fc_1.initialize(init=init.Xavier(), ctx=ctx)
self._color_bn_1.initialize(init=init.Zero(), ctx=ctx)
self._color_fc_2.initialize(init=init.Xavier(), ctx=ctx)
self._color_bn_2.initialize(init=init.Zero(), ctx=ctx)
self._color_out.initialize(init=init.Xavier(), ctx=ctx)
def __init__(self, channel, se_ratio=0.25):
super(SEModule, self).__init__()
# self.avg_pool = nn.contrib.AdaptiveAvgPooling2D()
self.fc = nn.HybridSequential()
self.fc.add(nn.Dense(int(channel*se_ratio), use_bias=False, in_units=channel),
nn.Swish(),
nn.Dense(int(channel), use_bias=False, in_units=int(channel*se_ratio)),
nn.Activation("sigmoid")) # in mobilenet-v3, this is Hsigmoid
def __init__(self, in_channel, channel, kernel_size, stride, expand=1.0, se_ratio=0.25, res_add=True):
super(BottleNeck, self).__init__()
self.add=res_add
self.out = nn.HybridSequential()
if expand==1.0:
self.out.add(
conv_bn(in_channel, in_channel, kernel_size, stride, groups=in_channel, activation=nn.Swish()),
SEModule(in_channel, se_ratio),
conv_1x1_bn(in_channel, channel, activation=nn.Swish())
)
else:
self.out.add(
conv_1x1_bn(in_channel, channel*expand, activation=nn.Swish()),
conv_bn(channel*expand, channel*expand, kernel_size, stride, groups=channel*expand, activation=nn.Swish()),
SEModule(channel*expand, se_ratio),
conv_1x1_bn(channel*expand, channel, activation=nn.Swish())
)
def __init__(self,
width_coeff=1.0,
depth_coeff=1.0,
dropout_rate=0.0,
scale=1,
se_ratio=0.25,
num_classes=256):
super(EfficientNet, self).__init__()
channels = [32, 16, 24, 40, 80, 112, 192, 320, 1280]
expands = [1, 6, 6, 6, 6, 6, 6]
repeats = [1, 2, 2, 3, 3, 4, 1]
strides = [1, 2, 2, 2, 1, 2, 1]
kernel_sizes = [3, 3, 5, 3, 5, 5, 3]
channels = [round(x * width_coeff)
for x in channels] # [int(x*width) for x in channels]
repeats = [round(x * depth_coeff)
for x in repeats] # [int(x*width) for x in repeats]
self.out = nn.HybridSequential()
if scale != 1:
# Here, should do interpolation to resize input_image to resolution in "bi" mode
# self.out.add(utils.UpSampling(scale))
pass
self.out.add(
nn.Conv2D(channels[0],
3,
2,
padding=1,
use_bias=False,
in_channels=3))
self.out.add(nn.BatchNorm(scale=True))
for i in range(7):
self.out.add(
utils.MBBlock(channels[i], channels[i + 1], repeats[i],
kernel_sizes[i], strides[i], expands[i],
se_ratio))
self.out.add(
utils.conv_1x1_bn(channels[7], channels[8], nn.Swish()),
# utils.AdaptiveAvgPool2D(1),
nn.GlobalAvgPool2D(),
nn.Flatten(),
nn.Dropout(dropout_rate),
nn.Dense(num_classes, use_bias=False, in_units=channels[8]),
nn.BatchNorm(scale=True),
nn.Swish())
def __init__(self, num_filter, kernel, pad, activation, **kwargs):
super(Conv_Block, self).__init__(**kwargs)
with self.name_scope():
self.conv = nn.Conv2D(num_filter, kernel, padding=pad)
self.norm = nn.BatchNorm()
if activation == "swish":
self.act = nn.Swish()
else:
self.act = nn.Activation(activation)
def __init__(self, act_type, channels=1, **kwargs):
super(Activation, self).__init__(**kwargs)
with self.name_scope():
if act_type == 'prelu':
self.activation = PReLU(channels=channels, prefix='prelu_')
elif act_type == 'relu':
self.activation = nn.Activation(act_type, prefix='relu_')
elif act_type == 'swish':
self.activation = nn.Swish(beta=1.0, prefix='swish_')
else:
sys.exit('Wrong activation type: %s' % act_type)
def __init__(self,
n_input,
n_encoder_state,
n_wave_output,
decoder_arch,
n_layers=1,
dropout=0.5,
device=None):
super(MPEWD, self).__init__()
self.wd = []
self.n_path = decoder_arch.n_layers # 暫時用這個代替
print('npath : ', self.n_path)
self.wa = [None] * self.n_path
for p in range(self.n_path):
self.wa[p] = copy.deepcopy(decoder_arch)
self.wa[p].size = [decoder_arch.size[p]]
self.wa[p].n_layers = n_layers
with self.name_scope():
self.e = Encoder(n_input, n_encoder_state, 1, dropout)
for p in range(self.n_path):
self.wd.append(
WaveDecoder(self.wa[p],
n_layers,
device=device,
last=False))
self.register_child(self.wd[p])
self.norm = nn.BatchNorm(axis=2)
#self.norm = nn.LayerNorm(axis = 2)
self.relu = nn.Swish()
#self.fc = nn.Dense(1, activation = "sigmoid", flatten = False)
self.fc = nn.Dense(1, flatten=False)
def __init__(self, act_type, r, skernel, dilation, channels, useReLU, useGlobal, asBackbone,
stride, downsample=False, in_channels=0, norm_layer=BatchNorm,
norm_kwargs=None, **kwargs):
super(ResBlockV2ATAC, self).__init__(**kwargs)
self.bn1 = norm_layer(**({} if norm_kwargs is None else norm_kwargs))
self.conv1 = _conv3x3(channels, stride, in_channels)
self.bn2 = norm_layer(**({} if norm_kwargs is None else norm_kwargs))
self.conv2 = _conv3x3(channels, 1, channels)
if downsample:
self.downsample = nn.Conv2D(channels, 1, stride, use_bias=False, in_channels=in_channels)
else:
self.downsample = None
if act_type == 'relu':
self.msAA1 = nn.Activation('relu')
self.msAA2 = nn.Activation('relu')
elif act_type == 'prelu':
self.msAA1 = nn.PReLU()
self.msAA2 = nn.PReLU()
elif act_type == 'elu':
self.msAA1 = nn.ELU()
self.msAA2 = nn.ELU()
elif act_type == 'selu':
self.msAA1 = nn.SELU()
self.msAA2 = nn.SELU()
elif act_type == 'gelu':
self.msAA1 = nn.GELU()
self.msAA2 = nn.GELU()
elif act_type == 'swish':
self.msAA1 = nn.Swish()
self.msAA2 = nn.Swish()
elif act_type == 'ChaATAC':
self.msAA1 = ChaATAC(channels=in_channels, r=r, useReLU=useReLU, useGlobal=useGlobal)
self.msAA2 = ChaATAC(channels=channels, r=r, useReLU=useReLU, useGlobal=useGlobal)
else:
raise ValueError("Unknown act_type in ResBlockV2ATAC")
def __init__(self, act_func, **kwargs):
super(Activation, self).__init__(**kwargs)
if act_func == "relu":
self.act = nn.Activation('relu')
elif act_func == "relu6":
self.act = ReLU6()
elif act_func == "hard_sigmoid":
self.act = HardSigmoid()
elif act_func == "swish":
self.act = nn.Swish()
elif act_func == "hard_swish":
self.act = HardSwish()
elif act_func == "leaky":
self.act = nn.LeakyReLU(alpha=0.375)
else:
raise NotImplementedError
def get_activation_layer(activation):
assert (activation is not None)
if isfunction(activation):
return activation()
elif isinstance(activation, str):
if activation == "relu6":
return ReLU6()
elif activation == "swish":
return nn.Swish()
elif activation == "hswish":
return HSwish()
elif activation == "hsigmoid":
return HSigmoid()
else:
return nn.Activation(activation)
else:
assert (isinstance(activation, HybridBlock))
return activation
def __init__(self, ctx=mx.cpu(), warmup=10, runs=50, inputs=None):
# Set the default Inputs.
# Default data is (32, 3, 256, 256) to mimic an input of batch_size=128 and a sample image of size 3*256*256.
default_parameters = {"data": (32, 3, 256, 256),
"data_initializer": nd.normal,
"run_backward": True,
"dtype": "float32"}
super().__init__(ctx=ctx, warmup=warmup, runs=runs, default_parameters=default_parameters,
custom_parameters=inputs)
self.data = get_mx_ndarray(ctx=self.ctx, in_tensor=self.inputs["data"],
dtype=self.inputs["dtype"],
initializer=self.inputs["data_initializer"],
attach_grad=self.inputs["run_backward"])
self.block = nn.Swish()
self.block.initialize(ctx=self.ctx)
def _add_conv(out,
channels=1,
kernel=1,
stride=1,
pad=0,
num_group=1,
active=True,
batchnorm=True):
out.add(SamePadding(kernel, stride, dilation=(1, 1)))
out.add(
nn.Conv2D(channels,
kernel,
stride,
pad,
groups=num_group,
use_bias=False))
if batchnorm:
out.add(nn.BatchNorm(scale=True, momentum=0.99, epsilon=1e-3))
if active:
out.add(nn.Swish())
def test_activations_swish():
act_layer = nn.Swish()
out = act_layer(mx.np.random.uniform(size=(10,)))
out.asnumpy()
test_x = np.arange(-20, 20, 0.1)
n_model = 20
n_batch = 5000
batch_size = 64
activations = {
'ReLU': mx.nd.relu,
'Sigmoid': mx.nd.sigmoid,
'Tanh': mx.nd.tanh,
'Relu6': lambda x: mx.nd.clip(mx.nd.relu(x), 0, 6),
'LeakyRelu': mx.nd.LeakyReLU,
'ELU': nn.ELU(),
'SELU': nn.SELU(),
'PReLU': nn.PReLU(),
'Swish': nn.Swish(),
}
legends = []
for act in activations:
test_err = np.zeros_like(test_x)
for i in range(n_model):
print("Train: %s %d/%d" % (act, i + 1, n_model))
net = Net(act=activations[act])
net.collect_params().initialize(mx.init.Xavier(), ctx=ctx)
train(net, train_x[0], train_x[1], batch_size, n_batch)
err = evaluate(net, test_x)
test_err += err
def __init__(self, layers, channels, classes,
act_type, r, skernel, dilation, useReLU, useGlobal, act_layers, replace_act,
act_order, asBackbone, norm_layer=BatchNorm, norm_kwargs=None, **kwargs):
super(ResNet20V2ATAC, self).__init__(**kwargs)
assert len(layers) == len(channels) - 1
with self.name_scope():
self.features = nn.HybridSequential(prefix='')
self.features.add(norm_layer(scale=False, center=False,
**({} if norm_kwargs is None else norm_kwargs)))
self.features.add(nn.Conv2D(channels[0], 3, 1, 1, use_bias=False))
in_channels = channels[0]
for i, num_layer in enumerate(layers):
stride = 1 if i == 0 else 2
if act_order == 'bac':
if i + act_layers < len(channels):
tmp_act_type = replace_act
else:
tmp_act_type = act_type
elif act_order == 'pre':
if i + 1 > act_layers:
tmp_act_type = replace_act
else:
tmp_act_type = act_type
else:
raise ValueError('Unknown act_order')
self.features.add(self._make_layer(
layers=num_layer, channels=channels[i+1], in_channels=in_channels,
stride=stride, stage_index=i+1, act_type=tmp_act_type, r=r, skernel=skernel,
dilation=dilation, useReLU=useReLU, useGlobal=useGlobal,
asBackbone=asBackbone, norm_layer=norm_layer, norm_kwargs=norm_kwargs
))
in_channels = channels[i+1]
self.features.add(norm_layer(**({} if norm_kwargs is None else norm_kwargs)))
if act_order == 'bac':
if act_layers <= 0:
tmp_act_type = replace_act
else:
tmp_act_type = act_type
elif act_order == 'pre':
if act_layers >= 4:
tmp_act_type = act_type
else:
tmp_act_type = replace_act
else:
raise ValueError('Unknown act_order')
if tmp_act_type == 'relu':
self.features.add(nn.Activation('relu'))
elif tmp_act_type == 'prelu':
self.features.add(nn.PReLU())
elif tmp_act_type == 'elu':
self.features.add(nn.ELU())
elif tmp_act_type == 'selu':
self.features.add(nn.SELU())
elif tmp_act_type == 'gelu':
self.features.add(nn.GELU())
elif tmp_act_type == 'swish':
self.features.add(nn.Swish())
elif tmp_act_type == 'ChaATAC':
self.features.add(ChaATAC(channels=in_channels, r=r, useReLU=useReLU,
useGlobal=useGlobal))
else:
raise ValueError("Unknown act_type in ResBlockV2ATAC")
self.features.add(nn.GlobalAvgPool2D())
self.features.add(nn.Flatten())
self.output = nn.Dense(classes, in_units=in_channels)
