def construct(self, hidden):
h, c = hidden # h, c dim = 2 x b x hidden_dim
h_in = P.Transpose()(h, (1, 0, 2)).view(-1, self.hidden_dim * 2)
hidden_reduced_h = P.ReLU()(self.reduce_h(h))
hidden_reduced_h = P.ExpandDims()(hidden_reduced_h, 0)
c_in = P.Transpose()(c, (1, 0, 2)).view(-1, self.hidden_dim * 2)
hidden_reduced_c = P.ReLU()(self.reduce_c(c_in))
hidden_reduced_c = P.ExpandDims()(hidden_reduced_c, 0)
return (hidden_reduced_h, hidden_reduced_c)
def __init__(self, cin, cout, stride=1, dilation=1):
super(BasicBlock, self).__init__()
self.conv_bn_act = nn.Conv2dBnAct(cin,
cout,
kernel_size=3,
stride=stride,
pad_mode='pad',
padding=dilation,
has_bias=False,
dilation=dilation,
has_bn=True,
momentum=BN_MOMENTUM,
activation='relu',
after_fake=False)
self.conv_bn = nn.Conv2dBnAct(cout,
cout,
kernel_size=3,
stride=1,
pad_mode='same',
has_bias=False,
dilation=dilation,
has_bn=True,
momentum=BN_MOMENTUM,
activation=None)
self.relu = ops.ReLU()
def __init__(self, lower: float = 1. / 8, upper: float = 1. / 3, seed=0):
super().__init__()
seed1, seed2 = _get_graph_seed(seed, 'uniform')
self.uniform = ops.UniformReal(seed1, seed2)
self.lower = Tensor(lower, mstype.float32)
self.upper = Tensor(upper, mstype.float32)
self.relu = ops.ReLU()
def __init__(self,
in_channels,
out_channels,
stride=1,
momentum=0.9):
super(ResidualBlock, self).__init__()
out_chls = out_channels // self.expansion
self.conv1 = _conv1x1(in_channels, out_chls, stride=1)
self.bn1 = _fused_bn(out_chls, momentum=momentum)
self.conv2 = _conv3x3(out_chls, out_chls, stride=stride)
self.bn2 = _fused_bn(out_chls, momentum=momentum)
self.conv3 = _conv1x1(out_chls, out_channels, stride=1)
self.bn3 = _fused_bn_last(out_channels, momentum=momentum)
self.relu = ops.ReLU()
self.downsample = (in_channels != out_channels)
self.stride = stride
if self.downsample:
self.conv_down_sample = _conv1x1(in_channels, out_channels,
stride=stride)
self.bn_down_sample = _fused_bn(out_channels, momentum=momentum)
elif self.stride != 1:
self.maxpool_down = nn.MaxPool2d(kernel_size=1, stride=2, pad_mode='same')
def __init__(self, block, num_classes=100, batch_size=32):
"""init"""
super(ResNet, self).__init__()
self.batch_size = batch_size
self.num_classes = num_classes
self.conv1 = conv7x7(3, 64, stride=2, padding=0)
self.bn1 = bn_with_initialize(64)
self.relu = ops.ReLU()
self.maxpool = nn.MaxPool2d(kernel_size=3, stride=2, pad_mode="same")
self.layer1 = MakeLayer0(block,
in_channels=64,
out_channels=256,
stride=1)
self.layer2 = MakeLayer1(block,
in_channels=256,
out_channels=512,
stride=2)
self.layer3 = MakeLayer2(block,
in_channels=512,
out_channels=1024,
stride=2)
self.layer4 = MakeLayer3(block,
in_channels=1024,
out_channels=2048,
stride=2)
self.pool = ops.ReduceMean(keep_dims=True)
self.squeeze = ops.Squeeze(axis=(2, 3))
self.fc = fc_with_initialize(512 * block.expansion, num_classes)
def rnn_relu_cell(input, hidden, w_ih, w_hh, b_ih, b_hh):
if b_ih is None:
igates = P.MatMul(False, True)(input, w_ih)
hgates = P.MatMul(False, True)(hidden, w_hh)
else:
igates = P.MatMul(False, True)(input, w_ih) + b_ih
hgates = P.MatMul(False, True)(hidden, w_hh) + b_hh
return P.ReLU()(igates + hgates)
def _get_activation(activation):
if activation == 'relu':
return P.ReLU()
elif activation == 'gelu':
return P.GeLU()
raise ValueError(
'Invalid activation operator. Activation should be relu/gelu, not {}'.
format(activation))
def rnn_relu_cell(inputs, hidden, w_ih, w_hh, b_ih, b_hh):
'''RNN cell function with relu activation'''
if b_ih is None:
igates = P.MatMul(False, True)(inputs, w_ih)
hgates = P.MatMul(False, True)(hidden, w_hh)
else:
igates = P.MatMul(False, True)(inputs, w_ih) + b_ih
hgates = P.MatMul(False, True)(hidden, w_hh) + b_hh
return P.ReLU()(igates + hgates)
def __init__(self, in_channels, out_channels, kernel_size, residual):
super(Root, self).__init__()
self.conv = nn.Conv2d(in_channels,
out_channels,
1,
stride=1,
has_bias=False,
pad_mode='pad',
padding=(kernel_size - 1) // 2)
self.bn = nn.BatchNorm2d(out_channels, momentum=BN_MOMENTUM)
self.relu = ops.ReLU()
self.residual = residual
self.cat = ops.Concat(axis=1)
def __init__(self, in_channels, out_channels, stride=1):
"""init block"""
super(ResidualBlock, self).__init__()
out_chls = out_channels // self.expansion
self.conv1 = conv1x1(in_channels, out_chls, stride=stride, padding=0)
self.bn1 = bn_with_initialize(out_chls)
self.conv2 = conv3x3(out_chls, out_chls, stride=1, padding=0)
self.bn2 = bn_with_initialize(out_chls)
self.conv3 = conv1x1(out_chls, out_channels, stride=1, padding=0)
self.bn3 = bn_with_initialize_last(out_channels)
self.relu = ops.ReLU()
self.add = ops.Add()
def __init__(self,
block,
layer_nums,
in_channels,
out_channels,
strides=(1, 2, 2, 2),
num_classes=100):
super(ResNet, self).__init__()
if not len(layer_nums) == len(in_channels) == len(out_channels) == 4:
raise ValueError("the length of "
"layer_num, inchannel, outchannel list must be 4!")
self.conv1 = _conv7x7(3, 64, stride=2)
self.bn1 = _fused_bn(64)
self.relu = ops.ReLU()
self.maxpool = nn.MaxPool2d(kernel_size=3, stride=2, pad_mode='same')
self.layer1 = self._make_layer(block,
layer_nums[0],
in_channel=in_channels[0],
out_channel=out_channels[0],
stride=strides[0])
self.layer2 = self._make_layer(block,
layer_nums[1],
in_channel=in_channels[1],
out_channel=out_channels[1],
stride=strides[1])
self.layer3 = self._make_layer(block,
layer_nums[2],
in_channel=in_channels[2],
out_channel=out_channels[2],
stride=strides[2])
self.layer4 = self._make_layer(block,
layer_nums[3],
in_channel=in_channels[3],
out_channel=out_channels[3],
stride=strides[3])
self.mean = ops.ReduceMean(keep_dims=True)
self.end_point = nn.Dense(out_channels[3], num_classes, has_bias=True,
weight_init=dense_weight_variable())
self.squeeze = ops.Squeeze()
self.cast = ops.Cast()
def __init__(self,
block,
layer_nums,
in_channels,
out_channels,
strides,
num_classes):
super(ResNet, self).__init__()
if not len(layer_nums) == len(in_channels) == len(out_channels) == 4:
raise ValueError("the length of layer_num, in_channels, out_channels list must be 4!")
self.conv1 = _conv7x7(3, 64, stride=2)
self.bn1 = _bn(64)
self.relu = ops.ReLU()
self.maxpool = nn.MaxPool2d(kernel_size=3, stride=2, pad_mode="same")
self.layer1 = self._make_layer(block,
layer_nums[0],
in_channel=in_channels[0],
out_channel=out_channels[0],
stride=strides[0])
self.layer2 = self._make_layer(block,
layer_nums[1],
in_channel=in_channels[1],
out_channel=out_channels[1],
stride=strides[1])
self.layer3 = self._make_layer(block,
layer_nums[2],
in_channel=in_channels[2],
out_channel=out_channels[2],
stride=strides[2])
self.layer4 = self._make_layer(block,
layer_nums[3],
in_channel=in_channels[3],
out_channel=out_channels[3],
stride=strides[3])
self.mean = ops.ReduceMean(keep_dims=True)
self.flatten = nn.Flatten()
self.end_point = _fc(out_channels[3], num_classes)
def __init__(self,
in_channels,
out_channels,
stride=1,
momentum=0.9):
super(BasicBlock, self).__init__()
self.conv1 = _conv3x3(in_channels, out_channels, stride=stride)
self.bn1 = _fused_bn(out_channels, momentum=momentum)
self.conv2 = _conv3x3(out_channels, out_channels)
self.bn2 = _fused_bn(out_channels, momentum=momentum)
self.relu = ops.ReLU()
self.down_sample_layer = None
self.downsample = (in_channels != out_channels)
if self.downsample:
self.down_sample_layer = nn.SequentialCell([_conv1x1(in_channels,
out_channels,
stride=stride,
padding=0),
_fused_bn(out_channels,
momentum=momentum)])
def __init__(self, in_channels, out_channels, stride=1, down_sample=False):
"""init block with down"""
super(ResidualBlockWithDown, self).__init__()
out_chls = out_channels // self.expansion
self.conv1 = conv1x1(in_channels, out_chls, stride=stride, padding=0)
self.bn1 = bn_with_initialize(out_chls)
self.conv2 = conv3x3(out_chls, out_chls, stride=1, padding=0)
self.bn2 = bn_with_initialize(out_chls)
self.conv3 = conv1x1(out_chls, out_channels, stride=1, padding=0)
self.bn3 = bn_with_initialize_last(out_channels)
self.relu = ops.ReLU()
self.down_sample = down_sample
self.conv_down_sample = conv1x1(in_channels,
out_channels,
stride=stride,
padding=0)
self.bn_down_sample = bn_with_initialize(out_channels)
self.add = ops.Add()
def __init__(self):
super(Head, self).__init__()
self.conv1 = conv7x7(3, 64, stride=2, padding=0)
self.bn1 = bn_with_initialize(64)
self.relu = ops.ReLU()
self.maxpool = nn.MaxPool2d(kernel_size=3, stride=2, pad_mode="same")
def __init__(self):
super(Stable_softplus, self).__init__()
self.log_op = P.Log()
self.abs_op = P.Abs()
self.relu_op = P.ReLU()
self.exp_op = P.Exp()
import os
import logging
from typing import Callable, Dict, Union, Any
import mindspore.nn as nn
import mindspore.ops as ops
from mindspore.train.serialization import load_checkpoint, load_param_into_net
from transformer import builder
logger = logging.getLogger(__name__)
ACT2FN = {
"relu": ops.ReLU(),
"swish": ops.HSwish(),
"gelu": ops.GeLU(),
"tanh": ops.Tanh(),
}
def get_activation(activation_string: str):
return ACT2FN[activation_string]
class MSPreTrainedModel(nn.Cell):
def __init__(self, config: Union[Dict[str, Any], Callable[..., None]],
**kwargs) -> None:
super().__init__(**kwargs)
if isinstance(config, Dict):
config = builder.build_config(config)
self.config = config
def relu(input):
return ops.ReLU()(input)
