def __init__(self,
in_channel,
out_channel,
stride=2,
kernel_size=(3, 3, 3),
down=True,
is_output=False):
super().__init__()
self.stride = stride
self.down = down
self.in_channel = in_channel
self.out_channel = out_channel
self.down_conv_1 = nn.Conv3d(in_channel, out_channel, kernel_size=(3, 3, 3), \
pad_mode="pad", stride=self.stride, padding=1)
self.is_output = is_output
if not is_output:
self.batchNormal1 = BatchNorm3d(num_features=self.out_channel)
self.relu1 = nn.PReLU()
if self.down:
self.down_conv_2 = nn.Conv3d(out_channel, out_channel, kernel_size=(3, 3, 3), \
pad_mode="pad", stride=1, padding=1)
self.relu2 = nn.PReLU()
if kernel_size[0] == 1:
self.residual = nn.Conv3d(in_channel, out_channel, kernel_size=(1, 1, 1), \
pad_mode="valid", stride=self.stride)
else:
self.residual = nn.Conv3d(in_channel, out_channel, kernel_size=(3, 3, 3), \
pad_mode="pad", stride=self.stride, padding=1)
self.batchNormal2 = BatchNorm3d(num_features=self.out_channel)
def __init__(self, vocab_size, embedding_size, target='CPU'):
super().__init__()
self.embedding_lookup = nn.EmbeddingLookup(vocab_size, embedding_size, param_init='ones', target=target)
self.bn = nn.BatchNorm2d(num_features=3)
self.mul = P.Mul()
self.reshape = P.Reshape()
self.relu = nn.PReLU()
def __init__(self,
inplanes,
planes,
stride=1,
downsample=None,
use_se=1,
pre_bn=1,
use_inference=0,
act_type='relu'):
super(IRBlock, self).__init__()
if pre_bn == 1:
self.bn1 = bn_with_initialize(inplanes,
use_inference=use_inference)
else:
self.bn1 = Cut()
self.conv1 = conv3x3(inplanes, planes, stride=1)
self.bn2 = bn_with_initialize(planes, use_inference=use_inference)
self.act_layer = nn.PReLU(planes) if act_type == 'prelu' else P.ReLU()
self.conv2 = conv3x3(planes, planes, stride=stride)
self.bn3 = bn_with_initialize(planes, use_inference=use_inference)
if downsample is None:
self.downsample = Cut()
else:
self.downsample = downsample
self.use_se = use_se
if use_se == 1:
self.se = SEBlock(planes, act_type=act_type)
self.add = TensorAdd()
self.cast = P.Cast()
def __init__(self, param_np, target='CPU'):
super().__init__()
self.param = Parameter(Tensor(param_np), name="w1")
self.embedding_lookup = nn.EmbeddingLookup(target=target)
self.bn = nn.BatchNorm2d(num_features=3)
self.mul = P.Mul()
self.reshape = P.Reshape()
self.relu = nn.PReLU()
def __init__(self, inplanes, planes):
super(IRBlockZ, self).__init__()
self.conv1 = nn.Conv2d(inplanes,
planes,
kernel_size=3,
stride=1,
pad_mode="same",
group=1,
has_bias=False,
dilation=1)
self.act_layer = nn.PReLU(planes)
def __init__(self, channel, reduction=16, act_type='relu'):
super(SEBlock, self).__init__()
self.fc1 = fc_with_initialize(channel, channel // reduction)
self.act_layer = nn.PReLU(
channel // reduction) if act_type == 'prelu' else P.ReLU()
self.fc2 = fc_with_initialize(channel // reduction, channel)
self.sigmoid = Sigmoid().add_flags_recursive(fp32=True)
self.reshape = P.Reshape()
self.shape = P.Shape()
self.reduce_mean = P.ReduceMean(True)
self.cast = P.Cast()
def __init__(self, in_channel, down_in_channel, out_channel, stride=2, is_output=False, dtype=mstype.float16):
super().__init__()
self.in_channel = in_channel
self.down_in_channel = down_in_channel
self.out_channel = out_channel
self.stride = stride
self.conv3d_transpose = Conv3DTranspose(in_channel=self.in_channel + self.down_in_channel, \
pad=1, out_channel=self.out_channel, kernel_size=(3, 3, 3), \
stride=self.stride, output_padding=(1, 1, 1))
self.concat = P.Concat(axis=1)
self.conv = ResidualUnit(self.out_channel, self.out_channel, stride=1, down=False, \
is_output=is_output).to_float(dtype)
self.batchNormal1 = BatchNorm3d(num_features=self.out_channel)
self.relu = nn.PReLU()
def test_auto_parallel_assign_sub_with_ref_key():
size = 8
context.set_auto_parallel_context(device_num=size, global_rank=0)
x = Tensor(np.random.rand(4, 4, 32, 64), dtype=ms.float32)
net = NetWithLoss(nn.PReLU(4))
context.set_auto_parallel_context(parallel_mode="auto_parallel")
reset_op_id()
_executor.compile(net, x, phase="train")
strategies = _executor._get_strategy(net)
for (k, v) in strategies.items():
if re.search('PReLU-op', k) is not None:
assert v == [[1, 1, 1, 8], [1]]
elif re.search('ReLU-op', k) is not None:
assert v == [[1]]
def test_auto_parallel_assign_sub_with_ref_key():
size = 8
context.set_auto_parallel_context(device_num=size, global_rank=0)
x = Tensor(np.random.rand(4, 4, 32, 64), dtype=ms.float32)
net = NetWithLoss(nn.PReLU(4))
context.set_auto_parallel_context(parallel_mode="auto_parallel")
reset_op_id()
_executor.compile(net, x, phase="train")
strategies = _executor._get_strategy(net)
expected_strategies = {
'Default/network-PReLU/PReLU-op2': [[1, 1, 1, 8], [1]],
'Default/network-PReLU/ReLU-op3': [[1]]
}
assert strategies == expected_strategies
def __init__(self):
super(BatchNormReshapeNet, self).__init__()
self.vd = P._VirtualDataset()
self.batch_norm = nn.BatchNorm1d(512, affine=False)
self.reshape = P.Reshape()
self.prelu = nn.PReLU(channel=256)
def __init__(self):
super().__init__()
self.bn = bn_with_initialize(16)
self.prelu = nn.PReLU(16)
def __init__(self):
super(NetForPReLU, self).__init__()
self.prelu = nn.PReLU()
'desc_bprop': [[1, 16]],
'skip': ['backward']}),
('L2Normalize', {
'block': P.L2Normalize(),
'desc_inputs': [[4, 128, 1024]],
'desc_bprop': [[4, 128, 1024]]}),
('ReLU', {
'block': P.ReLU(),
'desc_inputs': [[64, 64, 112, 112]],
'desc_bprop': [[64, 64, 112, 112]]}),
('SeqConvBnRelu', {
'block': SeqConvBnRelu(3, 64),
'desc_inputs': [[64, 3, 112, 112]],
'desc_bprop': [[64, 64, 112, 112]]}),
('PReluCell', {
'block': nn.PReLU(1, [np.float32(0.25)]),
'desc_inputs': [[128, 64, 112, 112]],
'desc_bprop': [[128, 64, 112, 112]]}),
('PRelu', {
'block': P.PReLU(),
'desc_inputs': [[128, 64, 112, 112], [64,]],
'desc_bprop': [[128, 64, 112, 112]]}),
('Cos', {
'block': P.Cos(),
'desc_inputs': [[8, 16]],
'desc_bprop': [[8, 16]]}),
('ACos', {
'block': P.ACos(),
'desc_inputs': [[8, 16]],
'desc_bprop': [[8, 16]]}),
('Exp', {
def __init__(self, block, layers, args):
super(FaceResNet, self).__init__()
self.act_type = args.act_type
self.inplanes = 64
self.use_se = args.use_se
self.conv1 = conv3x3(3, 64, stride=1)
self.bn1 = bn_with_initialize(64, use_inference=args.inference)
self.prelu = nn.PReLU(64) if self.act_type == 'prelu' else P.ReLU()
self.layer1 = MakeLayer(block,
planes=64,
inplanes=self.inplanes,
blocks=layers[0],
stride=2,
args=args)
self.inplanes = 64
self.layer2 = MakeLayer(block,
planes=128,
inplanes=self.inplanes,
blocks=layers[1],
stride=2,
args=args)
self.inplanes = 128
self.layer3 = MakeLayer(block,
planes=256,
inplanes=self.inplanes,
blocks=layers[2],
stride=2,
args=args)
self.inplanes = 256
self.layer4 = MakeLayer(block,
planes=512,
inplanes=self.inplanes,
blocks=layers[3],
stride=2,
args=args)
self.head = get_head(args)
np.random.seed(1)
for _, cell in self.cells_and_names():
if isinstance(cell, nn.Conv2d):
cell.weight.set_data(
initializer(
me_init.ReidKaimingUniform(a=math.sqrt(5),
mode='fan_out'),
cell.weight.shape))
if cell.bias is not None:
cell.bias.set_data(initializer('zeros', cell.bias.shape))
elif isinstance(cell, nn.Dense):
cell.weight.set_data(
initializer(
me_init.ReidKaimingNormal(a=math.sqrt(5),
mode='fan_out'),
cell.weight.shape))
if cell.bias is not None:
cell.bias.set_data(initializer('zeros', cell.bias.shape))
elif isinstance(cell, (nn.BatchNorm2d, nn.BatchNorm1d)):
# defulat gamma 1 and beta 0, and if you set should be careful for the IRBlock gamma value
pass
for _, cell in self.cells_and_names():
if isinstance(cell, IRBlock):
# be careful for bn3 Do not change the name unless IRBlock last bn change name
cell.bn3.gamma.set_data(
initializer('zeros', cell.bn3.gamma.shape))
def MyPReLU():
return nn.PReLU()
