def __init__(self, in_channel, out_channel, stride=2, kernel_size=(3, 3, 3), down=True, is_output=False): super().__init__() self.stride = stride self.down = down self.in_channel = in_channel self.out_channel = out_channel self.down_conv_1 = nn.Conv3d(in_channel, out_channel, kernel_size=(3, 3, 3), \ pad_mode="pad", stride=self.stride, padding=1) self.is_output = is_output if not is_output: self.batchNormal1 = BatchNorm3d(num_features=self.out_channel) self.relu1 = nn.PReLU() if self.down: self.down_conv_2 = nn.Conv3d(out_channel, out_channel, kernel_size=(3, 3, 3), \ pad_mode="pad", stride=1, padding=1) self.relu2 = nn.PReLU() if kernel_size[0] == 1: self.residual = nn.Conv3d(in_channel, out_channel, kernel_size=(1, 1, 1), \ pad_mode="valid", stride=self.stride) else: self.residual = nn.Conv3d(in_channel, out_channel, kernel_size=(3, 3, 3), \ pad_mode="pad", stride=self.stride, padding=1) self.batchNormal2 = BatchNorm3d(num_features=self.out_channel)
def __init__(self, vocab_size, embedding_size, target='CPU'): super().__init__() self.embedding_lookup = nn.EmbeddingLookup(vocab_size, embedding_size, param_init='ones', target=target) self.bn = nn.BatchNorm2d(num_features=3) self.mul = P.Mul() self.reshape = P.Reshape() self.relu = nn.PReLU()
def __init__(self, inplanes, planes, stride=1, downsample=None, use_se=1, pre_bn=1, use_inference=0, act_type='relu'): super(IRBlock, self).__init__() if pre_bn == 1: self.bn1 = bn_with_initialize(inplanes, use_inference=use_inference) else: self.bn1 = Cut() self.conv1 = conv3x3(inplanes, planes, stride=1) self.bn2 = bn_with_initialize(planes, use_inference=use_inference) self.act_layer = nn.PReLU(planes) if act_type == 'prelu' else P.ReLU() self.conv2 = conv3x3(planes, planes, stride=stride) self.bn3 = bn_with_initialize(planes, use_inference=use_inference) if downsample is None: self.downsample = Cut() else: self.downsample = downsample self.use_se = use_se if use_se == 1: self.se = SEBlock(planes, act_type=act_type) self.add = TensorAdd() self.cast = P.Cast()
def __init__(self, param_np, target='CPU'): super().__init__() self.param = Parameter(Tensor(param_np), name="w1") self.embedding_lookup = nn.EmbeddingLookup(target=target) self.bn = nn.BatchNorm2d(num_features=3) self.mul = P.Mul() self.reshape = P.Reshape() self.relu = nn.PReLU()
def __init__(self, inplanes, planes): super(IRBlockZ, self).__init__() self.conv1 = nn.Conv2d(inplanes, planes, kernel_size=3, stride=1, pad_mode="same", group=1, has_bias=False, dilation=1) self.act_layer = nn.PReLU(planes)
def __init__(self, channel, reduction=16, act_type='relu'): super(SEBlock, self).__init__() self.fc1 = fc_with_initialize(channel, channel // reduction) self.act_layer = nn.PReLU( channel // reduction) if act_type == 'prelu' else P.ReLU() self.fc2 = fc_with_initialize(channel // reduction, channel) self.sigmoid = Sigmoid().add_flags_recursive(fp32=True) self.reshape = P.Reshape() self.shape = P.Shape() self.reduce_mean = P.ReduceMean(True) self.cast = P.Cast()
def __init__(self, in_channel, down_in_channel, out_channel, stride=2, is_output=False, dtype=mstype.float16): super().__init__() self.in_channel = in_channel self.down_in_channel = down_in_channel self.out_channel = out_channel self.stride = stride self.conv3d_transpose = Conv3DTranspose(in_channel=self.in_channel + self.down_in_channel, \ pad=1, out_channel=self.out_channel, kernel_size=(3, 3, 3), \ stride=self.stride, output_padding=(1, 1, 1)) self.concat = P.Concat(axis=1) self.conv = ResidualUnit(self.out_channel, self.out_channel, stride=1, down=False, \ is_output=is_output).to_float(dtype) self.batchNormal1 = BatchNorm3d(num_features=self.out_channel) self.relu = nn.PReLU()
def test_auto_parallel_assign_sub_with_ref_key(): size = 8 context.set_auto_parallel_context(device_num=size, global_rank=0) x = Tensor(np.random.rand(4, 4, 32, 64), dtype=ms.float32) net = NetWithLoss(nn.PReLU(4)) context.set_auto_parallel_context(parallel_mode="auto_parallel") reset_op_id() _executor.compile(net, x, phase="train") strategies = _executor._get_strategy(net) for (k, v) in strategies.items(): if re.search('PReLU-op', k) is not None: assert v == [[1, 1, 1, 8], [1]] elif re.search('ReLU-op', k) is not None: assert v == [[1]]
def test_auto_parallel_assign_sub_with_ref_key(): size = 8 context.set_auto_parallel_context(device_num=size, global_rank=0) x = Tensor(np.random.rand(4, 4, 32, 64), dtype=ms.float32) net = NetWithLoss(nn.PReLU(4)) context.set_auto_parallel_context(parallel_mode="auto_parallel") reset_op_id() _executor.compile(net, x, phase="train") strategies = _executor._get_strategy(net) expected_strategies = { 'Default/network-PReLU/PReLU-op2': [[1, 1, 1, 8], [1]], 'Default/network-PReLU/ReLU-op3': [[1]] } assert strategies == expected_strategies
def __init__(self): super(BatchNormReshapeNet, self).__init__() self.vd = P._VirtualDataset() self.batch_norm = nn.BatchNorm1d(512, affine=False) self.reshape = P.Reshape() self.prelu = nn.PReLU(channel=256)
def __init__(self): super().__init__() self.bn = bn_with_initialize(16) self.prelu = nn.PReLU(16)
def __init__(self): super(NetForPReLU, self).__init__() self.prelu = nn.PReLU()
'desc_bprop': [[1, 16]], 'skip': ['backward']}), ('L2Normalize', { 'block': P.L2Normalize(), 'desc_inputs': [[4, 128, 1024]], 'desc_bprop': [[4, 128, 1024]]}), ('ReLU', { 'block': P.ReLU(), 'desc_inputs': [[64, 64, 112, 112]], 'desc_bprop': [[64, 64, 112, 112]]}), ('SeqConvBnRelu', { 'block': SeqConvBnRelu(3, 64), 'desc_inputs': [[64, 3, 112, 112]], 'desc_bprop': [[64, 64, 112, 112]]}), ('PReluCell', { 'block': nn.PReLU(1, [np.float32(0.25)]), 'desc_inputs': [[128, 64, 112, 112]], 'desc_bprop': [[128, 64, 112, 112]]}), ('PRelu', { 'block': P.PReLU(), 'desc_inputs': [[128, 64, 112, 112], [64,]], 'desc_bprop': [[128, 64, 112, 112]]}), ('Cos', { 'block': P.Cos(), 'desc_inputs': [[8, 16]], 'desc_bprop': [[8, 16]]}), ('ACos', { 'block': P.ACos(), 'desc_inputs': [[8, 16]], 'desc_bprop': [[8, 16]]}), ('Exp', {
def __init__(self, block, layers, args): super(FaceResNet, self).__init__() self.act_type = args.act_type self.inplanes = 64 self.use_se = args.use_se self.conv1 = conv3x3(3, 64, stride=1) self.bn1 = bn_with_initialize(64, use_inference=args.inference) self.prelu = nn.PReLU(64) if self.act_type == 'prelu' else P.ReLU() self.layer1 = MakeLayer(block, planes=64, inplanes=self.inplanes, blocks=layers[0], stride=2, args=args) self.inplanes = 64 self.layer2 = MakeLayer(block, planes=128, inplanes=self.inplanes, blocks=layers[1], stride=2, args=args) self.inplanes = 128 self.layer3 = MakeLayer(block, planes=256, inplanes=self.inplanes, blocks=layers[2], stride=2, args=args) self.inplanes = 256 self.layer4 = MakeLayer(block, planes=512, inplanes=self.inplanes, blocks=layers[3], stride=2, args=args) self.head = get_head(args) np.random.seed(1) for _, cell in self.cells_and_names(): if isinstance(cell, nn.Conv2d): cell.weight.set_data( initializer( me_init.ReidKaimingUniform(a=math.sqrt(5), mode='fan_out'), cell.weight.shape)) if cell.bias is not None: cell.bias.set_data(initializer('zeros', cell.bias.shape)) elif isinstance(cell, nn.Dense): cell.weight.set_data( initializer( me_init.ReidKaimingNormal(a=math.sqrt(5), mode='fan_out'), cell.weight.shape)) if cell.bias is not None: cell.bias.set_data(initializer('zeros', cell.bias.shape)) elif isinstance(cell, (nn.BatchNorm2d, nn.BatchNorm1d)): # defulat gamma 1 and beta 0, and if you set should be careful for the IRBlock gamma value pass for _, cell in self.cells_and_names(): if isinstance(cell, IRBlock): # be careful for bn3 Do not change the name unless IRBlock last bn change name cell.bn3.gamma.set_data( initializer('zeros', cell.bn3.gamma.shape))
def MyPReLU(): return nn.PReLU()