def __init__(self, in_channel_list, out_channel_list, kernel_size_list=3, expand_ratio_list=6, stride=1, act_func='relu6', use_se=False): super(DynamicMBConvLayer, self).__init__() self.in_channel_list = in_channel_list self.out_channel_list = out_channel_list self.kernel_size_list = val2list(kernel_size_list) self.expand_ratio_list = val2list(expand_ratio_list) self.stride = stride self.act_func = act_func self.use_se = use_se # build modules max_middle_channel = make_divisible( round(max(self.in_channel_list) * max(self.expand_ratio_list)), MyNetwork.CHANNEL_DIVISIBLE) if max(self.expand_ratio_list) == 1: self.inverted_bottleneck = None else: self.inverted_bottleneck = nn.Sequential( OrderedDict([ ('conv', DynamicConv2d(max(self.in_channel_list), max_middle_channel)), ('bn', DynamicBatchNorm2d(max_middle_channel)), ('act', build_activation(self.act_func)), ])) self.depth_conv = nn.Sequential( OrderedDict([('conv', DynamicSeparableConv2d(max_middle_channel, self.kernel_size_list, self.stride)), ('bn', DynamicBatchNorm2d(max_middle_channel)), ('act', build_activation(self.act_func))])) if self.use_se: self.depth_conv.add_module('se', DynamicSE(max_middle_channel)) self.point_linear = nn.Sequential( OrderedDict([ ('conv', DynamicConv2d(max_middle_channel, max(self.out_channel_list))), ('bn', DynamicBatchNorm2d(max(self.out_channel_list))), ])) self.active_kernel_size = max(self.kernel_size_list) self.active_expand_ratio = max(self.expand_ratio_list) self.active_out_channel = max(self.out_channel_list)
def __init__(self, in_channel_list, out_channel_list, expand_ratio_list=0.25, kernel_size=3, stride=1, act_func='relu', downsample_mode='avgpool_conv'): super(DynamicResNetBottleneckBlock, self).__init__() self.in_channel_list = in_channel_list self.out_channel_list = out_channel_list self.expand_ratio_list = val2list(expand_ratio_list) self.kernel_size = kernel_size self.stride = stride self.act_func = act_func self.downsample_mode = downsample_mode # build modules max_middle_channel = make_divisible( round(max(self.out_channel_list) * max(self.expand_ratio_list)), MyNetwork.CHANNEL_DIVISIBLE) self.conv1 = nn.Sequential(OrderedDict([ ('conv', DynamicConv2d(max(self.in_channel_list), max_middle_channel)), ('bn', DynamicBatchNorm2d(max_middle_channel)), ('act', build_activation(self.act_func, inplace=True)), ])) self.conv2 = nn.Sequential(OrderedDict([ ('conv', DynamicConv2d(max_middle_channel, max_middle_channel, kernel_size, stride)), ('bn', DynamicBatchNorm2d(max_middle_channel)), ('act', build_activation(self.act_func, inplace=True)) ])) self.conv3 = nn.Sequential(OrderedDict([ ('conv', DynamicConv2d(max_middle_channel, max(self.out_channel_list))), ('bn', DynamicBatchNorm2d(max(self.out_channel_list))), ])) if self.stride == 1 and self.in_channel_list == self.out_channel_list: self.downsample = IdentityLayer(max(self.in_channel_list), max(self.out_channel_list)) elif self.downsample_mode == 'conv': self.downsample = nn.Sequential(OrderedDict([ ('conv', DynamicConv2d(max(self.in_channel_list), max(self.out_channel_list), stride=stride)), ('bn', DynamicBatchNorm2d(max(self.out_channel_list))), ])) elif self.downsample_mode == 'avgpool_conv': self.downsample = nn.Sequential(OrderedDict([ ('avg_pool', nn.AvgPool2d(kernel_size=stride, stride=stride, padding=0, ceil_mode=True)), ('conv', DynamicConv2d(max(self.in_channel_list), max(self.out_channel_list))), ('bn', DynamicBatchNorm2d(max(self.out_channel_list))), ])) else: raise NotImplementedError self.final_act = build_activation(self.act_func, inplace=True) self.active_expand_ratio = max(self.expand_ratio_list) self.active_out_channel = max(self.out_channel_list)
def __init__(self, in_channel_list, out_channel_list, kernel_size=3, stride=1, dilation=1, use_bn=True, act_func='relu6'): super(DynamicConvLayer, self).__init__() self.in_channel_list = in_channel_list self.out_channel_list = out_channel_list self.kernel_size = kernel_size self.stride = stride self.dilation = dilation self.use_bn = use_bn self.act_func = act_func self.conv = DynamicConv2d( max_in_channels=max(self.in_channel_list), max_out_channels=max(self.out_channel_list), kernel_size=self.kernel_size, stride=self.stride, dilation=self.dilation, ) if self.use_bn: self.bn = DynamicBatchNorm2d(max(self.out_channel_list)) self.act = build_activation(self.act_func) self.active_out_channel = max(self.out_channel_list)
def __init__(self, in_channels, out_channels, kernel_size=3, stride=1, expand_ratio=6, mid_channels=None, act_func='relu6', use_se=False, groups=1): super(ReducedMBConvLayer, self).__init__() self.in_channels = in_channels self.out_channels = out_channels self.kernel_size = kernel_size self.stride = stride self.expand_ratio = expand_ratio self.mid_channels = mid_channels self.act_func = act_func self.use_se = use_se self.groups = groups if self.mid_channels is None: feature_dim = round(self.in_channels * self.expand_ratio) else: feature_dim = self.mid_channels pad = get_same_padding(self.kernel_size) groups = feature_dim if self.groups is None else min_divisible_value( feature_dim, self.groups) self.expand_conv = nn.Sequential( OrderedDict({ 'conv': nn.Conv2d(in_channels, feature_dim, kernel_size, stride, pad, groups=groups, bias=False), 'bn': nn.BatchNorm2d(feature_dim), 'act': build_activation(self.act_func, inplace=True), })) if self.use_se: self.expand_conv.add_module('se', SEModule(feature_dim)) self.reduce_conv = nn.Sequential( OrderedDict({ 'conv': nn.Conv2d(feature_dim, out_channels, 1, 1, 0, bias=False), 'bn': nn.BatchNorm2d(out_channels), }))
def __init__(self, in_channels, out_channels, kernel_size=3, stride=1, expand_ratio=6, mid_channels=None, act_func='relu6', use_se=False): super(MBInvertedConvLayer, self).__init__() self.in_channels = in_channels self.out_channels = out_channels self.kernel_size = kernel_size self.stride = stride self.expand_ratio = expand_ratio self.mid_channels = mid_channels self.act_func = act_func self.use_se = use_se if self.mid_channels is None: feature_dim = round(self.in_channels * self.expand_ratio) else: feature_dim = self.mid_channels if self.expand_ratio == 1: self.inverted_bottleneck = None else: self.inverted_bottleneck = nn.Sequential(OrderedDict([ ('conv', nn.Conv2d(self.in_channels, feature_dim, 1, 1, 0, bias=False)), ('bn', nn.BatchNorm2d(feature_dim)), ('act', build_activation(self.act_func, inplace=True)), ])) pad = get_same_padding(self.kernel_size) depth_conv_modules = [ ('conv', nn.Conv2d(feature_dim, feature_dim, kernel_size, stride, pad, groups=feature_dim, bias=False)), ('bn', nn.BatchNorm2d(feature_dim)), ('act', build_activation(self.act_func, inplace=True)) ] if self.use_se: depth_conv_modules.append(('se', SEModule(feature_dim))) self.depth_conv = nn.Sequential(OrderedDict(depth_conv_modules)) self.point_linear = nn.Sequential(OrderedDict([ ('conv', nn.Conv2d(feature_dim, out_channels, 1, 1, 0, bias=False)), ('bn', nn.BatchNorm2d(out_channels)), ]))
def __init__(self, in_features, out_features, bias=True, use_bn=False, act_func=None, dropout_rate=0, ops_order='weight_bn_act'): super(LinearLayer, self).__init__() self.in_features = in_features self.out_features = out_features self.bias = bias self.use_bn = use_bn self.act_func = act_func self.dropout_rate = dropout_rate self.ops_order = ops_order """ modules """ modules = {} # batch norm if self.use_bn: if self.bn_before_weight: modules['bn'] = nn.BatchNorm1d(in_features) else: modules['bn'] = nn.BatchNorm1d(out_features) else: modules['bn'] = None # activation modules['act'] = build_activation(self.act_func, self.ops_list[0] != 'act') # dropout if self.dropout_rate > 0: modules['dropout'] = nn.Dropout(self.dropout_rate, inplace=True) else: modules['dropout'] = None # linear modules['weight'] = { 'linear': nn.Linear(self.in_features, self.out_features, self.bias) } # add modules for op in self.ops_list: if modules[op] is None: continue elif op == 'weight': if modules['dropout'] is not None: self.add_module('dropout', modules['dropout']) for key in modules['weight']: self.add_module(key, modules['weight'][key]) else: self.add_module(op, modules[op])
def __init__(self, in_channels, out_channels, use_bn=True, act_func='relu', dropout_rate=0, ops_order='weight_bn_act'): super(My2DLayer, self).__init__() self.in_channels = in_channels self.out_channels = out_channels self.use_bn = use_bn self.act_func = act_func self.dropout_rate = dropout_rate self.ops_order = ops_order """ modules """ modules = {} # batch norm if self.use_bn: if self.bn_before_weight: modules['bn'] = nn.BatchNorm2d(in_channels) else: modules['bn'] = nn.BatchNorm2d(out_channels) else: modules['bn'] = None # activation modules['act'] = build_activation( self.act_func, self.ops_list[0] != 'act' and self.use_bn) # dropout if self.dropout_rate > 0: modules['dropout'] = nn.Dropout2d(self.dropout_rate, inplace=True) else: modules['dropout'] = None # weight modules['weight'] = self.weight_op() # add modules for op in self.ops_list: if modules[op] is None: continue elif op == 'weight': # dropout before weight operation if modules['dropout'] is not None: self.add_module('dropout', modules['dropout']) for key in modules['weight']: self.add_module(key, modules['weight'][key]) else: self.add_module(op, modules[op])
def __init__(self, main_branch, in_channels, out_channels, expand=1.0, kernel_size=3, act_func='relu', n_groups=2, downsample_ratio=2, upsample_type='bilinear', stride=1): super(LiteResidualModule, self).__init__() self.main_branch = main_branch self.lite_residual_config = { 'in_channels': in_channels, 'out_channels': out_channels, 'expand': expand, 'kernel_size': kernel_size, 'act_func': act_func, 'n_groups': n_groups, 'downsample_ratio': downsample_ratio, 'upsample_type': upsample_type, 'stride': stride, } kernel_size = 1 if downsample_ratio is None else kernel_size padding = get_same_padding(kernel_size) if downsample_ratio is None: pooling = MyGlobalAvgPool2d() else: pooling = nn.AvgPool2d(downsample_ratio, downsample_ratio, 0) num_mid = make_divisible(int(in_channels * expand), divisor=MyNetwork.CHANNEL_DIVISIBLE) self.lite_residual = nn.Sequential( OrderedDict({ 'pooling': pooling, 'conv1': nn.Conv2d(in_channels, num_mid, kernel_size, stride, padding, groups=n_groups, bias=False), 'bn1': nn.BatchNorm2d(num_mid), 'act': build_activation(act_func), 'conv2': nn.Conv2d(num_mid, out_channels, 1, 1, 0, bias=False), 'final_bn': nn.BatchNorm2d(out_channels), })) # initialize init_models(self.lite_residual) self.lite_residual.final_bn.weight.data.zero_()
def __init__(self, in_channels, out_channels, kernel_size=3, stride=1, expand_ratio=0.25, mid_channels=None, act_func='relu', groups=1, downsample_mode='avgpool_conv'): super(ResNetBottleneckBlock, self).__init__() self.in_channels = in_channels self.out_channels = out_channels self.kernel_size = kernel_size self.stride = stride self.expand_ratio = expand_ratio self.mid_channels = mid_channels self.act_func = act_func self.groups = groups self.downsample_mode = downsample_mode if self.mid_channels is None: feature_dim = round(self.out_channels * self.expand_ratio) else: feature_dim = self.mid_channels feature_dim = make_divisible(feature_dim, MyNetwork.CHANNEL_DIVISIBLE) self.mid_channels = feature_dim # build modules self.conv1 = nn.Sequential( OrderedDict([ ('conv', nn.Conv2d(self.in_channels, feature_dim, 1, 1, 0, bias=False)), ('bn', nn.BatchNorm2d(feature_dim)), ('act', build_activation(self.act_func, inplace=True)), ])) pad = get_same_padding(self.kernel_size) self.conv2 = nn.Sequential( OrderedDict([('conv', nn.Conv2d(feature_dim, feature_dim, kernel_size, stride, pad, groups=groups, bias=False)), ('bn', nn.BatchNorm2d(feature_dim)), ('act', build_activation(self.act_func, inplace=True))])) self.conv3 = nn.Sequential( OrderedDict([ ('conv', nn.Conv2d(feature_dim, self.out_channels, 1, 1, 0, bias=False)), ('bn', nn.BatchNorm2d(self.out_channels)), ])) if stride == 1 and in_channels == out_channels: self.downsample = IdentityLayer(in_channels, out_channels) elif self.downsample_mode == 'conv': self.downsample = nn.Sequential( OrderedDict([ ('conv', nn.Conv2d(in_channels, out_channels, 1, stride, 0, bias=False)), ('bn', nn.BatchNorm2d(out_channels)), ])) elif self.downsample_mode == 'avgpool_conv': self.downsample = nn.Sequential( OrderedDict([ ('avg_pool', nn.AvgPool2d(kernel_size=stride, stride=stride, padding=0, ceil_mode=True)), ('conv', nn.Conv2d(in_channels, out_channels, 1, 1, 0, bias=False)), ('bn', nn.BatchNorm2d(out_channels)), ])) else: raise NotImplementedError self.final_act = build_activation(self.act_func, inplace=True)