def __init__(self, config, img_size, in_channels=3): super(Embeddings, self).__init__() self.hybrid = None self.config = config img_size = _pair(img_size) if config.patches.get("grid") is not None: # ResNet grid_size = config.patches["grid"] patch_size = ( img_size[0] // 16 // grid_size[0], img_size[1] // 16 // grid_size[1], ) patch_size_real = (patch_size[0] * 16, patch_size[1] * 16) n_patches = (img_size[0] // patch_size_real[0]) * ( img_size[1] // patch_size_real[1]) self.hybrid = True else: patch_size = _pair(config.patches["size"]) n_patches = (img_size[0] // patch_size[0]) * (img_size[1] // patch_size[1]) self.hybrid = False if self.hybrid: self.hybrid_model = ResNetV2( block_units=config.resnet.num_layers, width_factor=config.resnet.width_factor, ) in_channels = self.hybrid_model.width * 16 self.patch_embeddings = Conv2d( in_channels=in_channels, out_channels=config.hidden_size, kernel_size=patch_size, stride=patch_size, ) # print(patch_size,"ooooo",img_size,grid_size, n_patches,"uuu", config.hidden_size) self.position_embeddings = nn.Parameter( torch.zeros(1, n_patches, config.hidden_size)) self.dropout = Dropout(config.transformer["dropout_rate"]) inplanes = 1024 interplanes = 1024 num_classes = 1 self.conva_gald = nn.Sequential( nn.Conv2d(inplanes, interplanes, 3, padding=1, bias=False), BatchNorm2d(interplanes), nn.ReLU(interplanes), ) self.a2block_gald = GALDBlock(interplanes, interplanes // 2) self.convb_gald = nn.Sequential( nn.Conv2d(interplanes, interplanes, 3, padding=1, bias=False), BatchNorm2d(interplanes), nn.ReLU(interplanes), ) self.bottleneck_gald = nn.Sequential( nn.Conv2d( inplanes + interplanes, interplanes, kernel_size=3, padding=1, dilation=1, bias=False, ), BatchNorm2d(interplanes), nn.ReLU(interplanes), nn.Conv2d(interplanes, num_classes, kernel_size=1, stride=1, padding=0, bias=True), )
def __init__(self): super(Encoder2, self).__init__() kernel_size = 3 stride = 2 padding = self.same_padding(kernel_size) # Inception 1 self.inc1_1 = Sequential( Conv2d(colors_dim, 4, kernel_size=1, stride=1), ReLU(), ) self.inc1_2 = Sequential( Conv2d(colors_dim, 4, kernel_size=kernel_size, stride=1, padding=padding), ReLU(), ) self.conv1 = Sequential( Conv2d(8, 8, kernel_size=kernel_size, stride=stride, padding=padding), ReLU(), ) # Inception 2 self.inc2_1 = Sequential( Conv2d(8, 8, kernel_size=1, stride=1), ReLU(), ) self.inc2_2 = Sequential( Conv2d(8, 8, kernel_size=kernel_size, stride=1, padding=padding), ReLU(), ) self.conv2 = Sequential( Conv2d(16, 16, kernel_size=kernel_size, stride=stride, padding=padding), BatchNorm2d(16), ReLU(), ) # Inception 3 self.inc3_1 = Sequential( Conv2d(16, 16, kernel_size=1, stride=1), ReLU(), ) self.inc3_2 = Sequential( Conv2d(16, 16, kernel_size=kernel_size, stride=1, padding=padding), ReLU(), ) self.conv3 = Sequential( Conv2d(32, 32, kernel_size=kernel_size, stride=stride, padding=padding), ReLU(), ) # Inception 4 self.inc4_1 = Sequential( Conv2d(32, 32, kernel_size=1, stride=1), ReLU(), ) self.inc4_2 = Sequential( Conv2d(32, 32, kernel_size=kernel_size, stride=1, padding=padding), ReLU(), ) self.conv4 = Sequential( Conv2d(64, 64, kernel_size=kernel_size, stride=stride, padding=padding), ReLU(), ) self.dense1 = Sequential( Flatten(), Linear(1024, 256), ReLU(), ) ## the following take the same input from dense1 self.dense_z_mu = Linear(256, parameter.latent_dim) if parameter.alpha: self.dense_z_std = Sequential( Linear(256, parameter.latent_dim), Softplus(), ) self.set_optimizer(parameter.optimizer, lr=parameter.learning_rate, betas=parameter.betas)
def transpose_convolution(c_in, c_out, k_size, stride=2, pad=1, bn=batch_norm): """A transposed convolution layer.""" layers = [ConvTranspose2d(c_in, c_out, k_size, stride, pad)] if bn: layers.append(BatchNorm2d(c_out)) return Sequential(*layers)
def test_conv_bn_relu(self, batch_size, input_channels_per_group, height, width, output_channels_per_group, groups, kernel_h, kernel_w, stride_h, stride_w, pad_h, pad_w, dilation, padding_mode, use_relu, eps, momentum, freeze_bn): input_channels = input_channels_per_group * groups output_channels = output_channels_per_group * groups dilation_h = dilation_w = dilation conv_op = Conv2d( input_channels, output_channels, (kernel_h, kernel_w), (stride_h, stride_w), (pad_h, pad_w), (dilation_h, dilation_w), groups, False, # No bias padding_mode).to(dtype=torch.double) bn_op = BatchNorm2d(output_channels, eps, momentum).to(dtype=torch.double) relu_op = ReLU() cls = ConvBnReLU2d if use_relu else ConvBn2d qat_op = cls( input_channels, output_channels, (kernel_h, kernel_w), (stride_h, stride_w), (pad_h, pad_w), (dilation_h, dilation_w), groups, None, # bias padding_mode, eps, momentum, freeze_bn=True, qconfig=default_qat_qconfig).to(dtype=torch.double) qat_op.apply(torch.ao.quantization.disable_fake_quant) if freeze_bn: qat_op.apply(torch.nn.intrinsic.qat.freeze_bn_stats) else: qat_op.apply(torch.nn.intrinsic.qat.update_bn_stats) # align inputs and internal parameters input = torch.randn(batch_size, input_channels, height, width, dtype=torch.double, requires_grad=True) conv_op.weight = torch.nn.Parameter(qat_op.weight.detach()) bn_op.running_mean = qat_op.bn.running_mean.clone() bn_op.running_var = qat_op.bn.running_var.clone() bn_op.weight = torch.nn.Parameter(qat_op.bn.weight.detach()) bn_op.bias = torch.nn.Parameter(qat_op.bn.bias.detach()) def compose(functions): # functions are reversed for natural reading order return reduce(lambda f, g: lambda x: f(g(x)), functions[::-1], lambda x: x) if not use_relu: def relu_op(x): return x if freeze_bn: def ref_op(x): x = conv_op(x) x = (x - bn_op.running_mean.reshape([1, -1, 1, 1])) * \ (bn_op.weight / torch.sqrt(bn_op.running_var + bn_op.eps)) \ .reshape([1, -1, 1, 1]) + bn_op.bias.reshape([1, -1, 1, 1]) x = relu_op(x) return x else: ref_op = compose([conv_op, bn_op, relu_op]) input_clone = input.clone().detach().requires_grad_() for i in range(2): result_ref = ref_op(input) result_actual = qat_op(input_clone) self.assertEqual(result_ref, result_actual) # backward dout = torch.randn(result_ref.size(), dtype=torch.double) loss = (result_ref - dout).sum() loss.backward() input_grad_ref = input.grad.cpu() weight_grad_ref = conv_op.weight.grad.cpu() gamma_grad_ref = bn_op.weight.grad.cpu() beta_grad_ref = bn_op.bias.grad.cpu() running_mean_ref = bn_op.running_mean running_var_ref = bn_op.running_var num_batches_tracked_ref = bn_op.num_batches_tracked loss = (result_actual - dout).sum() loss.backward() input_grad_actual = input_clone.grad.cpu() weight_grad_actual = qat_op.weight.grad.cpu() gamma_grad_actual = qat_op.bn.weight.grad.cpu() beta_grad_actual = qat_op.bn.bias.grad.cpu() running_mean_actual = qat_op.bn.running_mean running_var_actual = qat_op.bn.running_var num_batches_tracked_actual = qat_op.bn.num_batches_tracked precision = 1e-10 self.assertEqual(input_grad_ref, input_grad_actual, atol=precision, rtol=0) self.assertEqual(weight_grad_ref, weight_grad_actual, atol=precision, rtol=0) self.assertEqual(gamma_grad_ref, gamma_grad_actual, atol=precision, rtol=0) self.assertEqual(beta_grad_ref, beta_grad_actual, atol=precision, rtol=0) self.assertEqual(num_batches_tracked_ref, num_batches_tracked_actual, atol=precision, rtol=0) self.assertEqual(running_mean_ref, running_mean_actual, atol=precision, rtol=0) self.assertEqual(running_var_ref, running_var_actual, atol=precision, rtol=0)
def conv_1x1_bn(inp, oup): return nn.Sequential(Conv2d(inp, oup, 1, 1, 0, bias=False), BatchNorm2d(oup), nn.ReLU6(inplace=True))
def __init__(self, num_classes, device, is_test=False, config=None): super(VGGSSD, self).__init__() vgg_config = [ 64, 64, 'M', 128, 128, 'M', 256, 256, 256, 'C', 512, 512, 512, 'M', 512, 512, 512 ] self.num_classes = num_classes self.device = device self.base_net = ModuleList(self.vgg(vgg_config)) self.source_layer_indexes = [ (23, BatchNorm2d(512)), len(self.base_net), ] self.extras = ModuleList([ Sequential( Conv2d(in_channels=1024, out_channels=256, kernel_size=1), ReLU(), Conv2d(in_channels=256, out_channels=512, kernel_size=3, stride=2, padding=1), ReLU()), Sequential( Conv2d(in_channels=512, out_channels=128, kernel_size=1), ReLU(), Conv2d(in_channels=128, out_channels=256, kernel_size=3, stride=2, padding=1), ReLU()), Sequential( Conv2d(in_channels=256, out_channels=128, kernel_size=1), ReLU(), Conv2d(in_channels=128, out_channels=256, kernel_size=3), ReLU()), Sequential( Conv2d(in_channels=256, out_channels=128, kernel_size=1), ReLU(), Conv2d(in_channels=128, out_channels=256, kernel_size=3), ReLU()) ]) self.classification_headers = ModuleList([ Conv2d(in_channels=512, out_channels=4 * num_classes, kernel_size=3, padding=1), Conv2d(in_channels=1024, out_channels=6 * num_classes, kernel_size=3, padding=1), Conv2d(in_channels=512, out_channels=6 * num_classes, kernel_size=3, padding=1), Conv2d(in_channels=256, out_channels=6 * num_classes, kernel_size=3, padding=1), Conv2d(in_channels=256, out_channels=4 * num_classes, kernel_size=3, padding=1), Conv2d(in_channels=256, out_channels=4 * num_classes, kernel_size=1, padding=0), ]) self.regression_headers = ModuleList([ Conv2d(in_channels=512, out_channels=4 * 4, kernel_size=3, padding=1), Conv2d(in_channels=1024, out_channels=6 * 4, kernel_size=3, padding=1), Conv2d(in_channels=512, out_channels=6 * 4, kernel_size=3, padding=1), Conv2d(in_channels=256, out_channels=6 * 4, kernel_size=3, padding=1), Conv2d(in_channels=256, out_channels=4 * 4, kernel_size=3, padding=1), Conv2d(in_channels=256, out_channels=4 * 4, kernel_size=1, padding=0), ]) self.is_test = is_test self.config = config self.source_layer_add_ons = nn.ModuleList([ t[1] for t in self.source_layer_indexes if isinstance(t, tuple) and not isinstance(t, GraphPath) ]) self.priors = config.priors.to(self.device)
def __init__(self, device, num_nodes, dropout=0.3, supports=None, do_graph_conv=True, addaptadj=True, aptinit=None, in_dim=2, out_seq_len=12, residual_channels=32, dilation_channels=32, cat_feat_gc=False, skip_channels=256, end_channels=512, kernel_size=2, blocks=4, layers=2, stride=2, apt_size=10, verbose=0): super().__init__() self.dropout = dropout self.blocks = blocks self.layers = layers self.do_graph_conv = do_graph_conv self.cat_feat_gc = cat_feat_gc self.addaptadj = addaptadj self.verbose = verbose if self.cat_feat_gc: self.start_conv = nn.Conv2d( in_channels=1, # hard code to avoid errors out_channels=residual_channels, kernel_size=(1, 1)) self.cat_feature_conv = nn.Conv2d(in_channels=in_dim - 1, out_channels=residual_channels, kernel_size=(1, 1)) else: self.start_conv = nn.Conv2d(in_channels=in_dim, out_channels=residual_channels, kernel_size=(1, 1)) self.fixed_supports = supports or [] receptive_field = 1 self.supports_len = len(self.fixed_supports) if do_graph_conv and addaptadj: if aptinit is None: nodevecs = torch.randn(num_nodes, apt_size), torch.randn( apt_size, num_nodes) else: nodevecs = self.svd_init(apt_size, aptinit) self.supports_len += 1 self.nodevec1, self.nodevec2 = [ Parameter(n.to(device), requires_grad=True) for n in nodevecs ] depth = list(range(blocks * layers)) # 1x1 convolution for residual and skip connections (slightly different see docstring) self.residual_convs = ModuleList([ Conv1d(dilation_channels, residual_channels, (1, 1)) for _ in depth ]) self.skip_convs = ModuleList( [Conv1d(dilation_channels, skip_channels, (1, 1)) for _ in depth]) self.bn = ModuleList([BatchNorm2d(residual_channels) for _ in depth]) self.graph_convs = ModuleList([ GraphConvNet(dilation_channels, residual_channels, dropout, support_len=self.supports_len) for _ in depth ]) self.filter_convs = ModuleList() self.gate_convs = ModuleList() for b in range(blocks): additional_scope = kernel_size - 1 D = 1 # dilation for i in range(layers): # dilated convolutions self.filter_convs.append( Conv2d(residual_channels, dilation_channels, (1, kernel_size), dilation=D)) self.gate_convs.append( Conv1d(residual_channels, dilation_channels, (1, kernel_size), dilation=D)) D *= stride receptive_field += additional_scope additional_scope *= stride self.receptive_field = receptive_field self.end_conv_1 = Conv2d(skip_channels, end_channels, (1, 1), bias=True) self.end_conv_2 = Conv2d(end_channels, out_seq_len, (1, 1), bias=True)
def create_vgg_ssd(num_classes, is_test=False): vgg_config = [ 64, 64, 'M', 128, 128, 'M', 256, 256, 256, 'C', 512, 512, 512, 'M', 512, 512, 512 ] base_net = ModuleList(vgg(vgg_config)) source_layer_indexes = [ (23, BatchNorm2d(512)), len(base_net), ] extras = ModuleList([ Sequential( Conv2d(in_channels=1024, out_channels=256, kernel_size=1), ReLU(), Conv2d(in_channels=256, out_channels=512, kernel_size=3, stride=2, padding=1), ReLU()), Sequential( Conv2d(in_channels=512, out_channels=128, kernel_size=1), ReLU(), Conv2d(in_channels=128, out_channels=256, kernel_size=3, stride=2, padding=1), ReLU()), Sequential(Conv2d(in_channels=256, out_channels=128, kernel_size=1), ReLU(), Conv2d(in_channels=128, out_channels=256, kernel_size=3), ReLU()), Sequential(Conv2d(in_channels=256, out_channels=128, kernel_size=1), ReLU(), Conv2d(in_channels=128, out_channels=256, kernel_size=3), ReLU()) ]) regression_headers = ModuleList([ Conv2d(in_channels=512, out_channels=4 * 4, kernel_size=3, padding=1), Conv2d(in_channels=1024, out_channels=6 * 4, kernel_size=3, padding=1), Conv2d(in_channels=512, out_channels=6 * 4, kernel_size=3, padding=1), Conv2d(in_channels=256, out_channels=6 * 4, kernel_size=3, padding=1), Conv2d(in_channels=256, out_channels=4 * 4, kernel_size=3, padding=1), Conv2d(in_channels=256, out_channels=4 * 4, kernel_size=3, padding=1), # TODO: change to kernel_size=1, padding=0? ]) classification_headers = ModuleList([ Conv2d(in_channels=512, out_channels=4 * num_classes, kernel_size=3, padding=1), Conv2d(in_channels=1024, out_channels=6 * num_classes, kernel_size=3, padding=1), Conv2d(in_channels=512, out_channels=6 * num_classes, kernel_size=3, padding=1), Conv2d(in_channels=256, out_channels=6 * num_classes, kernel_size=3, padding=1), Conv2d(in_channels=256, out_channels=4 * num_classes, kernel_size=3, padding=1), Conv2d(in_channels=256, out_channels=4 * num_classes, kernel_size=3, padding=1), # TODO: change to kernel_size=1, padding=0? ]) return SSD(num_classes, base_net, source_layer_indexes, extras, classification_headers, regression_headers, is_test=is_test, config=config)
def __init__(self, config, num_classes=10): super(CNV, self).__init__() self.config = config weight_quant_type = get_quant_type(config.weight_bit_width) act_quant_type = get_quant_type(config.activation_bit_width) in_quant_type = get_quant_type(config.input_bit_width) stats_op = get_stats_op(weight_quant_type) self.preproc_features = ModuleList() self.conv_features = ModuleList() self.linear_features = ModuleList() if config.colortrans: self.preproc_features.append(ColorSpaceTransformation(3, 3)) if config.preproc_mode == 'trained_dithering': self.preproc_features.append( TrainedDithering(config.input_bit_width, 3, 3)) elif config.preproc_mode == 'fixed_dithering': self.preproc_features.append( FixedDithering(config.input_bit_width, 3)) elif config.preproc_mode == 'quant': self.preproc_features.append(Quantization(config.input_bit_width)) in_ch = 3 for i, out_ch, is_pool_enabled in CNV_OUT_CH_POOL: self.conv_features.append( get_quant_conv2d(in_ch=in_ch, out_ch=out_ch, bit_width=config.weight_bit_width, quant_type=weight_quant_type, stats_op=stats_op)) in_ch = out_ch if is_pool_enabled: self.conv_features.append(MaxPool2d(kernel_size=2)) if i == 5: self.conv_features.append(Sequential()) self.conv_features.append(BatchNorm2d(in_ch)) self.conv_features.append( get_act_quant(config.activation_bit_width, act_quant_type)) for in_features, out_features in INTERMEDIATE_FC_FEATURES: self.linear_features.append( get_quant_linear( in_features=in_features, out_features=out_features, per_out_ch_scaling=INTERMEDIATE_FC_PER_OUT_CH_SCALING, bit_width=config.weight_bit_width, quant_type=weight_quant_type, stats_op=stats_op)) self.linear_features.append(BatchNorm1d(out_features)) self.linear_features.append( get_act_quant(config.activation_bit_width, act_quant_type)) self.classifier = get_quant_linear( in_features=LAST_FC_IN_FEATURES, out_features=num_classes, per_out_ch_scaling=LAST_FC_PER_OUT_CH_SCALING, bit_width=config.weight_bit_width, quant_type=weight_quant_type, stats_op=stats_op)
def __init__(self, useIntraGCN=True, useInterGCN=True, useRandomMatrix=False, useAllOneMatrix=False, useCov=False, useCluster=False): super(Backbone_VGG, self).__init__() self.layer1 = Sequential(Conv2d(3, 64, kernel_size=3, padding=1), BatchNorm2d(64), ReLU(inplace=True), Conv2d(64, 64, kernel_size=3, padding=1), BatchNorm2d(64), ReLU(inplace=True), MaxPool2d(kernel_size=2, stride=2)) self.layer2 = Sequential(Conv2d(64, 128, kernel_size=3, padding=1), BatchNorm2d(128), ReLU(inplace=True), Conv2d(128, 128, kernel_size=3, padding=1), BatchNorm2d(128), ReLU(inplace=True), MaxPool2d(kernel_size=2, stride=2)) self.layer3 = Sequential(Conv2d(128, 256, kernel_size=3, padding=1), BatchNorm2d(256), ReLU(inplace=True), Conv2d(256, 256, kernel_size=3, padding=1), BatchNorm2d(256), ReLU(inplace=True), Conv2d(256, 256, kernel_size=3, padding=1), BatchNorm2d(256), ReLU(inplace=True), MaxPool2d(kernel_size=2, stride=2)) self.layer4 = Sequential(Conv2d(256, 512, kernel_size=3, padding=1), BatchNorm2d(512), ReLU(inplace=True), Conv2d(512, 512, kernel_size=3, padding=1), BatchNorm2d(512), ReLU(inplace=True), Conv2d(512, 512, kernel_size=3, padding=1), BatchNorm2d(512), ReLU(inplace=True), Conv2d(512, 512, kernel_size=3, padding=1), BatchNorm2d(512), ReLU(inplace=True), Conv2d(512, 512, kernel_size=3, padding=1), BatchNorm2d(512), ReLU(inplace=True), Conv2d(512, 512, kernel_size=3, padding=1), BatchNorm2d(512), ReLU(inplace=True), MaxPool2d(kernel_size=2, stride=2)) self.output_layer = Sequential( nn.Conv2d(in_channels=512, out_channels=64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1)), nn.ReLU(), nn.AdaptiveAvgPool2d((1, 1))) self.Crop_Net = nn.ModuleList([ Sequential( Conv2d(128, 256, kernel_size=3, padding=1), BatchNorm2d(256), ReLU(inplace=True), Conv2d(256, 256, kernel_size=3, padding=1), BatchNorm2d(256), ReLU(inplace=True), Conv2d(256, 512, kernel_size=3, padding=1), BatchNorm2d(512), ReLU(inplace=True), Conv2d(512, 512, kernel_size=3, padding=1), BatchNorm2d(512), ReLU(inplace=True), nn.Conv2d(in_channels=512, out_channels=64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1)), nn.ReLU()) for i in range(5) ]) self.fc = nn.Linear(64 + 320, 7) self.loc_fc = nn.Linear(320, 7) self.GAP = nn.AdaptiveAvgPool2d((1, 1)) #self.GCN = GCN(64, 128, 64) self.GCN = GCNwithIntraAndInterMatrix(64, 128, 64, useIntraGCN=useIntraGCN, useInterGCN=useInterGCN, useRandomMatrix=useRandomMatrix, useAllOneMatrix=useAllOneMatrix) self.SourceMean = ( CountMeanAndCovOfFeature(64 + 320) if useCov else CountMeanOfFeature(64 + 320) ) if not useCluster else CountMeanOfFeatureInCluster(64 + 320) self.TargetMean = ( CountMeanAndCovOfFeature(64 + 320) if useCov else CountMeanOfFeature(64 + 320) ) if not useCluster else CountMeanOfFeatureInCluster(64 + 320) self.SourceBN = BatchNorm1d(64 + 320) self.TargetBN = BatchNorm1d(64 + 320)
def __init__(self, config, data_feature): super().__init__(config, data_feature) self.device = config.get('device', torch.device('cpu')) self._scaler = self.data_feature.get('scaler') self.num_nodes = self.data_feature.get('num_nodes', 1) self.feature_dim = self.data_feature.get('feature_dim', 1) self.output_dim = self.data_feature.get('output_dim', 1) self.transmit = self.data_feature.get('transmit').to(self.device) self.adj_mx = self.data_feature.get('adj_mx') self.adj_mx_cluster = self.data_feature.get('adj_mx_cluster').to( self.device) self.centers_ind_groups = self.data_feature.get('centers_ind_groups') self._logger = getLogger() self.input_window = config.get('input_window', 12) self.output_window = config.get('output_window', 12) self.cluster_nodes = config.get('cluster_nodes', 1) self.dropout = config.get('dropout', 0) self.channels = config.get('channels', 32) self.skip_channels = config.get('skip_channels', 32) self.end_channels = config.get('end_channels', 512) self.supports = [torch.tensor(self.adj_mx)] self.supports_cluster = [self.adj_mx_cluster.clone().detach()] self.supports_len = torch.tensor(0, device=self.device) self.supports_len_cluster = torch.tensor(0, device=self.device) self.supports_len += len(self.supports) self.supports_len_cluster += len(self.supports_cluster) self.start_conv = nn.Conv2d(in_channels=self.feature_dim, out_channels=self.channels, kernel_size=(1, 1)) self.start_conv_cluster = nn.Conv2d(in_channels=self.feature_dim, out_channels=self.channels, kernel_size=(1, 1)) self.h = Parameter(torch.zeros(self.num_nodes, self.num_nodes), requires_grad=True) nn.init.uniform_(self.h, a=0, b=0.0001) self.h_cluster = Parameter(torch.zeros(self.cluster_nodes, self.cluster_nodes), requires_grad=True) nn.init.uniform_(self.h_cluster, a=0, b=0.0001) self.supports_len += 1 self.supports_len_cluster += 1 self.nodevec1 = nn.Parameter(torch.randn(self.num_nodes, 10), requires_grad=True) self.nodevec2 = nn.Parameter(torch.randn(10, self.num_nodes), requires_grad=True) self.nodevec1_c = nn.Parameter(torch.randn(self.cluster_nodes, 10), requires_grad=True) self.nodevec2_c = nn.Parameter(torch.randn(10, self.cluster_nodes), requires_grad=True) self.block1 = GCNPool(2 * self.channels, self.channels, self.num_nodes, self.input_window - 6, 3, self.dropout, self.num_nodes, self.supports_len) self.block2 = GCNPool(2 * self.channels, self.channels, self.num_nodes, self.input_window - 9, 2, self.dropout, self.num_nodes, self.supports_len) self.block_cluster1 = GCNPool(c_in=self.channels, c_out=self.channels, num_nodes=self.cluster_nodes, tem_size=self.input_window - 6, Kt=3, dropout=self.dropout, pool_nodes=self.cluster_nodes, support_len=self.supports_len) self.block_cluster2 = GCNPool(c_in=self.channels, c_out=self.channels, num_nodes=self.cluster_nodes, tem_size=self.input_window - 9, Kt=2, dropout=self.dropout, pool_nodes=self.cluster_nodes, support_len=self.supports_len) self.skip_conv1 = Conv2d(2 * self.channels, self.skip_channels, kernel_size=(1, 1), stride=(1, 1), bias=True) self.skip_conv2 = Conv2d(2 * self.channels, self.skip_channels, kernel_size=(1, 1), stride=(1, 1), bias=True) self.end_conv_1 = nn.Conv2d(in_channels=self.skip_channels, out_channels=self.end_channels, kernel_size=(1, 3), bias=True) self.end_conv_2 = nn.Conv2d(in_channels=self.end_channels, out_channels=self.output_window, kernel_size=(1, 1), bias=True) self.bn = BatchNorm2d(self.feature_dim, affine=False) self.bn_cluster = BatchNorm2d(self.feature_dim, affine=False) self.gate1 = gate(2 * self.channels) self.gate2 = gate(2 * self.channels) self.gate3 = gate(2 * self.channels) self.transmit1 = Transmit(self.channels, self.input_window, self.transmit, self.num_nodes, self.cluster_nodes) self.transmit2 = Transmit(self.channels, self.input_window - 6, self.transmit, self.num_nodes, self.cluster_nodes) self.transmit3 = Transmit(self.channels, self.input_window - 9, self.transmit, self.num_nodes, self.cluster_nodes) self.linear = nn.Linear(1, self.output_dim, bias=True)
def __init__(self): super(Backbone_VGG_onlyGlobal, self).__init__() self.layer1 = Sequential(Conv2d(3, 64, kernel_size=3, padding=1), BatchNorm2d(64), ReLU(inplace=True), Conv2d(64, 64, kernel_size=3, padding=1), BatchNorm2d(64), ReLU(inplace=True), MaxPool2d(kernel_size=2, stride=2)) self.layer2 = Sequential(Conv2d(64, 128, kernel_size=3, padding=1), BatchNorm2d(128), ReLU(inplace=True), Conv2d(128, 128, kernel_size=3, padding=1), BatchNorm2d(128), ReLU(inplace=True), MaxPool2d(kernel_size=2, stride=2)) self.layer3 = Sequential(Conv2d(128, 256, kernel_size=3, padding=1), BatchNorm2d(256), ReLU(inplace=True), Conv2d(256, 256, kernel_size=3, padding=1), BatchNorm2d(256), ReLU(inplace=True), Conv2d(256, 256, kernel_size=3, padding=1), BatchNorm2d(256), ReLU(inplace=True), MaxPool2d(kernel_size=2, stride=2)) self.layer4 = Sequential(Conv2d(256, 512, kernel_size=3, padding=1), BatchNorm2d(512), ReLU(inplace=True), Conv2d(512, 512, kernel_size=3, padding=1), BatchNorm2d(512), ReLU(inplace=True), Conv2d(512, 512, kernel_size=3, padding=1), BatchNorm2d(512), ReLU(inplace=True), Conv2d(512, 512, kernel_size=3, padding=1), BatchNorm2d(512), ReLU(inplace=True), Conv2d(512, 512, kernel_size=3, padding=1), BatchNorm2d(512), ReLU(inplace=True), Conv2d(512, 512, kernel_size=3, padding=1), BatchNorm2d(512), ReLU(inplace=True), MaxPool2d(kernel_size=2, stride=2)) self.GAP = nn.AdaptiveAvgPool2d((1, 1)) self.output_layer = Sequential( nn.Conv2d(in_channels=512, out_channels=64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1)), nn.ReLU(), nn.AdaptiveAvgPool2d((1, 1))) self.fc = nn.Linear(64, 7)
def __init__( self, block, layers, groups, reduction, dropout_p=0.2, inplanes=128, input_3x3=True, downsample_kernel_size=3, downsample_padding=1, num_classes=1000, ): """ Parameters ---------- block (nn.Module): Bottleneck class. - For SENet154: SEBottleneck - For SE-ResNet models: SEResNetBottleneck - For SE-ResNeXt models: SEResNeXtBottleneck layers (list of ints): Number of residual blocks for 4 layers of the network (layer1...layer4). groups (int): Number of groups for the 3x3 convolution in each bottleneck block. - For SENet154: 64 - For SE-ResNet models: 1 - For SE-ResNeXt models: 32 reduction (int): Reduction ratio for Squeeze-and-Excitation modules. - For all models: 16 dropout_p (float or None): Drop probability for the Dropout layer. If `None` the Dropout layer is not used. - For SENet154: 0.2 - For SE-ResNet models: None - For SE-ResNeXt models: None inplanes (int): Number of input channels for layer1. - For SENet154: 128 - For SE-ResNet models: 64 - For SE-ResNeXt models: 64 input_3x3 (bool): If `True`, use three 3x3 convolutions instead of a single 7x7 convolution in layer0. - For SENet154: True - For SE-ResNet models: False - For SE-ResNeXt models: False downsample_kernel_size (int): Kernel size for downsampling convolutions in layer2, layer3 and layer4. - For SENet154: 3 - For SE-ResNet models: 1 - For SE-ResNeXt models: 1 downsample_padding (int): Padding for downsampling convolutions in layer2, layer3 and layer4. - For SENet154: 1 - For SE-ResNet models: 0 - For SE-ResNeXt models: 0 num_classes (int): Number of outputs in `last_linear` layer. - For all models: 1000 """ super(SENet, self).__init__() self.inplanes = inplanes if input_3x3: layer0_modules = [ ('conv1', nn.Conv2d(3, 64, 3, stride=2, padding=1, bias=False)), ('bn1', BatchNorm2d(64)), ('relu1', nn.ReLU(inplace=True)), ('conv2', nn.Conv2d(64, 64, 3, stride=1, padding=1, bias=False)), ('bn2', BatchNorm2d(64)), ('relu2', nn.ReLU(inplace=True)), ('conv3', nn.Conv2d(64, inplanes, 3, stride=1, padding=1, bias=False)), ('bn3', BatchNorm2d(inplanes)), ('relu3', nn.ReLU(inplace=True)), ] else: layer0_modules = [ ( 'conv1', nn.Conv2d(3, inplanes, kernel_size=7, stride=2, padding=3, bias=False), ), ('bn1', BatchNorm2d(inplanes)), ('relu1', nn.ReLU(inplace=True)), ] # To preserve compatibility with Caffe weights `ceil_mode=True` # is used instead of `padding=1`. self.pool = nn.MaxPool2d(3, stride=2, ceil_mode=True) self.layer0 = nn.Sequential(OrderedDict(layer0_modules)) self.layer1 = self._make_layer( block, planes=64, blocks=layers[0], groups=groups, reduction=reduction, downsample_kernel_size=1, downsample_padding=0, ) self.layer2 = self._make_layer( block, planes=128, blocks=layers[1], stride=2, groups=groups, reduction=reduction, downsample_kernel_size=downsample_kernel_size, downsample_padding=downsample_padding, ) self.layer3 = self._make_layer( block, planes=256, blocks=layers[2], stride=2, groups=groups, reduction=reduction, downsample_kernel_size=downsample_kernel_size, downsample_padding=downsample_padding, ) self.layer4 = self._make_layer( block, planes=512, blocks=layers[3], stride=2, groups=groups, reduction=reduction, downsample_kernel_size=downsample_kernel_size, downsample_padding=downsample_padding, ) self.avg_pool = nn.AvgPool2d(7, stride=1) self.dropout = nn.Dropout(dropout_p) if dropout_p is not None else None self.last_linear = nn.Linear(512 * block.expansion, num_classes) self._initialize_weights()
def __init__(self, input_num_channels, output_size, batch_norm, use_pooling, pooling_method, conv1_kernel_width, conv1_num_kernels, conv1_stride, conv1_dropout, pool1_kernel_size, pool1_stride, conv2_kernel_size, conv2_num_kernels, conv2_stride, conv2_dropout, pool2_kernel_size, pool2_stride, fcs_hidden_size, fcs_num_hidden_layers, fcs_dropout): super(LeNet, self).__init__() # Instance attributes for use in self.forward() later. self.input_num_channels = input_num_channels self.batch_norm = batch_norm try: assert isinstance(output_size, int) except AssertionError: raise AssertionError( '{}: output_size should be a Python integer. Got {} of type {}' .format(__name__, output_size, type(output_size))) input_size = output_size / self.input_num_channels if input_size.is_integer(): input_size = int(input_size) else: raise ValueError( 'output_size / input_num_channels = {} / {} = {}'.format( output_size, input_num_channels, input_size)) # If not using pooling, set all pooling operations to 1 by 1. if use_pooling is False: # warnings.warn('lenet: not using pooling') pool1_kernel_size = 1 pool1_stride = 1 pool2_kernel_size = 1 pool2_stride = 1 # ##################################################################### # Conv1 # ##################################################################### conv1_input_width = 65 conv1_input_height = 2 conv1_input_depth = input_num_channels conv1_kernel_width = conv1_kernel_width conv1_kernel_height = 2 conv1_stride_width = conv1_stride conv1_stride_height = conv1_kernel_height conv1_pad_width = 0 conv1_pad_height = 1 conv1_output_height, conv1_output_width, conv1_output_depth = get_conv_output_dims( (conv1_input_height, conv1_input_width, conv1_input_depth), (conv1_pad_height, conv1_pad_width), (conv1_kernel_height, conv1_kernel_width), (conv1_stride_height, conv1_stride_width), conv1_num_kernels) # conv1_output_size = (conv1_num_kernels, (input_size - conv1_kernel_width) / conv1_stride + 1) conv1_output_size = (conv1_output_height, conv1_output_width, conv1_output_depth) if printing: print('lenet.__init__: conv1_output_size (W,H,D) =', conv1_output_size) # self.conv1 = nn.Conv1d(input_channel, conv1_num_kernels, conv1_kernel_width, stride=conv1_stride) # NOTE: THIS IS CORRECT!!!! CONV doesn't depend on num_features! # pytorch.conv2d accepts shape (Batch (assumed), Channel, Height, Width) - ((Batch, 1, 2, 65)) # https://pytorch.org/docs/stable/nn.html#torch.nn.Conv2d self.conv1 = Conv2d(input_num_channels, conv1_num_kernels, (conv1_kernel_height, conv1_kernel_width), stride=(conv1_stride_height, conv1_stride_width), padding=(conv1_pad_height, conv1_pad_width)) nn.init.kaiming_normal_(self.conv1.weight.data) self.conv1.bias.data.fill_(0) self.conv1_drop = Dropout2d(p=conv1_dropout) if batch_norm is True: self.batch_norm1 = BatchNorm2d(conv1_num_kernels) # ##################################################################### # ##################################################################### # Pool1 # ##################################################################### pool1_input_height = conv1_output_height pool1_input_width = conv1_output_width pool1_input_depth = conv1_output_depth pool1_kernel_height = 1 pool1_kernel_width = pool1_kernel_size pool1_stride_width = pool1_stride pool1_stride_height = pool1_kernel_height pool1_output_height, pool1_output_width, pool1_output_depth = get_pool_output_dims( (pool1_input_height, pool1_input_width, pool1_input_depth), (pool1_kernel_height, pool1_kernel_width), (pool1_stride_height, pool1_stride_width)) pool1_output_size = (pool1_output_height, pool1_output_width, pool1_output_depth) if printing: print('lenet.__init__: pool1_output_size (W,H,D) =', pool1_output_size) # self.pool1 = nn.MaxPool1d(pool1_kernel_size, stride=pool1_stride) # stride=pool1_kernel_size by default self.pool1 = MaxPool2d( pool1_kernel_size, stride=pool1_stride) # stride=pool1_kernel_size by default ####################################################################### # ##################################################################### # Conv2 # ##################################################################### conv2_input_width = pool1_output_width conv2_input_height = pool1_output_height conv2_input_depth = pool1_output_depth conv2_kernel_width = conv2_kernel_size conv2_kernel_height = 2 conv2_stride_width = conv2_stride conv2_stride_height = conv2_kernel_height conv2_pad_width = 0 conv2_pad_height = 0 conv2_output_height, conv2_output_width, conv2_output_depth = get_conv_output_dims( (conv2_input_height, conv2_input_width, conv2_input_depth), (conv2_pad_height, conv2_pad_width), (conv2_kernel_height, conv2_kernel_width), (conv2_stride_height, conv2_stride_width), conv2_num_kernels) conv2_output_size = (conv2_output_height, conv2_output_width, conv2_output_depth) if printing: print('lenet.__init__: conv2_output_size (W,H,D) =', conv2_output_size) # self.conv2 = nn.Conv1d(conv1_num_kernels, conv2_num_kernels, (2, conv2_kernel_size), stride=conv2_stride) # NOTE: THIS IS CORRECT!!!! CONV doesn't depend on num_features! self.conv2 = Conv2d(conv1_num_kernels, conv2_num_kernels, (conv2_kernel_height, conv2_kernel_width), stride=(conv2_stride_height, conv2_stride_width), padding=(conv2_pad_height, conv2_pad_width)) nn.init.kaiming_normal_(self.conv2.weight.data) self.conv2.bias.data.fill_(0) self.conv2_drop = Dropout2d(p=conv2_dropout) if batch_norm is True: self.batch_norm2 = BatchNorm2d(conv2_num_kernels) # ##################################################################### # ##################################################################### # Pool2 # ##################################################################### pool2_input_width = conv2_output_width pool2_input_height = conv2_output_height pool2_input_depth = conv2_output_depth pool2_kernel_width = pool2_kernel_size pool2_kernel_height = 1 pool2_stride_width = pool2_stride pool2_stride_height = pool2_kernel_height pool2_output_height, pool2_output_width, pool2_output_depth = get_pool_output_dims( (pool2_input_height, pool2_input_width, pool2_input_depth), (pool2_kernel_height, pool2_kernel_width), (pool2_stride_height, pool2_stride_width)) pool2_output_size = (pool2_output_height, pool2_output_width, pool2_output_depth) if printing: print('lenet.__init__: pool2_output_size (W,H,D) =', pool2_output_size) self.pool2 = MaxPool2d( pool2_kernel_size, stride=pool2_stride) # stride=pool1_kernel_size by default ####################################################################### # # # Pool2 # pool2_output_size = (conv2_num_kernels, # (conv2_output_size[1] - pool2_kernel_size) / pool2_stride + 1) # # # self.pool2 = nn.MaxPool1d(pool2_kernel_size, stride=pool2_stride) # stride=pool1_kernel_size by default # self.pool2 = MaxPool2d(pool2_kernel_size, stride=pool2_stride) # stride=pool1_kernel_size by default # FCs # fcs_input_size = pool2_output_size[0] * pool2_output_size[1] fcs_input_size = pool2_output_height * pool2_output_width * pool2_output_depth self.fcs = FullyConnectedNet(fcs_input_size, output_size, fcs_dropout, batch_norm, fcs_hidden_size, fcs_num_hidden_layers)
def __init__( self, in_features, image_size, kernel_size=[2, 3, 2], stride=[2, 2, 1], in_channel_size=[6, 9, 6], activation_function="Tanh", ffnn_layer_size=[100, 1000], dropout=None, dropout2d=None, ): super().__init__() ActFunc = getattr(torch.nn.modules.activation, activation_function) # TRANSPOSED CONVOLUTIONAL LAYERS ------------------------------------------------------ convs_reversed = [] out_channels, out_height, out_width = image_size parameters = list(zip(in_channel_size, kernel_size, stride)) for in_channel_size_, kernel_size_, stride_ in parameters[::-1]: if len(convs_reversed) > 0: convs_reversed.append(ActFunc()) if dropout2d is not None: convs_reversed.append(Dropout2d(dropout2d)) convs_reversed.append(BatchNorm2d(out_channels)) M_height = out_height - kernel_size_ + stride_ M_width = out_width - kernel_size_ + stride_ out_padding_height = M_height % stride_ out_padding_width = M_width % stride_ in_height = (M_height - out_padding_height) // stride_ in_width = (M_width - out_padding_width) // stride_ conv_layer = ConvTranspose2d( in_channels=in_channel_size_, out_channels=out_channels, kernel_size=kernel_size_, stride=stride_, padding=0, output_padding=(out_padding_height, out_padding_width), ) convs_reversed.append(conv_layer) out_channels = in_channel_size_ out_height = in_height out_width = in_width convs = convs_reversed[::-1] self.CNN = Sequential(*convs) self._conv_in_shape = (in_channel_size_, out_height, out_width) # FEED FORWARD LAYERS ----------------------------------------------------------------- if ffnn_layer_size is None: ffnn_layer_size = [100, 100] self._ffnn_layer_size = ffnn_layer_size layers = [] in_size = in_features for out_size in ffnn_layer_size: layers.append(Linear(in_size, out_size)) layers.append(ActFunc()) if dropout is not None: layers.append(Dropout(dropout)) in_size = out_size layers.append(Linear(in_size, np.prod(self._conv_in_shape))) self.ffnn = Sequential(*layers)
def norm_layer(features): base = BatchNorm2d(features) base.register_buffer('num_batches_tracked', torch.tensor(0, dtype=torch.float64)) return base
def __init__( self, image_size, out_features, kernel_size=[3, 5, 3], stride=[1, 2, 2], out_channel_size=[6, 9, 6], padding_size=[1, 2, 1], activation_function="Tanh", ffnn_layer_size=[100, 1000], dropout=None, dropout2d=None, maxpool_kernel=2, maxpool_stride=2, ): super().__init__() self.dropout = dropout self.dropout2d = dropout2d self._activation_function = activation_function ActFunc = getattr(torch.nn.modules.activation, activation_function) # CONVOLUTIONAL LAYERS ------------------------------------------------------------------ convs = [] channels, height, width = image_size for parameters in list( zip(out_channel_size, kernel_size, stride, padding_size)): convs.append( Conv2d( in_channels=channels, out_channels=parameters[0], kernel_size=parameters[1], stride=parameters[2], padding=parameters[3], )) convs.append(BatchNorm2d(parameters[0])) if dropout2d is not None: convs.append(Dropout2d(dropout2d)) convs.append(ActFunc()) height = compute_conv_dim(height, parameters[1], parameters[3], parameters[2]) width = compute_conv_dim(width, parameters[1], parameters[3], parameters[2]) channels = parameters[0] convs.append( MaxPool2d(kernel_size=maxpool_kernel, stride=maxpool_stride)) height = compute_conv_dim(height, maxpool_kernel, 0, maxpool_stride) width = compute_conv_dim(width, maxpool_kernel, 0, maxpool_stride) self.CNN = Sequential(*convs) self.linear_in_features = channels * height * width self.last_im_size = (channels, height, width) # FEED FORWARD LAYERS ----------------------------------------------------------------- if ffnn_layer_size is None: ffnn_layer_size = [100, 100] self._ffnn_layer_size = ffnn_layer_size layers = [] in_size = self.linear_in_features for out_size in ffnn_layer_size: layers.append(Linear(in_size, out_size)) layers.append(ActFunc()) if dropout is not None: layers.append(Dropout(dropout)) in_size = out_size layers.append(Linear(in_size, out_features)) self.ffnn = Sequential(*layers)
def __init__(self, *args, **kwargs): super().__init__() self.bn = BatchNorm2d(*args, **kwargs)
def __init__(self, syn_type='inter', classification=False, classes=None): super().__init__() self.syn_type = syn_type self.input_mean = [0.5 * 255, 0.5 * 255, 0.5 * 255] self.input_std = [0.5 * 255, 0.5 * 255, 0.5 * 255] self.classification = classification self.classes = classes # self.syn_type = config.syn_type # bn_param = config.bn_param self.relu = nn.ReLU(inplace=True) self.pool = nn.MaxPool2d(kernel_size=2, stride=2) self.conv1 = nn.Conv2d(6, 64, kernel_size=5, stride=1, padding=2, bias=False) self.conv1_bn = BatchNorm2d(64) self.conv2 = nn.Conv2d(64, 128, kernel_size=5, stride=1, padding=2, bias=False) self.conv2_bn = BatchNorm2d(128) self.conv3 = nn.Conv2d(128, 256, kernel_size=3, stride=1, padding=1, bias=False) self.conv3_bn = BatchNorm2d(256) self.bottleneck = nn.Conv2d(256, 256, kernel_size=3, stride=1, padding=1, bias=False) self.bottleneck_bn = BatchNorm2d(256) self.deconv1 = nn.Conv2d(512, 256, kernel_size=3, stride=1, padding=1, bias=False) self.deconv1_bn = BatchNorm2d(256) self.deconv2 = nn.Conv2d(384, 128, kernel_size=5, stride=1, padding=2, bias=False) self.deconv2_bn = BatchNorm2d(128) self.deconv3 = nn.Conv2d(192, 64, kernel_size=5, stride=1, padding=2, bias=False) self.deconv3_bn = BatchNorm2d(64) self.conv4 = nn.Conv2d(64, 3, kernel_size=5, stride=1, padding=2) if (classification): self.avg_pool = nn.AdaptiveAvgPool2d((1, 1)) self.linear = nn.Linear(256, classes) for m in self.modules(): if isinstance(m, nn.Conv2d): m.weight.data.normal_(0, 0.01) if m.bias is not None: m.bias.data.zero_() elif isinstance(m, BatchNorm2d): m.weight.data.fill_(1) m.bias.data.zero_()
def Squeeze_block(c_in, c_out): return Sequential( Conv2d(c_in, c_out, kernel_size=(1, 1), stride=(1, 1), bias=False), BatchNorm2d(c_out, eps=1e-05, momentum=0.1, affine=True), LeakyReLU(0.1))
def __init__(self, input_size, block, layers, radix=1, groups=1, bottleneck_width=64, dilated=False, dilation=1, deep_stem=False, stem_width=64, avg_down=False, rectified_conv=False, rectify_avg=False, avd=False, avd_first=False, final_drop=0.0, dropblock_prob=0, last_gamma=False, norm_layer=nn.BatchNorm2d): self.cardinality = groups self.bottleneck_width = bottleneck_width # ResNet-D params self.inplanes = stem_width * 2 if deep_stem else 64 self.avg_down = avg_down self.last_gamma = last_gamma # ResNeSt params self.radix = radix self.avd = avd self.avd_first = avd_first super(ResNet, self).__init__() self.rectified_conv = rectified_conv self.rectify_avg = rectify_avg if rectified_conv: from rfconv import RFConv2d conv_layer = RFConv2d else: conv_layer = nn.Conv2d conv_kwargs = {'average_mode': rectify_avg} if rectified_conv else {} if deep_stem: self.conv1 = nn.Sequential( conv_layer(3, stem_width, kernel_size=3, stride=2, padding=1, bias=False, **conv_kwargs), norm_layer(stem_width), nn.PReLU(stem_width), conv_layer(stem_width, stem_width, kernel_size=3, stride=1, padding=1, bias=False, **conv_kwargs), norm_layer(stem_width), nn.PReLU(stem_width), conv_layer(stem_width, stem_width * 2, kernel_size=3, stride=1, padding=1, bias=False, **conv_kwargs), ) else: self.conv1 = conv_layer(3, 64, kernel_size=7, stride=2, padding=3, bias=False, **conv_kwargs) self.bn1 = norm_layer(self.inplanes) self.relu = nn.PReLU(self.inplanes) #self.maxpool = nn.MaxPool2d(kernel_size=3, stride=2, padding=1) self.layer1 = self._make_layer(block, 64, layers[0], norm_layer=norm_layer, is_first=False) self.layer2 = self._make_layer(block, 128, layers[1], stride=2, norm_layer=norm_layer) if dilated or dilation == 4: self.layer3 = self._make_layer(block, 256, layers[2], stride=1, dilation=2, norm_layer=norm_layer, dropblock_prob=dropblock_prob) self.layer4 = self._make_layer(block, 512, layers[3], stride=1, dilation=4, norm_layer=norm_layer, dropblock_prob=dropblock_prob) elif dilation == 2: self.layer3 = self._make_layer(block, 256, layers[2], stride=2, dilation=1, norm_layer=norm_layer, dropblock_prob=dropblock_prob) self.layer4 = self._make_layer(block, 512, layers[3], stride=1, dilation=2, norm_layer=norm_layer, dropblock_prob=dropblock_prob) else: self.layer3 = self._make_layer(block, 256, layers[2], stride=2, norm_layer=norm_layer, dropblock_prob=dropblock_prob) self.layer4 = self._make_layer(block, 512, layers[3], stride=2, norm_layer=norm_layer, dropblock_prob=dropblock_prob) #self.avgpool = GlobalAvgPool2d() #self.drop = nn.Dropout(final_drop) if final_drop > 0.0 else None #self.fc = nn.Linear(512 * block.expansion, num_classes) self.fc = Sequential(BatchNorm2d(512 * block.expansion), Dropout(0.4), Flatten(), Linear(512 * block.expansion * 7 * 7, 512), BatchNorm1d(512, affine=False)) for m in self.modules(): if isinstance(m, nn.Conv2d): n = m.kernel_size[0] * m.kernel_size[1] * m.out_channels m.weight.data.normal_(0, math.sqrt(2. / n)) elif isinstance(m, norm_layer): m.weight.data.fill_(1) m.bias.data.zero_() elif isinstance(m, nn.Linear): nn.init.kaiming_normal_(m.weight, mode='fan_out', nonlinearity='relu') if m.bias is not None: m.bias.data.zero_()
def __init__(self, numOfLayer): super(Backbone, self).__init__() unit_module = bottleneck_IR self.input_layer = Sequential( Conv2d(in_channels=3, out_channels=64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False), BatchNorm2d(64), PReLU(64)) blocks = get_blocks(numOfLayer) self.layer1 = Sequential(*[ unit_module(bottleneck.in_channel, bottleneck.depth, bottleneck.stride) for bottleneck in blocks[0] ]) #get_block(in_channel=64, depth=64, num_units=3)]) self.layer2 = Sequential(*[ unit_module(bottleneck.in_channel, bottleneck.depth, bottleneck.stride) for bottleneck in blocks[1] ]) #get_block(in_channel=64, depth=128, num_units=4)]) self.layer3 = Sequential(*[ unit_module(bottleneck.in_channel, bottleneck.depth, bottleneck.stride) for bottleneck in blocks[2] ]) #get_block(in_channel=128, depth=256, num_units=14)]) self.layer4 = Sequential(*[ unit_module(bottleneck.in_channel, bottleneck.depth, bottleneck.stride) for bottleneck in blocks[3] ]) #get_block(in_channel=256, depth=512, num_units=3)]) self.output_layer = Sequential( nn.Conv2d(in_channels=512, out_channels=64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1)), nn.ReLU(), nn.AdaptiveAvgPool2d((1, 1))) cropNet_modules = [] cropNet_blocks = [ get_block(in_channel=128, depth=256, num_units=2), get_block(in_channel=256, depth=512, num_units=2) ] for block in cropNet_blocks: for bottleneck in block: cropNet_modules.append( unit_module(bottleneck.in_channel, bottleneck.depth, bottleneck.stride)) cropNet_modules += [ nn.Conv2d(in_channels=512, out_channels=64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1)), nn.ReLU() ] self.Crop_Net = nn.ModuleList( [copy.deepcopy(nn.Sequential(*cropNet_modules)) for i in range(5)]) self.IC_Channel = nn.Linear(64 + 320, 100) self.ID_Channel = nn.Linear(64 + 320, 100) self.IC_fc = nn.Linear(100 + 100, 1) self.ID_fc = nn.Linear(100, 7) self.fusion_fc = Sequential(nn.Linear(100, 7), nn.Dropout(0.6), nn.Linear(7, 7), nn.Dropout(0.6), nn.Linear(7, 7)) self.GAP = nn.AdaptiveAvgPool2d((1, 1))
def conv_bn(inp, oup, stride): return nn.Sequential(Conv2d(inp, oup, 3, stride, 1, bias=False), BatchNorm2d(oup), nn.ReLU6(inplace=True))
def __init__(self, in_c): super(GNAP, self).__init__() self.bn1 = BatchNorm2d(in_c, affine=False) self.pool = nn.AdaptiveAvgPool2d((1, 1)) self.bn2 = BatchNorm1d(in_c, affine=False)
def __init__(self, in_c, out_c, kernel=(1, 1), stride=(1, 1), padding=(0, 0), groups=1): super(Linear_block, self).__init__() self.conv = Conv2d(in_c, out_channels=out_c, kernel_size=kernel, groups=groups, stride=stride, padding=padding, bias=False) self.bn = BatchNorm2d(out_c)
def _make_stage(self, features, out_features, size): prior = nn.AdaptiveAvgPool2d(output_size=(size, size)) conv = nn.Conv2d(features, out_features, kernel_size=1, bias=False) bn = BatchNorm2d(out_features) return nn.Sequential(prior, conv, bn)
def __init__(self, config, data_feature): super().__init__(config, data_feature) print("========batch_g_gat_2l===========") heads = config.get('heads', 8) feat_drop = config.get('feat_drop', 0.6) attn_drop = config.get('attn_drop', 0.6) negative_slope = config.get('negative_slope', 0.2) receptive_field = config.get('receptive_field', 1) out_dim = data_feature.get('out', 12) residual_channels = config.get('residual_channels', 40) dilation_channels = config.get('dilation_channels', 40) skip_channels = config.get('skip_channels', 320) end_channels = config.get('end_channels', 640) kernel_size = config.get('kernel_size', 2) blocks = config.get('blocks', 4) layers = config.get('layers', 2) self.g = self.data_feature.get('adj_mx', 1) self.g = nx.from_numpy_array(self.g) self.g = self.g.to_directed() self.g = dgl.from_networkx(self.g, node_attrs=[], edge_attrs=['weight']) self.dropout = config.get('dropout', 0.3) self.blocks = blocks self.layers = layers self.run_gconv = config.get('run_gconv', 1) self._scaler = self.data_feature.get('scaler') self._logger = getLogger() self.device = config.get('device', torch.device('cpu')) self.g = self.g.to(self.device) self.output_dim = self.data_feature.get('output_dim', 1) self.start_conv = nn.Conv2d( in_channels=1, # hard code to avoid errors out_channels=residual_channels, kernel_size=(1, 1)) self.cat_feature_conv = nn.Conv2d(in_channels=1, out_channels=residual_channels, kernel_size=(1, 1)) depth = list(range(blocks * layers)) # 1x1 convolution for residual and skip connections (slightly different see docstring) self.residual_convs = ModuleList([ Conv1d(dilation_channels, residual_channels, (1, 1)) for _ in depth ]) self.skip_convs = ModuleList( [Conv1d(dilation_channels, skip_channels, (1, 1)) for _ in depth]) self.bn = ModuleList([BatchNorm2d(residual_channels) for _ in depth]) self.gat_layers = nn.ModuleList() self.gat_layers1 = nn.ModuleList() self.filter_convs = ModuleList() self.gate_convs = ModuleList() # time_len change: 12/10/9/7/6/4/3/1 for b in range(blocks): additional_scope = kernel_size - 1 D = 1 # dilation for i in range(layers): # dilated convolutions self.filter_convs.append( Conv2d(residual_channels, dilation_channels, (1, kernel_size), dilation=D)) self.gate_convs.append( Conv1d(residual_channels, dilation_channels, (1, kernel_size), dilation=D)) # batch, channel, height, width # N,C,H,W # d = (d - kennel_size + 2 * padding) / stride + 1 # H_out = [H_in + 2*padding[0] - dilation[0]*(kernal_size[0]-1)-1]/stride[0] + 1 # W_out = [W_in + 2*padding[1] - dilation[1]*(kernal_size[1]-1)-1]/stride[1] + 1 D *= 2 receptive_field += additional_scope additional_scope *= 2 self.gat_layers.append( GATConv(dilation_channels * (14 - receptive_field), dilation_channels * (14 - receptive_field), heads, feat_drop, attn_drop, negative_slope, residual=False, activation=F.elu)) self.gat_layers1.append( GATConv(dilation_channels * (14 - receptive_field), dilation_channels * (14 - receptive_field), heads, feat_drop, attn_drop, negative_slope, residual=False, activation=F.elu)) self.receptive_field = receptive_field self.end_conv_1 = Conv2d(skip_channels, end_channels, (1, 1), bias=True) self.end_conv_2 = Conv2d(end_channels, out_dim, (1, 1), bias=True)
def __init__(self, config, data_feature): """ 构造模型 :param config: 源于各种配置的配置字典 :param data_feature: 从数据集Dataset类的`get_data_feature()`接口返回的必要的数据相关的特征 """ # 1.初始化父类(必须) super().__init__(config, data_feature) # 2.初始化device(必须) self.device = config.get('device', torch.device('cpu')) # 3.从data_feature获取想要的信息,注意不同模型使用不同的Dataset类,其返回的data_feature内容不同(必须) self._scaler = self.data_feature.get('scaler') # 用于数据归一化 self.num_nodes = self.data_feature.get('num_nodes') # 节点个数 self.feature_dim = self.data_feature.get('feature_dim') # 输入维度 self.output_dim = self.data_feature.get('output_dim') # 输出维度 self.transmit = self.data_feature.get('transmit').to( self.device) #trans矩阵 self.adj_mx = self.data_feature.get('adj_mx') # 邻接矩阵 self.adj_mx_cluster = self.data_feature.get('adj_mx_cluster').to( self.device) # region邻接矩阵 self.centers_ind_groups = self.data_feature.get( 'centers_ind_groups') #聚类后 区域标号[节点标号] # 4.初始化log用于必要的输出(必须) self._logger = getLogger() # 5.初始化输入输出时间步的长度(非必须) self.input_window = config.get('input_window') self.output_window = config.get('output_window') # 6.从config中取用到的其他参数,主要是用于构造模型结构的参数(必须) self.cluster_nodes = config['cluster_nodes'] #聚类节点(区域)个数 self.dropout = config['dropout'] self.length = config['length'] self.in_dim = config['in_dim'] self.in_dim_cluster = config['in_dim_cluster'] self.out_dim = config['out_dim'] self.residual_channels = config['residual_channels'] self.dilation_channels = config['dilation_channels'] self.skip_channels = config['skip_channels'] self.end_channels = config['end_channels'] self.kernel_size = config['kernel_size'] self.K = config['K'] self.Kt = config['Kt'] # 7.构造深度模型的层次结构(必须) self.supports = [torch.tensor(self.adj_mx)] self.supports_cluster = [torch.tensor(self.adj_mx_cluster)] self.supports_len = torch.tensor(0, device=self.device) self.supports_len_cluster = torch.tensor(0, device=self.device) if self.supports is not None: self.supports_len += len(self.supports) self.supports_len_cluster += len(self.supports_cluster) if self.supports is None: self.supports = [] self.supports_cluster = [] self.start_conv = nn.Conv2d(in_channels=self.in_dim, out_channels=self.residual_channels, kernel_size=(1, 1)) self.start_conv_cluster = nn.Conv2d( in_channels=self.in_dim_cluster, out_channels=self.residual_channels, kernel_size=(1, 1)) self.h = Parameter(torch.zeros(self.num_nodes, self.num_nodes), requires_grad=True) nn.init.uniform_(self.h, a=0, b=0.0001) self.h_cluster = Parameter(torch.zeros(self.cluster_nodes, self.cluster_nodes), requires_grad=True) nn.init.uniform_(self.h_cluster, a=0, b=0.0001) self.supports_len += 1 self.supports_len_cluster += 1 self.nodevec1 = nn.Parameter(torch.randn(self.num_nodes, 10), requires_grad=True) self.nodevec2 = nn.Parameter(torch.randn(10, self.num_nodes), requires_grad=True) self.nodevec1_c = nn.Parameter(torch.randn(self.cluster_nodes, 10), requires_grad=True) self.nodevec2_c = nn.Parameter(torch.randn(10, self.cluster_nodes), requires_grad=True) self.block1 = GCNPool(2 * self.dilation_channels, self.dilation_channels, self.num_nodes, self.length - 6, 3, self.dropout, self.num_nodes, self.supports_len) self.block2 = GCNPool(2 * self.dilation_channels, self.dilation_channels, self.num_nodes, self.length - 9, 2, self.dropout, self.num_nodes, self.supports_len) self.block_cluster1 = GCNPool(self.dilation_channels, self.dilation_channels, self.cluster_nodes, self.length - 6, 3, self.dropout, self.cluster_nodes, self.supports_len) self.block_cluster2 = GCNPool(self.dilation_channels, self.dilation_channels, self.cluster_nodes, self.length - 9, 2, self.dropout, self.cluster_nodes, self.supports_len) self.skip_conv1 = Conv2d(2 * self.dilation_channels, self.skip_channels, kernel_size=(1, 1), stride=(1, 1), bias=True) self.skip_conv2 = Conv2d(2 * self.dilation_channels, self.skip_channels, kernel_size=(1, 1), stride=(1, 1), bias=True) self.end_conv_1 = nn.Conv2d(in_channels=self.skip_channels, out_channels=self.end_channels, kernel_size=(1, 3), bias=True) self.end_conv_2 = nn.Conv2d(in_channels=self.end_channels, out_channels=self.out_dim, kernel_size=(1, 1), bias=True) self.bn = BatchNorm2d(self.in_dim, affine=False) self.conv_cluster1 = Conv2d(self.dilation_channels, self.out_dim, kernel_size=(1, 3), stride=(1, 1), bias=True) self.bn_cluster = BatchNorm2d(self.in_dim_cluster, affine=False) self.gate1 = gate(2 * self.dilation_channels) self.gate2 = gate(2 * self.dilation_channels) self.gate3 = gate(2 * self.dilation_channels) self.transmit1 = Transmit(self.dilation_channels, self.length, self.transmit, self.num_nodes, self.cluster_nodes) self.transmit2 = Transmit(self.dilation_channels, self.length - 6, self.transmit, self.num_nodes, self.cluster_nodes) self.transmit3 = Transmit(self.dilation_channels, self.length - 9, self.transmit, self.num_nodes, self.cluster_nodes)
def convolution(c_in, c_out, k_size, stride=2, pad=1, bn=batch_norm): """A convolutional layer.""" layers = [Conv2d(c_in, c_out, k_size, stride, pad)] if bn: layers.append(BatchNorm2d(c_out)) return Sequential(*layers)
def __init__(self, config): super().__init__() self.dataset = config['dataset'] inChan = 1 if self.dataset == "mnist" else 3 outChan = 100 if self.dataset == "cifar100" else 10 epsilon = 1e-4 # Some epsilon alpha = 1 - 0.9 # Exponential moving average factor for BN. self.conv1 = Conv2dBNN(inChan, 128, (3, 3), padding=1, H=1, W_LR_scale="Glorot") self.bn1 = BatchNorm2d(128, epsilon, alpha) self.tanh1 = SignBNN() self.conv2 = Conv2dBNN(128, 128, (3, 3), padding=1, H=1, W_LR_scale="Glorot") self.maxpool2 = MaxPool2d((2, 2), stride=(2, 2)) self.bn2 = BatchNorm2d(128, epsilon, alpha) self.tanh2 = SignBNN() self.conv3 = Conv2dBNN(128, 256, (3, 3), padding=1, H=1, W_LR_scale="Glorot") self.bn3 = BatchNorm2d(256, epsilon, alpha) self.tanh3 = SignBNN() self.conv4 = Conv2dBNN(256, 256, (3, 3), padding=1, H=1, W_LR_scale="Glorot") self.maxpool4 = MaxPool2d((2, 2), stride=(2, 2)) self.bn4 = BatchNorm2d(256, epsilon, alpha) self.tanh4 = SignBNN() self.conv5 = Conv2dBNN(256, 512, (3, 3), padding=1, H=1, W_LR_scale="Glorot") self.bn5 = BatchNorm2d(512, epsilon, alpha) self.tanh5 = SignBNN() self.conv6 = Conv2dBNN(512, 512, (3, 3), padding=1, H=1, W_LR_scale="Glorot") self.maxpool6 = MaxPool2d((2, 2), stride=(2, 2)) self.bn6 = BatchNorm2d(512, epsilon, alpha) self.tanh6 = SignBNN() self.linear7 = LinearBNN(25088, 1024, H=1, W_LR_scale="Glorot") self.tanh7 = SignBNN() self.linear8 = LinearBNN(1024, 1024, H=1, W_LR_scale="Glorot") self.tanh8 = SignBNN() self.linear9 = LinearBNN(1024, outChan, H=1, W_LR_scale="Glorot")