def forward(self, x): x = F.leaky_relu(self.fc1(x), 0.2, inplace=True) x = F.leaky_relu(self.fc11(x), 0.2, inplace=True) x = F.leaky_relu(self.fc2(x), 0.2, inplace=True) x = F.leaky_relu(self.fc3(x), 0.2, inplace=True) x = F.tanh(self.out(x)) return x
def forward(self, x): # If image_size is 64, output shape is as below. out = F.leaky_relu(self.conv1(x), 0.05) # (?, 64, 32, 32) out = F.leaky_relu(self.conv2(out), 0.05) # (?, 128, 16, 16) out = F.leaky_relu(self.conv3(out), 0.05) # (?, 256, 8, 8) out = F.leaky_relu(self.conv4(out), 0.05) # (?, 512, 4, 4) out = self.fc(out).squeeze() return out
def forward(self, x): out = F.leaky_relu(self.conv1(x), 0.05) # (?, 32, 14, 14) out = F.leaky_relu(self.conv2(out), 0.05) # (?, 64, 7, 7) out = F.leaky_relu(self.conv3(out), 0.05) # (?, 128, 3, 3) out = F.leaky_relu(self.conv4(out), 0.05) # (?, 256, 1, 1) out = out.squeeze() return torch.sigmoid(self.linear(out))
def forward(self, x): """ In the forward function we accept a Variable of input data and we must return a Variable of output data. We can use Modules defined in the constructor as well as arbitrary operators on Variables. """ x = self.first_deconv(x) x = self.first_batch_norm(x) x = F.leaky_relu(x) x = self.second_deconv(x) x = self.second_batch_norm(x) x = F.leaky_relu(x) x = self.third_deconv(x) x = self.third_batch_norm(x) x = F.leaky_relu(x) x = self.fourth_deconv(x) x = self.fourth_batch_norm(x) x = self.fifth_deconv(x) #sigmoid_out = nn.functional.sigmoid(x) tanh_out = nn.functional.tanh(x) out = (tanh_out + 1) * 255 /2 #print 'out.shape =', out.shape return out
def forward(self, input): x = F.leaky_relu(self.fc1(input), 0.2) x = F.leaky_relu(self.fc2(x), 0.2) x = F.leaky_relu(self.fc3(x), 0.2) x = F.tanh(self.fc4(x)) return x
def forward(self, state, action): """Build a critic (value) network that maps (state, action) pairs -> Q-values.""" xs = F.leaky_relu(self.fcs1(state)) x = torch.cat((xs, action), dim=1) x = F.leaky_relu(self.fc2(x)) x = F.leaky_relu(self.fc3(x)) return self.fc4(x)
def forward(self, z): z = z.view(z.size(0), z.size(1), 1, 1) # If image_size is 64, output shape is as below. out = self.fc(z) # (?, 512, 4, 4) out = F.leaky_relu(self.deconv1(out), 0.05) # (?, 256, 8, 8) out = F.leaky_relu(self.deconv2(out), 0.05) # (?, 128, 16, 16) out = F.leaky_relu(self.deconv3(out), 0.05) # (?, 64, 32, 32) out = F.tanh(self.deconv4(out)) # (?, 3, 64, 64) return out
def forward(self, input): x = F.leaky_relu(self.conv1(input), 0.2) x = F.leaky_relu(self.conv2_bn(self.conv2(x)), 0.2) x = F.leaky_relu(self.conv3_bn(self.conv3(x)), 0.2) x = F.leaky_relu(self.conv4_bn(self.conv4(x)), 0.2) x = F.sigmoid(self.conv5(x)) return x
def forward(self, x): out = F.leaky_relu(self.conv1(x), 0.05) # (?, 32, 27, 27) out = self.maxpool(out) # (?, 32, 13, 13) out = F.leaky_relu(self.conv2(out), 0.05) # (?, 64, 10, 10) out = self.maxpool(out) # (?, 64, 6, 6) out = out.view(-1, self.conv_dim*2*6*6) # (?, 64*8*8) out = F.leaky_relu(self.linear(out), 0.05) # (?, 1024) return torch.sigmoid(self.output(out).squeeze())
def forward(self, z): z = z.view(z.size(0), z.size(1), 1, 1) out = self.fc(z) # (?, 256, 2, 2) out = F.leaky_relu(self.deconv1(out), 0.05) # (?, 128, 4, 4) out = F.leaky_relu(self.deconv2(out), 0.05) # (?, 64, 7, 7) out = F.leaky_relu(self.deconv3(out), 0.05) # (?, 32, 14, 14) out = torch.sigmoid(self.deconv4(out)) # (?, 1, 28, 28) return out
def forward(self, x): x = F.leaky_relu(self.conv1(x), 0.2, inplace=True) x = F.leaky_relu(self.batch2(self.conv2(x)), 0.2, inplace=True) x = F.leaky_relu(self.batch3(self.conv3(x)), 0.2, inplace=True) x = F.leaky_relu(self.batch4(self.conv4(x)), 0.2, inplace=True) x = self.final_conv(x) return(x)
def forward(self, x): out = F.leaky_relu(self.conv1(x), 0.05) # (?, 32, 14, 14) out = F.leaky_relu(self.conv2(out), 0.05) # (?, 64, 7, 7) out = F.leaky_relu(self.conv3(out), 0.05) # (?, 128, 3, 3) out = F.leaky_relu(self.conv4(out), 0.05) # (?, 256, 1, 1) out = out.squeeze() mean, logvar = self.linear1(out), self.linear2(out) return mean, logvar
def forward(self, x): x = F.leaky_relu(self.batch1(self.convt1(x)), 0.2, inplace=True) x = F.leaky_relu(self.batch2(self.convt2(x)), 0.2, inplace=True) x = F.leaky_relu(self.batch3(self.convt3(x)), 0.2, inplace=True) x = F.leaky_relu(self.batch4(self.convt4(x)), 0.2, inplace=True) x = self.final_convt(x) x = F.tanh(x) return (x)
def forward(self, x1, x2): out = self.c0(torch.cat((x1, x2), 1)) out = self.bnc1(self.c1(F.leaky_relu(out, negative_slope=0.2))) out = self.bnc2(self.c2(F.leaky_relu(out, negative_slope=0.2))) out = self.bnc3(self.c3(F.leaky_relu(out, negative_slope=0.2))) out = self.c4(F.leaky_relu(out, negative_slope=0.2)) out = F.sigmoid(out) return out
def forward(self, x1, x2): h = self.c0(torch.cat((x1, x2),1)) h = self.bnc1(self.c1(F.leaky_relu(h, negative_slope=0.2))) h = self.bnc2(self.c2(F.leaky_relu(h, negative_slope=0.2))) h = self.bnc3(self.c3(F.leaky_relu(h, negative_slope=0.2))) h = self.c4(F.leaky_relu(h, negative_slope=0.2)) h = F.sigmoid(h) return h
def forward(self, image): # feed through neural network h = image.view(-1, self.n_pixels) h = F.leaky_relu(self.fc1(h)) h = F.leaky_relu(self.fc2(h)) latent_mean = self.sigmoid(self.fc3(h)) return latent_mean
def forward(self, latent_vars): # feed through neural network assert latent_vars.shape[1] == self.latent_dim h = F.leaky_relu(self.fc1(latent_vars)) h = F.leaky_relu(self.fc2(h)) image_mean = self.sigmoid(self.fc3(h)).view(-1, self.slen, self.slen) return image_mean
def forward(self, input): x = F.leaky_relu(self.fc1(input), 0.2) x = F.dropout(x, 0.3) x = F.leaky_relu(self.fc2(x), 0.2) x = F.dropout(x, 0.3) x = F.leaky_relu(self.fc3(x), 0.2) x = F.dropout(x, 0.3) x = F.sigmoid(self.fc4(x)) return x
def forward(self, input): x = F.leaky_relu(self.conv1(input), 0.2) x = F.leaky_relu(self.conv2_bn(self.conv2(x)), 0.2) x = F.leaky_relu(self.conv3_bn(self.conv3(x)), 0.2) x = F.leaky_relu(self.conv4_bn(self.conv4(x)), 0.2) x = x.view(-1, 2*2*self.d*8) x = torch.sigmoid(self.linear(x)) return x # (CS287 gans all run in 20 seconds ish, mine (model 5) takes 5 minutes. # Backprop eats up most time. It’s much bigger so this makes sense. Can I go faster by increasing batch size?)
def __init__( self, layer_type, in_channels, out_channels, kernel_size, stride, padding, activation=lambda x: F.leaky_relu(x, 0.2), dropout=True, batch_norm=True, dilation=1, sample_norm=False): super(ConvLayer, self).__init__() self.sample_norm = sample_norm self.dropout = dropout self.l1 = layer_type( in_channels, out_channels, kernel_size, stride, padding, bias=False, dilation=dilation) self.bn = None if batch_norm: if '1d' in layer_type.__name__: self.bn = nn.BatchNorm1d(out_channels) else: self.bn = nn.BatchNorm2d(out_channels) self.activation = activation
def forward(self, x): """ In the forward function we accept a Variable of input data and we must return a Variable of output data. We can use Modules defined in the constructor as well as arbitrary operators on Variables. """ x = self.dense(x) x = x.view(-1, 512, 4, 4) x = self.dense_batch_norm(x) x = F.leaky_relu(x) x = self.first_deconv(x) x = self.first_batch_norm(x) x = F.leaky_relu(x) #x = self.decA1(x) x = self.second_deconv(x) x = self.second_batch_norm(x) x = F.leaky_relu(x) #x = self.decA2(x) x = self.third_deconv(x) x = self.third_batch_norm(x) x = F.leaky_relu(x) # self-attention layer x = self.self_attention(x) #x = self.decA3(x) x = self.fourth_deconv(x) x = self.fourth_batch_norm(x) x = F.leaky_relu(x) x = self.fifth_deconv(x) #sigmoid_out = nn.functional.sigmoid(x) tanh_out = nn.functional.tanh(x) #out = (tanh_out + 1) * 255 / 2 #out = sigmoid_out out = tanh_out # print 'out.shape =', out.shape return out
def forward(self, x): x = super().forward(x) if self.activation == "leaky_relu": return functional.leaky_relu(x, negative_slope=self.slope, inplace=True) elif self.activation == "elu": return functional.elu(x, inplace=True) else: return x
def forward(self, x): ind = -2 self.loss = None outputs = dict() for block in self.blocks: ind = ind + 1 #if ind > 0: # return x if block['type'] == 'net': continue elif block['type'] == 'convolutional' or block['type'] == 'maxpool' or block['type'] == 'reorg' or block['type'] == 'avgpool' or block['type'] == 'softmax' or block['type'] == 'connected': x = self.models[ind](x) outputs[ind] = x elif block['type'] == 'route': layers = block['layers'].split(',') layers = [int(i) if int(i) > 0 else int(i)+ind for i in layers] if len(layers) == 1: x = outputs[layers[0]] outputs[ind] = x elif len(layers) == 2: x1 = outputs[layers[0]] x2 = outputs[layers[1]] x = torch.cat((x1,x2),1) outputs[ind] = x elif block['type'] == 'shortcut': from_layer = int(block['from']) activation = block['activation'] from_layer = from_layer if from_layer > 0 else from_layer + ind x1 = outputs[from_layer] x2 = outputs[ind-1] x = x1 + x2 if activation == 'leaky': x = F.leaky_relu(x, 0.1, inplace=True) elif activation == 'relu': x = F.relu(x, inplace=True) outputs[ind] = x elif block['type'] == 'region': continue if self.loss: self.loss = self.loss + self.models[ind](x) else: self.loss = self.models[ind](x) outputs[ind] = None elif block['type'] == 'cost': continue else: print('unknown type %s' % (block['type'])) return x
def __init__( self, in_channels, out_channels, kernel_size, stride, padding, dropout=True, activation=lambda x: F.leaky_relu(x, 0.2)): super(DecoderLayer, self).__init__( in_channels, out_channels, kernel_size, stride, padding, activation, dropout)
def forward(self, x): en0 = self.c0(x) en1 = self.bnc1(self.c1(F.leaky_relu(en0, negative_slope=0.2))) en2 = self.bnc2(self.c2(F.leaky_relu(en1, negative_slope=0.2))) en3 = self.bnc3(self.c3(F.leaky_relu(en2, negative_slope=0.2))) en4 = self.bnc4(self.c4(F.leaky_relu(en3, negative_slope=0.2))) en5 = self.bnc5(self.c5(F.leaky_relu(en4, negative_slope=0.2))) en6 = self.bnc6(self.c6(F.leaky_relu(en5, negative_slope=0.2))) en7 = self.c7(F.leaky_relu(en6, negative_slope=0.2)) de7 = self.bnd7(self.d7(F.relu(en7))) de6 = F.dropout(self.bnd6(self.d6(F.relu(torch.cat((en6, de7), 1))))) de5 = F.dropout(self.bnd5(self.d5(F.relu(torch.cat((en5, de6), 1))))) de4 = F.dropout(self.bnd4(self.d4(F.relu(torch.cat((en4, de5), 1))))) de3 = self.bnd3(self.d3(F.relu(torch.cat((en3, de4), 1)))) de2 = self.bnd2(self.d2(F.relu(torch.cat((en2, de3), 1)))) de1 = self.bnd1(self.d1(F.relu(torch.cat((en1, de2), 1)))) de0 = F.tanh(self.d0(F.relu(torch.cat((en0, de1), 1)))) return de0
def __init__( self, in_channels, out_channels, kernel_size, stride, padding, activation=lambda x: F.leaky_relu(x, 0.2), dropout=True, batch_norm=True, dilation=1, sample_norm=False): super(ConvTranspose2d, self).__init__( nn.ConvTranspose2d, in_channels, out_channels, kernel_size, stride, padding, activation=activation, dropout=dropout, batch_norm=batch_norm, dilation=dilation, sample_norm=sample_norm)
def forward(self, x): for c in self.convs: xt = F.leaky_relu(x, LRELU_SLOPE) xt = c(xt) x = xt + x return x
def prune_model_keep_size2(model, prune_idx, CBL_idx, CBLidx2mask): pruned_model = deepcopy(model) activations = [] for i, model_def in enumerate(model.module_defs): if model_def['type'] == 'convolutional': activation = torch.zeros(int(model_def['filters'])).cuda() if i in prune_idx: mask = torch.from_numpy(CBLidx2mask[i]).cuda() #之前变成numpy了 bn_module = pruned_model.module_list[i][1] bn_module.weight.data.mul_(mask) #加mask if model_def[ 'activation'] == 'leaky': #把bn的beta(bias)放入CBL的激活函数,输出,往下传递 activation = F.leaky_relu((1 - mask) * bn_module.bias.data, 0.1) elif model_def['activation'] == 'mish': activation = (1 - mask) * bn_module.bias.data.mul( F.softplus(bn_module.bias.data).tanh()) update_activation(i, pruned_model, activation, CBL_idx) bn_module.bias.data.mul_(mask) activations.append(activation) elif model_def['type'] == 'shortcut': actv1 = activations[i - 1] #+++++++++++++++++++++++++ insert +++++++++++++++++++++++++# # from_layer = int(model_def['from']) from_layer = int(model_def['from'][0]) #+++++++++++++++++++++++++ insert end++++++++++++++++++++++# actv2 = activations[i + from_layer] activation = actv1 + actv2 update_activation(i, pruned_model, activation, CBL_idx) activations.append(activation) elif model_def['type'] == 'route': #spp不参与剪枝,其中的route不用更新,仅占位 #+++++++++++++++++++++++++ insert +++++++++++++++++++++++++# # from_layers = [int(s) for s in model_def['layers'].split(',')] from_layers = model_def['layers'] #+++++++++++++++++++++++++ insert end++++++++++++++++++++++# activation = None if len(from_layers) == 1: activation = activations[ i + from_layers[0] if from_layers[0] < 0 else from_layers[0]] if 'groups' in model_def: activation = activation[(activation.shape[0] // 2):] update_activation(i, pruned_model, activation, CBL_idx) elif len(from_layers) == 2: actv1 = activations[i + from_layers[0]] actv2 = activations[ i + from_layers[1] if from_layers[1] < 0 else from_layers[1]] activation = torch.cat((actv1, actv2)) update_activation(i, pruned_model, activation, CBL_idx) activations.append(activation) elif model_def['type'] == 'upsample': # activation = torch.zeros(int(model.module_defs[i - 1]['filters'])).cuda() activations.append(activations[i - 1]) elif model_def['type'] == 'yolo': activations.append(None) elif model_def['type'] == 'maxpool': #区分spp和tiny if model.module_defs[i + 1]['type'] == 'route': activations.append(None) else: activation = activations[i - 1] update_activation(i, pruned_model, activation, CBL_idx) activations.append(activation) return pruned_model
def edge_attention(self, edges): # edge UDF for equation (2) z2 = torch.cat( [edges.src['z_concat_edges'], edges.dst['z_concat_edges']], dim=1) a = self.attn_fc(z2) return {'e': F.leaky_relu(a)}
def forward(self, state, action): """Build a critic (value) network that maps (state, action) pairs -> Q-values.""" xs = F.leaky_relu(self.fc1(state)) x = torch.cat((xs, action), dim=1) x = F.leaky_relu(self.fc2(x)) return self.fc3(x)
>>>>>>> main self.negative_slope = negative_slope self.scale = scale self.bias = nn.Parameter(torch.zeros(channel)) def forward(self, input): return fused_leaky_relu(input, self.bias, self.negative_slope, self.scale) def fused_leaky_relu(input, bias, negative_slope=0.2, scale=2 ** 0.5): if input.device.type == "cpu": <<<<<<< HEAD rest_dim = [1] * (input.ndim - bias.ndim - 1) return ( F.leaky_relu(input + bias.view(1, bias.shape[0], *rest_dim), negative_slope=0.2 ) * scale ) else: return FusedLeakyReLUFunction.apply(input, bias, negative_slope, scale) ======= if bias is not None: rest_dim = [1] * (input.ndim - bias.ndim - 1) return (F.leaky_relu(input + bias.view(1, bias.shape[0], *rest_dim), negative_slope=negative_slope ) * scale ) else: return F.leaky_relu(input, negative_slope=0.2) * scale else:
def forward(self, x): x = F.leaky_relu(self.fn1(x)) x = F.leaky_relu(self.fn2(x)) x = F.leaky_relu(self.fn3(x)) return x
def forwardaux(self, x): x = x / 255 x = torch.stack( [ x[:, 0, :, :] - self.transform[0][0], x[:, 1, :, :] - self.transform[0][1], x[:, 2, :, :] - self.transform[0][2], ], dim=1, ) x = torch.stack( [ x[:, 0, :, :] / self.transform[1][0], x[:, 1, :, :] / self.transform[1][1], x[:, 2, :, :] / self.transform[1][2], ], dim=1, ) x1 = F.leaky_relu(self.conv11(x)) x1 = F.leaky_relu(self.conv12(x1)) x1p = F.max_pool2d(x1, kernel_size=2, stride=2) x2 = F.leaky_relu(self.conv21(x1p)) x2 = F.leaky_relu(self.conv22(x2)) x2p = F.max_pool2d(x2, kernel_size=2, stride=2) x3 = F.leaky_relu(self.conv31(x2p)) x3 = F.leaky_relu(self.conv32(x3)) x3 = F.leaky_relu(self.conv33(x3)) x3p = F.max_pool2d(x3, kernel_size=2, stride=2) x4 = F.leaky_relu(self.conv41(x3p)) x4 = F.leaky_relu(self.conv42(x4)) x4 = F.leaky_relu(self.conv43(x4)) x4p = F.max_pool2d(x4, kernel_size=2, stride=2) x5 = F.leaky_relu(self.conv51(x4p)) x5 = F.leaky_relu(self.conv52(x5)) x5 = F.leaky_relu(self.conv53(x5)) x_grad_16 = self.gradientdoor16(x5) x5u = F.upsample_nearest(x5, scale_factor=2) x4 = torch.cat((x5u, x4), 1) x4 = F.leaky_relu(self.conv43d(x4)) x4 = F.leaky_relu(self.conv42d(x4)) x4 = F.leaky_relu(self.conv41d(x4)) x_grad_8 = self.gradientdoor8(x4) x4u = F.upsample_nearest(x4, scale_factor=2) x3 = torch.cat((x4u, x3), 1) x3 = F.leaky_relu(self.conv33d(x3)) x3 = F.leaky_relu(self.conv32d(x3)) x3 = F.leaky_relu(self.conv31d(x3)) x_grad_4 = self.gradientdoor4(x3) x3u = F.upsample_nearest(x3, scale_factor=2) x2 = torch.cat((x3u, x2), 1) x2 = F.leaky_relu(self.conv22d(x2)) x2 = F.leaky_relu(self.conv21d(x2)) x_grad_2 = self.gradientdoor2(x2) x2u = F.upsample_nearest(x2, scale_factor=2) x1 = torch.cat((x2u, x1), 1) x1 = F.leaky_relu(self.conv12d(x1)) x = self.conv11d(x1) return x, (x_grad_2, x_grad_4, x_grad_8, x_grad_16)
def prune_model_keep_size2(model, prune_idx, CBL_idx, CBLidx2mask): pruned_model = deepcopy(model) activations = [] for i, model_def in enumerate(model.module_defs): if model_def['type'] == 'convolutional': activation = torch.zeros(int(model_def['filters'])).cuda() if i in prune_idx: mask = torch.from_numpy(CBLidx2mask[i]).cuda() # mask = torch.from_numpy(CBLidx2mask[i]) bn_module = pruned_model.module_list[i][1] bn_module.weight.data.mul_(mask) if model_def['activation'] == 'leaky': activation = F.leaky_relu((1 - mask) * bn_module.bias.data, 0.1) elif model_def['activation'] == 'mish': activation = (1 - mask) * bn_module.bias.data.mul(F.softplus(bn_module.bias.data).tanh()) elif model_def['activation'] == 'SiLU': #yolov5-v4 activation=(1 - mask) * bn_module.bias.data.mul(F.sigmoid(bn_module.bias.data)) elif model_def['activation'] == 'Hardswish': activation=(1 - mask) *bn_module.bias.data.mul(F.hardtanh(bn_module.bias.data + 3, 0., 6.) / 6.) update_activation(i, pruned_model, activation, CBL_idx) bn_module.bias.data.mul_(mask) activations.append(activation) elif model_def['type'] == 'shortcut': actv1 = activations[i - 1] from_layer = int(model_def['from']) actv2 = activations[i + from_layer] activation = actv1 + actv2 update_activation(i, pruned_model, activation, CBL_idx) activations.append(activation) elif model_def['type'] == 'route': #spp不参与剪枝,其中的route不用更新,仅占位 from_layers = [int(s) for s in model_def['layers'].split(',')] activation = None if len(from_layers) == 1: activation = activations[i + from_layers[0] if from_layers[0] < 0 else from_layers[0]] if 'groups' in model_def: activation = activation[(activation.shape[0]//2):] update_activation(i, pruned_model, activation, CBL_idx) elif len(from_layers) == 2: actv1 = activations[i + from_layers[0]] actv2 = activations[i + from_layers[1] if from_layers[1] < 0 else from_layers[1]] activation = torch.cat((actv1, actv2)) # update_activation(i, pruned_model, activation, CBL_idx) #update_activation_nconv next_idx = i + 1 if pruned_model.module_defs[next_idx]['type'] == 'convolutional_noconv': next_conv1 = pruned_model.module_list[i + from_layers[0]][0] next_conv2 = pruned_model.module_list[i + from_layers[1] if from_layers[1] < 0 else from_layers[1]][0] conv_sum1 = next_conv1.weight.data.sum(dim=(2, 3)) conv_sum2 = next_conv2.weight.data.sum(dim=(2, 3)) offset1 = conv_sum1.matmul(actv1.reshape(-1, 1)).reshape(-1) offset2 = conv_sum2.matmul(actv2.reshape(-1, 1)).reshape(-1) offset=torch.cat((offset1, offset2)) if next_idx in CBL_idx: next_bn = pruned_model.module_list[next_idx][0] next_bn.running_mean.data.sub_(offset) else: update_activation(i, pruned_model, activation, CBL_idx) activations.append(activation) elif model_def['type'] == 'upsample': # activation = torch.zeros(int(model.module_defs[i - 1]['filters'])).cuda() activations.append(activations[i-1]) elif model_def['type'] == 'yolo': activations.append(None) elif model_def['type'] == 'focus': activations.append(None) elif model_def['type'] == 'convolutional_nobias': activations.append(activations[i - 1]) # activation = torch.zeros(int(model_def['filters'])).cuda() # activations.append(activation) elif model_def['type'] == 'convolutional_noconv': activation = torch.zeros(int(model_def['filters'])).cuda() if i in prune_idx: mask = torch.from_numpy(CBLidx2mask[i]).cuda() # mask = torch.from_numpy(CBLidx2mask[i]) bn_module = pruned_model.module_list[i][0] bn_module.weight.data.mul_(mask) activation = F.leaky_relu((1 - mask) * bn_module.bias.data, 0.1) # if model_def['activation'] == 'leaky': # activation = F.leaky_relu((1 - mask) * bn_module.bias.data, 0.1) # elif model_def['activation'] == 'mish': # activation = (1 - mask) * bn_module.bias.data.mul(F.softplus(bn_module.bias.data).tanh()) update_activation(i, pruned_model, activation, CBL_idx) bn_module.bias.data.mul_(mask) activations.append(activation) elif model_def['type'] == 'maxpool': #区分spp和tiny if model.module_defs[i + 1]['type'] == 'route': activations.append(None) else: activation = activations[i-1] update_activation(i, pruned_model, activation, CBL_idx) activations.append(activation) return pruned_model
def actvn(self, x): return F.leaky_relu(x, 2e-1)
def forward_raw(self, input: torch.Tensor) -> torch.Tensor: return F.leaky_relu(input, self._negative_slope, self._inplace)
def forward(self, x): y = self.m_conv(x) y = F.relu(self.m_bn(y), inplace=True) if not self.m_useLeakyReLU \ else F.leaky_relu(self.m_bn(y), inplace=True) return y
def forward(self, x): x = self.conv1(x) x = self.pool(F.leaky_relu(self.bn2d1(x))) x = self.conv2(x) x = self.pool(F.leaky_relu(self.bn2d2(x))) x = self.conv3(x) x = self.pool(F.leaky_relu(self.bn2d3(x))) x = x.view(x.size(0), -1) x = self.bn1d(self.fc1(x)) x = x.view(x.size(0), int(self.rep_dim / (4 * 4)), 4, 4) x = F.leaky_relu(x) x = self.deconv1(x) x = F.interpolate(F.leaky_relu(self.bn2d4(x)), scale_factor=2) x = self.deconv2(x) x = F.interpolate(F.leaky_relu(self.bn2d5(x)), scale_factor=2) x = self.deconv3(x) x = F.interpolate(F.leaky_relu(self.bn2d6(x)), scale_factor=2) x = self.deconv4(x) x = F.interpolate(F.leaky_relu(self.bn2d7(x)), scale_factor=2) x = self.deconv5(x) x = F.interpolate(F.leaky_relu(self.bn2d8(x)), scale_factor=2) x = self.deconv6(x) x = torch.sigmoid(x) """ print(x.size()) x = self.conv1(x) print(x.size()) x = self.pool(F.leaky_relu(self.bn2d1(x))) print(x.size()) x = self.conv2(x) print(x.size()) x = self.pool(F.leaky_relu(self.bn2d2(x))) print(x.size()) x = self.conv3(x) print(x.size()) x = self.pool(F.leaky_relu(self.bn2d3(x))) print(x.size()) x = x.view(x.size(0), -1) print(x.size()) x = self.bn1d(self.fc1(x)) print(x.size()) x = x.view(x.size(0), int(self.rep_dim / (4 * 4)), 4, 4) print(x.size()) x = F.leaky_relu(x) print(x.size()) print("-----------deconv-------------") x = self.deconv1(x) print(x.size()) x = F.interpolate(F.leaky_relu(self.bn2d4(x)), scale_factor=2) print(x.size()) x = self.deconv2(x) print(x.size()) x = F.interpolate(F.leaky_relu(self.bn2d5(x)), scale_factor=2) print(x.size()) x = self.deconv3(x) print(x.size()) x = F.interpolate(F.leaky_relu(self.bn2d6(x)), scale_factor=2) print(x.size()) x = self.deconv4(x) print(x.size()) x = F.interpolate(F.leaky_relu(self.bn2d7(x)), scale_factor=2) print(x.size()) x = self.deconv5(x) print(x.size()) x = F.interpolate(F.leaky_relu(self.bn2d8(x)), scale_factor=2) print(x.size()) x = self.deconv6(x) x = torch.sigmoid(x) print(x.size())""" return x
def forward(self, atom_list, bond_list, atom_degree_list, bond_degree_list, atom_mask, descriptors): desc = list(descriptors) atom_mask = atom_mask.unsqueeze(2) batch_size,mol_length,num_atom_feat = atom_list.size() atom_feature = F.leaky_relu(self.atom_linear_layer(atom_list)) bond_neighbor = [bond_list[i][bond_degree_list[i]] for i in range(batch_size)] bond_neighbor = torch.stack(bond_neighbor, dim=0) atom_neighbor = [atom_list[i][atom_degree_list[i]] for i in range(batch_size)] atom_neighbor = torch.stack(atom_neighbor, dim=0) neighbor_feature = torch.cat([atom_neighbor, bond_neighbor],dim=-1) neighbor_feature = F.leaky_relu(self.neighbor_linear_layer(neighbor_feature)) attend_mask = atom_degree_list.clone() attend_mask[attend_mask != mol_length-1] = 1 attend_mask[attend_mask == mol_length-1] = 0 attend_mask = attend_mask.type(torch.cuda.FloatTensor).unsqueeze(-1) softmax_mask = atom_degree_list.clone() softmax_mask[softmax_mask != mol_length-1] = 0 softmax_mask[softmax_mask == mol_length-1] = -9e8 softmax_mask = softmax_mask.type(torch.cuda.FloatTensor).unsqueeze(-1) batch_size, mol_length, max_neighbor_num, descriptor_dim = neighbor_feature.shape atom_feature_expand = atom_feature.unsqueeze(-2).expand(batch_size, mol_length, max_neighbor_num, descriptor_dim) feature_align = torch.cat([atom_feature_expand, neighbor_feature],dim=-1) align_score = F.leaky_relu(self.align[0](self.dropout(feature_align))) align_score = align_score + softmax_mask attention_weight = F.softmax(align_score,-2) attention_weight = attention_weight * attend_mask neighbor_feature_transform = self.attend[0](self.dropout(neighbor_feature)) context = torch.sum(torch.mul(attention_weight,neighbor_feature_transform),-2) context = F.elu(context) context_reshape = context.view(batch_size*mol_length, descriptor_dim) atom_feature_reshape = atom_feature.view(batch_size*mol_length, descriptor_dim) atom_feature_reshape = self.GRUCell[0](context_reshape, atom_feature_reshape) atom_feature = atom_feature_reshape.view(batch_size, mol_length, descriptor_dim) activated_features = F.relu(atom_feature) for d in range(self.radius-1): neighbor_feature = [activated_features[i][atom_degree_list[i]] for i in range(batch_size)] neighbor_feature = torch.stack(neighbor_feature, dim=0) atom_feature_expand = activated_features.unsqueeze(-2).expand(batch_size, mol_length, max_neighbor_num, descriptor_dim) feature_align = torch.cat([atom_feature_expand, neighbor_feature],dim=-1) align_score = F.leaky_relu(self.align[d+1](self.dropout(feature_align))) align_score = align_score + softmax_mask attention_weight = F.softmax(align_score,-2) attention_weight = attention_weight * attend_mask neighbor_feature_transform = self.attend[d+1](self.dropout(neighbor_feature)) context = torch.sum(torch.mul(attention_weight,neighbor_feature_transform),-2) context = F.elu(context) context_reshape = context.view(batch_size*mol_length, descriptor_dim) atom_feature_reshape = self.GRUCell[d+1](context_reshape, atom_feature_reshape) atom_feature = atom_feature_reshape.view(batch_size, mol_length, descriptor_dim) activated_features = F.relu(atom_feature) mol_feature = torch.sum(activated_features * atom_mask, dim=-2) activated_features_mol = F.relu(mol_feature) mol_softmax_mask = atom_mask.clone() mol_softmax_mask[mol_softmax_mask == 0] = -9e8 mol_softmax_mask[mol_softmax_mask == 1] = 0 mol_softmax_mask = mol_softmax_mask.type(torch.cuda.FloatTensor) for t in range(self.T): mol_prediction_expand = activated_features_mol.unsqueeze(-2).expand(batch_size, mol_length, descriptor_dim) mol_align = torch.cat([mol_prediction_expand, activated_features], dim=-1) mol_align_score = F.leaky_relu(self.mol_align(mol_align)) mol_align_score = mol_align_score + mol_softmax_mask mol_attention_weight = F.softmax(mol_align_score,-2) mol_attention_weight = mol_attention_weight * atom_mask activated_features_transform = self.mol_attend(self.dropout(activated_features)) mol_context = torch.sum(torch.mul(mol_attention_weight,activated_features_transform),-2) mol_context = F.elu(mol_context) mol_feature = self.mol_GRUCell(mol_context, mol_feature) activated_features_mol = F.relu(mol_feature) x = self.dropout(mol_feature) x = self.fc_g1(x) for i in range(len(desc)): desc[i] = torch.unsqueeze(desc[i], 0) desc = torch.cat(desc, 0) desc = self.SmallNetwork(desc) x_con_desc = torch.cat((x, desc), 1) mol_feature = self.fc1(x_con_desc) mol_feature = self.relu(mol_feature) mol_feature = self.dropout(mol_feature) mol_feature = self.fc2(mol_feature) mol_feature = self.relu(mol_feature) mol_feature = self.dropout(mol_feature) mol_prediction = self.output(mol_feature) return atom_feature, mol_prediction
def forward(self, x): x = F.leaky_relu(self.map1(x), 0.1) x = F.leaky_relu(self.map2(x), 0.1) return F.sigmoid(self.map3(x))
def forward(self, x): x = F.leaky_relu(self.fc1(x)) x = F.leaky_relu(self.fc2(x)) x = self.fc3(x) return x
def forward(self, input): out = F.leaky_relu(input, negative_slope=self.negative_slope) return out * math.sqrt(2)
def forward(self, x): x = grad_reverse(x, self.alpha) # x = F.leaky_relu(self.drop(self.fc1(x))) x = F.leaky_relu(self.fc1(x)) return self.fc2(x)
def forward(self, rgb, global_step, test, batch_num=0): # input: rgb (1, 3, 127, 127) loss = 0 if not test: if cfg.CONST.dynamic_dict: if (global_step % 1000) == 0 and batch_num == 0: with torch.no_grad(): self.update_embedding_dynamically() embed = self.encodingnet(rgb) #print(embed.mean(), embed.std()) embed_shape = embed.shape distances = (torch.sum(embed**2, dim=1, keepdim=True) + torch.sum(self.embedding.weight**2, dim=1) - 2 * torch.matmul(embed, self.embedding.weight.t())) encoding_indices = torch.argmin(distances, dim=1).unsqueeze(1) for idx in encoding_indices.view(-1): self.prototype_usage[idx] += 1 encodings = torch.zeros(encoding_indices.shape[0], vqvae_dict_size, device=embed.device) encodings.scatter_(1, encoding_indices, 1) '''Quantize and unflatten''' quantized = torch.matmul(encodings, self.embedding.weight).view(embed_shape) '''Loss''' e_latent_loss = F.mse_loss(quantized.detach(), embed) q_latent_loss = F.mse_loss(quantized, embed.detach()) loss = q_latent_loss + self.commitment_cost * e_latent_loss quantized = embed + (quantized - embed).detach() embed = quantized if hypernet_nonlinear: embed = self.hidden1(embed) embed = F.leaky_relu(embed) embed = self.hidden2(embed) embed = F.leaky_relu(embed) feat_kernels_enc_conv = [ (torch.matmul(embed, self.kernel_conv_encoderWeights[i])).view( self.enc_conv_wts_size[i]) for i in range(len(self.kernel_conv_encoderWeights)) ] feat_bias_enc_conv = self.bias_conv_encoderWeights #[(torch.matmul(embed,self.bias_conv_encoderWeights[i])).view([B]+self.enc_conv_bias_size[i][0]) for i in range(len(self.bias_conv_encoderWeights))] feat_kernels_enc_fc = [ (torch.matmul(embed, self.kernel_fc_encoderWeights[i])).view( self.enc_fc_wts_size[i]) for i in range(len(self.kernel_fc_encoderWeights)) ] feat_bias_enc_fc = self.bias_fc_encoderWeights #[(torch.matmul(embed,self.bias_fc_encoderWeights[i])).view([B]+self.enc_fc_bias_size[i][0]) for i in range(len(self.bias_fc_encoderWeights))] feat_kernels_dec_conv = [ (torch.matmul(embed, self.kernel_conv_decoderWeights[i])).view( self.dec_conv_wts_size[i]) for i in range(len(self.kernel_conv_decoderWeights)) ] feat_bias_dec_conv = self.bias_conv_decoderWeights #[(torch.matmul(embed,self.bias_conv_decoderWeights[i])).view([B]+self.dec_conv_bias_size[i][0]) for i in range(len(self.bias_conv_decoderWeights))] feat_kernels_enc_3dgru = [ (torch.matmul(embed, self.kernel_3dgru_encoderWeights[i])).view( self.enc_3dgru_wts_size[i]) for i in range(len(self.kernel_3dgru_encoderWeights)) ] feat_bias_enc_3dgru = self.bias_3dgru_encoderWeights #[(torch.matmul(embed,self.bias_3dgru_encoderWeights[i])).view([B]+self.enc_3dgru_bias_size[i]) for i in range(len(self.bias_3dgru_encoderWeights))] return loss, feat_kernels_enc_conv, feat_bias_enc_conv, feat_kernels_enc_fc, feat_bias_enc_fc, feat_kernels_enc_3dgru, feat_bias_enc_3dgru, feat_kernels_dec_conv, feat_bias_dec_conv
def forward(self, x): x, trans, trans_feat = self.feat(x) x = F.leaky_relu(self.bn1(self.fc1(x))) x = F.leaky_relu(self.bn2(self.dropout(self.fc2(x)))) x = self.fc3(x) return torch.sigmoid(x), trans, trans_feat
def forward(self, x): x = leaky_relu(self.map1(x), 0.1) return sigmoid(self.map2(x))
def forward(self, input): if self.bn: # BN融合 if self.bias is not None: bias = reshape_to_bias(self.beta + (self.bias - self.running_mean) * ( self.gamma / torch.sqrt(self.running_var + self.eps))) else: bias = reshape_to_bias( self.beta - self.running_mean * self.gamma / torch.sqrt( self.running_var + self.eps)) # b融running weight = self.weight * reshape_to_weight( self.gamma / torch.sqrt(self.running_var + self.eps)) # w融running else: bias = self.bias weight = self.weight # 量化A和bn融合后的W q_weight = self.weight_quantizer(weight) q_bias = self.bias_quantizer(bias) if self.quantizer_output == True: # 输出量化参数txt文档 # 创建的quantizer_output输出文件夹 if not os.path.isdir('./quantizer_output'): os.makedirs('./quantizer_output') if not os.path.isdir('./quantizer_output/q_weight_out'): os.makedirs('./quantizer_output/q_weight_out') if not os.path.isdir('./quantizer_output/w_scale_out'): os.makedirs('./quantizer_output/w_scale_out') if not os.path.isdir('./quantizer_output/q_weight_max'): os.makedirs('./quantizer_output/q_weight_max') if not os.path.isdir('./quantizer_output/max_weight_count'): os.makedirs('./quantizer_output/max_weight_count') if not os.path.isdir('./quantizer_output/q_weight_reorder'): os.makedirs('./quantizer_output/q_weight_reorder') if not os.path.isdir('./quantizer_output/q_bias_reorder'): os.makedirs('./quantizer_output/q_bias_reorder') #######################输出当前层的权重量化因子 weight_scale = self.weight_quantizer.get_scale() np.savetxt(('./quantizer_output/w_scale_out/%f.txt' % time.time()), weight_scale, delimiter='\n') #######################输出当前层的量化权重 q_weight_txt = self.weight_quantizer.get_quantize_value(weight) #############权重重排序 w_para = q_weight_txt # 重排序参数 if self.reorder == True: #print("use weights reorder!") shape_output = w_para.shape[0] shape_input = w_para.shape[1] num_TN = int(shape_input / self.TN) remainder_TN = shape_input % self.TN num_TM = int(shape_output / self.TM) remainder_TM = shape_output % self.TM first = True reorder_w_para = None if self.activate == 'linear': print('layer-linear reorder!') for k in range(num_TN): temp = w_para[0:remainder_TM, k * self.TN:(k + 1) * self.TN, :, :] temp = temp.view(temp.shape[0], temp.shape[1], temp.shape[2] * temp.shape[3]) temp = temp.permute(2, 0, 1).contiguous().view(-1) if first: reorder_w_para = temp.clone().cpu().data.numpy() first = False else: reorder_w_para = np.append(reorder_w_para, temp.cpu().data.numpy()) else: for j in range(num_TM): if shape_input == 3 or shape_input == 1: # 第一层 print('The first layer~~~~~~~~~~~~') temp = w_para[j * self.TM:(j + 1) * self.TM, num_TN * self.TN:num_TN * self.TN + remainder_TN, :, :] temp = temp.view(temp.shape[0], temp.shape[1], temp.shape[2] * temp.shape[3]) fill = torch.zeros(self.TM, self.TN, temp.shape[2]).to(temp.device) fill[:, 0:remainder_TN, :] = temp temp = fill.permute(2, 0, 1).contiguous().view(-1) if first: # 创建数组存储 reorder_w_para = temp.clone().cpu().data.numpy() first = False else: reorder_w_para = np.append(reorder_w_para, temp.cpu().data.numpy()) else: for k in range(num_TN): temp = w_para[j * self.TM:(j + 1) * self.TM, k * self.TN:(k + 1) * self.TN, :, :] # #合并成论文图10(a)的TM*TN*(K2)的张量格式 temp = temp.view(temp.shape[0], temp.shape[1], temp.shape[2] * temp.shape[3]) # 转换为图10(b)的重排序格式 temp = temp.permute(2, 0, 1).contiguous().view(-1) if first: reorder_w_para = temp.clone().cpu().data.numpy() first = False else: reorder_w_para = np.append(reorder_w_para, temp.cpu().data.numpy()) w_para_flatten = reorder_w_para # print(reorder_w_para.size) #####验证重排序结果的正确性 '''if w_para_flatten.size == w_para.shape[0] * w_para.shape[1] * w_para.shape[2] * w_para.shape[3]: print("weights convert correctly!") else: print("weights convert mismatchingly!")''' q_weight_reorder = w_para_flatten q_weight_reorder = np.array(q_weight_reorder).reshape(1, -1) np.savetxt(('./quantizer_output/q_weight_reorder/%f.txt' % time.time()), q_weight_reorder, delimiter='\n') ################权重重排序结束 q_weight_txt = np.array(q_weight_txt.cpu()).reshape(1, -1) q_weight_max = [np.max(q_weight_txt)] # q_weight_max = np.argmax(q_weight_txt) max_weight_count = [np.sum(abs(q_weight_txt) >= 127)] # 统计该层溢出的数目 np.savetxt(('./quantizer_output/max_weight_count/%f.txt' % time.time()), max_weight_count) np.savetxt(('./quantizer_output/q_weight_max/%f.txt' % time.time()), q_weight_max) np.savetxt(('./quantizer_output/q_weight_out/%f.txt' % time.time()), q_weight_txt, delimiter='\n') # io.savemat('save.mat',{'q_weight_txt':q_weight_txt}) #######################创建输出偏置txt的文件夹 if not os.path.isdir('./quantizer_output/q_bias_out'): os.makedirs('./quantizer_output/q_bias_out') if not os.path.isdir('./quantizer_output/b_scale_out'): os.makedirs('./quantizer_output/b_scale_out') #######################输出当前层偏置的量化因子 bias_scale = self.bias_quantizer.get_scale() np.savetxt(('./quantizer_output/b_scale_out/%f.txt' % time.time()), bias_scale, delimiter='\n') #######################输出当前层的量化偏置 q_bias_txt = self.bias_quantizer.get_quantize_value(bias) q_bias_txt = np.array(q_bias_txt.cpu()).reshape(1, -1) np.savetxt(('./quantizer_output/q_bias_out/%f.txt' % time.time()), q_bias_txt, delimiter='\n') #############偏置重排序 if self.reorder == True: b_para = np.zeros(2048, dtype=int) b_para[0:q_bias_txt.size] = q_bias_txt # print(b_para.shape) # b_para = np.array(b_para.cpu()).reshape(1, -1) np.savetxt(('./quantizer_output/q_bias_reorder/%f.txt' % time.time()), b_para, delimiter='\n') ################偏置重排序结束 # 量化卷积 output = F.conv2d( input=input, weight=q_weight, bias=q_bias, # 注意,这里加bias,做完整的conv+bn stride=self.stride, padding=self.padding, dilation=self.dilation, groups=self.groups ) if self.activate == 'leaky': output = F.leaky_relu(output, 0.125, inplace=True) elif self.activate == 'relu6': output = F.relu6(output, inplace=True) elif self.activate == 'h_swish': output = output * (F.relu6(output + 3.0, inplace=True) / 6.0) elif self.activate == 'relu': output = F.relu(output, inplace=True) elif self.activate == 'mish': output = output * F.softplus(output).tanh() elif self.activate == 'linear': # return output pass else: print(self.activate + "%s is not supported !") if self.quantizer_output == True: if not os.path.isdir('./quantizer_output/q_activation_out'): os.makedirs('./quantizer_output/q_activation_out') if not os.path.isdir('./quantizer_output/a_scale_out'): os.makedirs('./quantizer_output/a_scale_out') if not os.path.isdir('./quantizer_output/q_activation_max'): os.makedirs('./quantizer_output/q_activation_max') if not os.path.isdir('./quantizer_output/max_activation_count'): os.makedirs('./quantizer_output/max_activation_count') if not os.path.isdir('./quantizer_output/q_activation_reorder'): os.makedirs('./quantizer_output/q_activation_reorder') ##################输出当前激活的量化因子 activation_scale = self.activation_quantizer.get_scale() np.savetxt(('./quantizer_output/a_scale_out/%f.txt' % time.time()), activation_scale, delimiter='\n') ##################输出当前层的量化激活 q_activation_txt = self.activation_quantizer.get_quantize_value(output) a_para = q_activation_txt #############输入特征图重排序 if self.reorder == True: # 重排序参数 #print("use activation reorder!") shape_input = a_para.shape[1] num_TN = int(shape_input / self.TN) remainder_TN = shape_input % self.TN first = True reorder_a_para = None if self.activate == 'linear': print('layer-linear reorder!') temp = a_para[:, 0:remainder_TN, :, :] temp = temp.view(temp.shape[1], temp.shape[2], temp.shape[3]) temp = temp.permute(2, 1, 0).contiguous().view(-1) if first: reorder_a_para = temp.clone().cpu().data.numpy() first = False else: reorder_a_para = np.append(reorder_a_para, temp.cpu().data.numpy()) else: for k in range(num_TN): temp = a_para[:, k * self.TN:(k + 1) * self.TN, :, :] temp = temp.view(temp.shape[1], temp.shape[2], temp.shape[3]) temp = temp.permute(2, 1, 0).contiguous().view(-1) if first: reorder_a_para = temp.clone().cpu().data.numpy() first = False else: reorder_a_para = np.append(reorder_a_para, temp.cpu().data.numpy()) a_para_flatten = reorder_a_para #####验证重排序结果的正确性 '''if a_para_flatten.size == a_para.shape[0] * a_para.shape[1] * a_para.shape[2] * a_para.shape[3]: print("activation convert correctly!") else: print("activation convert mismatchingly!")''' q_activation_reorder = a_para_flatten q_activation_reorder = np.array(q_activation_reorder).reshape(1, -1) np.savetxt(('./quantizer_output/q_activation_reorder/%f.txt' % time.time()), q_activation_reorder, delimiter='\n') ##########特征图重排序结束 q_activation_txt = np.array(q_activation_txt.cpu()).reshape(1, -1) q_activation_max = [np.max(q_activation_txt)] # 统计该层的最大值(即查看是否有溢出) max_activation_count = [np.sum(abs(q_activation_txt) >= 127)] # 统计该层溢出的数目 # q_weight_max = np.argmax(q_weight_txt) np.savetxt(('./quantizer_output/max_activation_count/%f.txt' % time.time()), max_activation_count) np.savetxt(('./quantizer_output/q_activation_max/%f.txt' % time.time()), q_activation_max) np.savetxt(('./quantizer_output/q_activation_out/%f.txt' % time.time()), q_activation_txt, delimiter='\n') output = self.activation_quantizer(output) return output
def forward(self, input): x = input.view(input.shape[0], -1) x = F.leaky_relu(self.fc1(x), 0.2, inplace=True) x = F.leaky_relu(self.fc2(x), 0.2, inplace=True) x = F.sigmoid(self.out(x)) return x
def forward(self, x): return F.leaky_relu(self.bn(self.conv(x)), 0.2)
def forward(self, x): out = (F.relu(self.linear0(x))) out = (F.relu(self.linear1(out))) out = (F.leaky_relu(self.linear2(out))) return out
def forward(self, state): """Build an actor (policy) network that maps states -> actions.""" x = F.leaky_relu(self.fc1(state)) x = F.leaky_relu(self.fc2(x)) return torch.tanh(self.fc3(x))
def forward(self, input): out = F.leaky_relu(self.fc1(input), LEAK) out = F.leaky_relu(self.fc2(out), LEAK) out = F.leaky_relu(self.fc3(out), LEAK) out = F.tanh(self.fc4(out)) return out
def forward(self, x): x11 = F.leaky_relu(self.bn1(self.conv1(x))) x12 = F.leaky_relu(self.bn2(self.conv2(x11))) x12 += self.skip(x) xp = self.ds(x12) return xp, xp, x12.size()
def forward(self, x): x = self.fc_1(x) x = F.leaky_relu(self.bn(x)) x = self.fc_2(x) return x
def forward(self, x): batch_size = x.size(0) # shape of x : batch, feature, npoints, neighbors # convolution(shared mlp to xi, xj-xi & max) # => # transformer(shared wq, wk, wv to xi) if self.ape: ptcld = x else: ptcld = None x, abs_x, deg1, idx1 = get_neighbors(x, k=self.k, deg=self.deg, delta=self.delta, neighbors=self.neighbors, bn=self.deg_bn1, layer=1) # b, 64, 1024, 20 x1 = self.conv1(x, abs_x, deg1, idx1) # b, 64, 1024 x1 = self.act1(self.bn1(x1)).squeeze(3) x, abs_x, deg2, idx2 = get_neighbors(x1, k=self.k, deg=self.deg, delta=self.delta, neighbors=self.neighbors, bn=self.deg_bn2, layer=2) # b, 64, 1024, 20 x2 = self.conv2(x, abs_x, deg2, idx2) # b, 64, 1024 x2 = self.act2(self.bn2(x2)).squeeze(3) x, abs_x, deg3, idx3 = get_neighbors(x2, k=self.k, deg=self.deg, delta=self.delta, neighbors=self.neighbors, bn=self.deg_bn3, layer=3) # b, 64, 1024, 20 x3 = self.conv3(x, abs_x, deg3, idx3) # b, 128, 1024 x3 = self.act3(self.bn3(x3)).squeeze(3) x, abs_x, deg4, idx4 = get_neighbors(x3, k=self.k, deg=self.deg, delta=self.delta, neighbors=self.neighbors, bn=self.deg_bn4, layer=4) # b, 64, 1024, 20 x4 = self.conv4(x, abs_x, deg4, idx4) # b, 256, 1024, 20 x4 = self.act4(self.bn4(x4)).squeeze(3) x = self.conv5(x4) x = F.adaptive_max_pool1d(x, 1).view(batch_size, -1) x = F.leaky_relu(self.bn6(self.linear1(x)), negative_slope=0.2) x = self.dp1(x) x = F.leaky_relu(self.bn7(self.linear2(x)), negative_slope=0.2) x = self.dp2(x) x = self.linear3(x) return x
def forward(self, x, vars=None, dropout_training=True, bn_training=True): """ This function can be called by finetunning, however, in finetunning, we dont wish to update running_mean/running_var. Thought weights/bias of bn is updated, it has been separated by fast_weights. Indeed, to not update running_mean/running_var, we need set update_bn_statistics=False but weight/bias will be updated and not dirty initial theta parameters via fast_weiths. :param x: [b, 1, 28, 28] :param vars: :param bn_training: set False to not update :return: x, loss, likelihood, kld """ if vars is None: vars = self.vars idx = 0 bn_idx = 0 for name, param in self.config: if name == 'conv2d': w, b = vars[idx], vars[idx + 1] # remember to keep synchrozied of forward_encoder and forward_decoder! x = F.conv2d(x, w, b, stride=param[4], padding=param[5]) idx += 2 # print(name, param, '\tout:', x.shape) elif name == 'convt2d': w, b = vars[idx], vars[idx + 1] # remember to keep synchrozied of forward_encoder and forward_decoder! x = F.conv_transpose2d(x, w, b, stride=param[4], padding=param[5]) idx += 2 # print(name, param, '\tout:', x.shape) elif name == 'linear': w, b = vars[idx], vars[idx + 1] x = F.linear(x, w, b) idx += 2 # print('forward:', idx, x.norm().item()) elif name == 'bn': w, b = vars[idx], vars[idx + 1] running_mean, running_var = self.vars_bn[bn_idx], self.vars_bn[ bn_idx + 1] x = F.batch_norm(x, running_mean, running_var, weight=w, bias=b, training=bn_training) idx += 2 bn_idx += 2 elif name == 'flatten': # print(x.shape) x = x.view(x.size(0), -1) #x = F.mean(x, [1, 2]) elif name == 'reshape': # [b, 8] => [b, 2, 2, 2] x = x.view(x.size(0), *param) elif name == 'relu': x = F.relu(x, inplace=param[0]) elif name == 'leakyrelu': x = F.leaky_relu(x, negative_slope=param[0], inplace=param[1]) elif name == 'tanh': x = F.tanh(x) elif name == 'sigmoid': x = torch.sigmoid(x) elif name == 'upsample': x = F.upsample_nearest(x, scale_factor=param[0]) elif name == 'dropout': x = F.dropout(x, p=param[0], training=dropout_training) elif name == 'max_pool2d': x = F.max_pool2d(x, param[0], param[1], param[2]) elif name == 'avg_pool2d': x = F.avg_pool2d(x, param[0], param[1], param[2]) else: raise NotImplementedError # make sure variable is used properly #assert idx == len(vars) #assert bn_idx == len(self.vars_bn) return x
def forward(self, G): h_dict = self.layer1(G, self.embed) h_dict = {k : F.leaky_relu(h) for k, h in h_dict.items()} h_dict = self.layer2(G, h_dict) return h_dict
def forward(self, x): out = F.leaky_relu(self.conv1(x), 0.05) # (?, 64, 16, 16) out = F.leaky_relu(self.conv2(out), 0.05) # (?, 128, 8, 8) out = F.leaky_relu(self.conv3(out), 0.05) # (?, 256, 4, 4) return self.fc(out).squeeze()