return FAN(fname_pretrained="./wing.ckpt") if __name__ == '__main__': from paddorch.convert_pretrain_model import load_pytorch_pretrain_model import torch as pytorch import torchvision # place = fluid.CPUPlace() place = fluid.CUDAPlace(0) np.random.seed(0) x=np.random.randn(1,3,256,256).astype("float32") with fluid.dygraph.guard(place=place): model=FAN() model.eval() pytorch_model=eval_pytorch_model() pytorch_model.eval() pytorch_model. torch_output = pytorch_model(pytorch.FloatTensor(x).)[1][0] pytorch_model pytorch_state_dict=pytorch_model.state_dict() load_pytorch_pretrain_model(model, pytorch_state_dict) torch.save(model.state_dict(),"wing") paddle_output = model(torch.Tensor(x))[1][0] print("torch mean",torch_output.mean()) print("paddle mean", torch.mean(paddle_output).numpy())
mu, sigma = torch.mean(pool, 0), torch_cov(pool, rowvar=False) else: mu, sigma = np.mean(pool.numpy(), axis=0), np.cov(pool.numpy(), rowvar=False) if prints: print('Covariances calculated, getting FID...') if use_torch: FID = torch_calculate_frechet_distance( mu, sigma, torch.tensor(data_mu), torch.tensor(data_sigma)) FID = float(FID.numpy()) else: FID = numpy_calculate_frechet_distance(mu, sigma, data_mu, data_sigma) # Delete mu, sigma, pool, logits, and labels, just in case del mu, sigma, pool, logits, labels return IS_mean, IS_std, FID return get_inception_metrics if __name__ == '__main__': from paddle import fluid place = fluid.CUDAPlace(0) with fluid.dygraph.guard(place=place): inception_model = inception_v3() import torch as pytorch torch_state_dict = pytorch.load("inception_model.pth") from paddorch.convert_pretrain_model import load_pytorch_pretrain_model load_pytorch_pretrain_model(inception_model, torch_state_dict) torch.save(inception_model.state_dict(), "inception_model.pdparams")
def main(args): dgl.random.seed(args.seed) np.random.seed(args.seed) torch.manual_seed(args.seed) if args.gpu >= 0: torch.cuda.manual_seed(args.seed) if args.resume: if os.path.isfile(args.resume): print("=> loading checkpoint '{}'".format(args.resume)) checkpoint = torch.load(args.resume, map_location="cpu") pretrain_args = checkpoint["opt"] pretrain_args.fold_idx = args.fold_idx pretrain_args.gpu = args.gpu pretrain_args.finetune = args.finetune pretrain_args.resume = args.resume pretrain_args.cv = args.cv pretrain_args.dataset = args.dataset pretrain_args.epochs = args.epochs pretrain_args.num_workers = args.num_workers if args.dataset in GRAPH_CLASSIFICATION_DSETS: # HACK for speeding up finetuning on graph classification tasks pretrain_args.num_workers = 0 pretrain_args.batch_size = args.batch_size args = pretrain_args else: print("=> no checkpoint found at '{}'".format(args.resume)) args = option_update(args) print(args) if args.gpu >= 0: assert args.gpu is not None and torch.cuda.is_available() print("Use GPU: {} for training".format(args.gpu)) assert args.positional_embedding_size % 2 == 0 print("setting random seeds") mem = psutil.virtual_memory() print("before construct dataset", mem.used / 1024**3) if args.finetune: if args.dataset in GRAPH_CLASSIFICATION_DSETS: dataset = GraphClassificationDatasetLabeled( dataset=args.dataset, rw_hops=args.rw_hops, subgraph_size=args.subgraph_size, restart_prob=args.restart_prob, positional_embedding_size=args.positional_embedding_size, ) labels = dataset.dataset.data.y.tolist() else: dataset = NodeClassificationDatasetLabeled( dataset=args.dataset, rw_hops=args.rw_hops, subgraph_size=args.subgraph_size, restart_prob=args.restart_prob, positional_embedding_size=args.positional_embedding_size, ) labels = dataset.data.y.argmax(dim=1).tolist() skf = StratifiedKFold(n_splits=10, shuffle=True, random_state=args.seed) idx_list = [] for idx in skf.split(np.zeros(len(labels)), labels): idx_list.append(idx) assert (0 <= args.fold_idx and args.fold_idx < 10), "fold_idx must be from 0 to 9." train_idx, test_idx = idx_list[args.fold_idx] train_dataset = torch.utils.data.Subset(dataset, train_idx) valid_dataset = torch.utils.data.Subset(dataset, test_idx) elif args.dataset == "dgl": train_dataset = LoadBalanceGraphDataset( rw_hops=args.rw_hops, restart_prob=args.restart_prob, positional_embedding_size=args.positional_embedding_size, num_workers=args.num_workers, num_samples=args.num_samples, dgl_graphs_file="./data/small.bin", num_copies=args.num_copies, ) else: if args.dataset in GRAPH_CLASSIFICATION_DSETS: train_dataset = GraphClassificationDataset( dataset=args.dataset, rw_hops=args.rw_hops, subgraph_size=args.subgraph_size, restart_prob=args.restart_prob, positional_embedding_size=args.positional_embedding_size, ) else: train_dataset = NodeClassificationDataset( dataset=args.dataset, rw_hops=args.rw_hops, subgraph_size=args.subgraph_size, restart_prob=args.restart_prob, positional_embedding_size=args.positional_embedding_size, ) mem = psutil.virtual_memory() print("before construct dataloader", mem.used / 1024**3) train_loader = torch.utils.data.graph.Dataloader( dataset=train_dataset, batch_size=args.batch_size, collate_fn=labeled_batcher() if args.finetune else batcher(), shuffle=True if args.finetune else False, num_workers=args.num_workers, worker_init_fn=None if args.finetune or args.dataset != "dgl" else worker_init_fn, ) if args.finetune: valid_loader = torch.utils.data.DataLoader( dataset=valid_dataset, batch_size=args.batch_size, collate_fn=labeled_batcher(), num_workers=args.num_workers, ) mem = psutil.virtual_memory() print("before training", mem.used / 1024**3) # create model and optimizer # n_data = train_dataset.total n_data = None import gcc.models.graph_encoder gcc.models.graph_encoder.final_dropout = 0 ##disable dropout model, model_ema = [ GraphEncoder( positional_embedding_size=args.positional_embedding_size, max_node_freq=args.max_node_freq, max_edge_freq=args.max_edge_freq, max_degree=args.max_degree, freq_embedding_size=args.freq_embedding_size, degree_embedding_size=args.degree_embedding_size, output_dim=args.hidden_size, node_hidden_dim=args.hidden_size, edge_hidden_dim=args.hidden_size, num_layers=args.num_layer, num_step_set2set=args.set2set_iter, num_layer_set2set=args.set2set_lstm_layer, norm=args.norm, gnn_model=args.model, degree_input=True, ) for _ in range(2) ] # copy weights from `model' to `model_ema' if args.moco: moment_update(model, model_ema, 0) # set the contrast memory and criterion contrast = MemoryMoCo(args.hidden_size, n_data, args.nce_k, args.nce_t, use_softmax=True) if args.gpu >= 0: contrast = contrast.cuda(args.gpu) if args.finetune: criterion = nn.CrossEntropyLoss() else: criterion = NCESoftmaxLoss() if args.moco else NCESoftmaxLossNS() if args.gpu >= 0: criterion = criterion.cuda(args.gpu) if args.gpu >= 0: model = model.cuda(args.gpu) model_ema = model_ema.cuda(args.gpu) if args.finetune: output_layer = nn.Linear(in_features=args.hidden_size, out_features=dataset.num_classes) if args.gpu >= 0: output_layer = output_layer.cuda(args.gpu) output_layer_optimizer = torch.optim.Adam( output_layer.parameters(), lr=args.learning_rate, betas=(args.beta1, args.beta2), weight_decay=args.weight_decay, ) def clear_bn(m): classname = m.__class__.__name__ if classname.find("BatchNorm") != -1: m.reset_running_stats() model.apply(clear_bn) if args.optimizer == "sgd": optimizer = torch.optim.SGD( model.parameters(), lr=args.learning_rate, momentum=args.momentum, weight_decay=args.weight_decay, ) elif args.optimizer == "adam": optimizer = torch.optim.Adam( model.parameters(), lr=args.learning_rate, betas=(args.beta1, args.beta2), weight_decay=args.weight_decay, ) elif args.optimizer == "adagrad": optimizer = torch.optim.Adagrad( model.parameters(), lr=args.learning_rate, lr_decay=args.lr_decay_rate, weight_decay=args.weight_decay, ) else: raise NotImplementedError # optionally resume from a checkpoint args.start_epoch = 1 if True: # print("=> loading checkpoint '{}'".format(args.resume)) # checkpoint = torch.load(args.resume, map_location="cpu") import torch as th checkpoint = th.load("torch_models/ckpt_epoch_100.pth", map_location=th.device('cpu')) torch_input_output_grad = th.load( "torch_models/torch_input_output_grad.pt", map_location=th.device('cpu')) from paddorch.convert_pretrain_model import load_pytorch_pretrain_model print("loading.............. model") paddle_state_dict = load_pytorch_pretrain_model( model, checkpoint["model"]) model.load_state_dict(paddle_state_dict) print("loading.............. contrast") paddle_state_dict2 = load_pytorch_pretrain_model( contrast, checkpoint["contrast"]) contrast.load_state_dict(paddle_state_dict2) print("loading.............. model_ema") paddle_state_dict3 = load_pytorch_pretrain_model( model_ema, checkpoint["model_ema"]) if args.moco: model_ema.load_state_dict(paddle_state_dict3) print("=> loaded successfully '{}' (epoch {})".format( args.resume, checkpoint["epoch"])) del checkpoint if args.gpu >= 0: torch.cuda.empty_cache() optimizer = torch.optim.Adam( model.parameters(), lr=args.learning_rate * 0.1, betas=(args.beta1, args.beta2), weight_decay=args.weight_decay, ) for _ in range(1): graph_q, graph_k = train_dataset[0] graph_q2, graph_k2 = train_dataset[1] graph_q, graph_k = dgl.batch([graph_q, graph_q2 ]), dgl.batch([graph_k, graph_k2]) input_output_grad = [] input_output_grad.append([graph_q, graph_k]) model.train() model_ema.eval() feat_q = model(graph_q) with torch.no_grad(): feat_k = model_ema(graph_k) out = contrast(feat_q, feat_k) loss = criterion(out) optimizer.zero_grad() loss.backward() input_output_grad.append([feat_q, out, loss]) print("loss:", loss.numpy()) optimizer.step() moment_update(model, model_ema, args.alpha) print( "max diff feat_q:", np.max( np.abs(torch_input_output_grad[1][0].detach().numpy() - feat_q.numpy()))) print( "max diff out:", np.max( np.abs(torch_input_output_grad[1][1].detach().numpy() - out.numpy()))) print( "max diff loss:", np.max( np.abs(torch_input_output_grad[1][2].detach().numpy() - loss.numpy()))) name2grad = dict() for name, p in dict(model.named_parameters()).items(): if p.grad is not None: name2grad[name] = p.grad torch_grad = torch_input_output_grad[2][name].numpy() if "linear" in name and "weight" in name: torch_grad = torch_grad.T max_grad_diff = np.max(np.abs(p.grad - torch_grad)) print("max grad diff:", name, max_grad_diff) input_output_grad.append(name2grad)
if "optim" in torch_fn: continue # skip optimizer file if "best3" not in torch_fn: continue import torch as pytorch torch_state_dict = pytorch.load(torch_fn) # Next, build the model print(torch_fn) out_fn = torch_fn.replace(".pth", ".pdparams") if os.path.basename(torch_fn).startswith("G"): G = model.Generator(**config) torch_G = get_pytorch_G_model(config) torch_G.load_state_dict(torch_state_dict) load_pytorch_pretrain_model(G, torch_state_dict) z = np.zeros((1, 128)) y = np.ones((1, 1)) import torch as pytorch torch_X = torch_G(pytorch.Tensor(z), pytorch.LongTensor(y)) X = G( paddorch.Tensor(z).astype("float32"), paddorch.Tensor(y).astype("int64")) print(torch_X.detach().numpy().mean(), X.detach().numpy().mean()) print("saved file:", out_fn) paddorch.save(G.state_dict(), out_fn) elif os.path.basename(torch_fn).startswith("D"): D = model.Discriminator(**config) load_pytorch_pretrain_model(D, torch_state_dict) paddorch.save(D.state_dict(), out_fn)
def build_model(args): import torch as pytorch from paddorch.convert_pretrain_model import load_pytorch_pretrain_model pytorch_state_dict = pytorch.load("../../../starganv2_paddle/expr/checkpoints/celeba_hq/100000_nets_ema.pt", map_location=pytorch.device('cpu')) generator, mapping_network, style_encoder = eval_pytorch_model(args) x=np.ones( (1,3,256,256)).astype("float32") batch_size=2000 batch_size2=3 x= np.random.randn(batch_size2,3,256,256).astype("float32") y = np.random.randint(0,2, batch_size2).astype("int32") s = np.random.randn(args.style_dim*batch_size2).astype("float32").reshape(batch_size2,-1) z = np.random.randn(16*batch_size2).astype("float32").reshape(batch_size2, -1) generator.load_state_dict(pytorch_state_dict['generator']) g_out=generator(pytorch.FloatTensor(x), pytorch.FloatTensor(s)) mapping_network.load_state_dict(pytorch_state_dict['mapping_network']) m_out=mapping_network(pytorch.FloatTensor(z), pytorch.LongTensor(y)) y_train = np.random.randint(0,2, batch_size).astype("int32") z_train = np.random.randn(16*batch_size).astype("float32").reshape(batch_size, -1) m_out_train,m_out_train_1,m_out_train_2 = mapping_network.finetune(pytorch.FloatTensor(z_train), pytorch.LongTensor(y_train)) style_encoder.load_state_dict(pytorch_state_dict['style_encoder']) s_out=style_encoder(pytorch.FloatTensor(x), pytorch.LongTensor(y) ) import paddle.fluid as fluid import paddorch as torch import paddorch.nn.functional as F # place = fluid.CPUPlace() place = fluid.CUDAPlace(0) with fluid.dygraph.guard(place=place): import sys sys.path.append("../../../starganv2_paddle") import core.model generator_ema = core.model.Generator(args.img_size, args.style_dim, w_hpf=args.w_hpf) #copy.deepcopy(generator) mapping_network_ema =core.model.MappingNetwork(args.latent_dim, args.style_dim, args.num_domains) # copy.deepcopy(mapping_network) style_encoder_ema =core.model.StyleEncoder(args.img_size, args.style_dim, args.num_domains) # copy.deepcopy(style_encoder) nets = Munch(generator=generator, mapping_network=mapping_network, style_encoder=style_encoder, ) print("load generator") load_pytorch_pretrain_model(generator_ema, pytorch_state_dict['generator']) print("load mapping_network") load_pytorch_pretrain_model(mapping_network_ema, pytorch_state_dict['mapping_network']) print("load style_encoder") load_pytorch_pretrain_model(style_encoder_ema, pytorch_state_dict['style_encoder']) nets_ema = Munch(generator=generator_ema, mapping_network=mapping_network_ema, style_encoder=style_encoder_ema) nets_ema_state_dict=dict() # nets_ema['mapping_network'].load_state_dict( # torch.load("../../expr/checkpoints/celeba_hq/100000_nets_ema.ckpt/mapping_network.pdparams")) nets_ema['mapping_network'].train() d_optimizer = fluid.optimizer.AdamOptimizer(learning_rate=1e-4, parameter_list=nets_ema['mapping_network'].parameters()) from tqdm import tqdm z_train_p = torch.Tensor(z_train) y_train_p = torch.LongTensor(y_train) m_out_train_p = torch.Tensor(m_out_train.detach().numpy()) m_out_train_1p = torch.Tensor(m_out_train_1.detach().numpy()) m_out_train_2p = torch.Tensor(m_out_train_2.detach().numpy()) # dummy_net=fluid.dygraph.Linear(16,1) # d_optimizer = fluid.optimizer.AdamOptimizer(learning_rate=1e-4, # parameter_list=dummy_net.parameters()) for ii in range(10000): # predictions=dummy_net(z_train_p ) # predictions2=torch.varbase_to_tensor(predictions) # d_avg_cost = fluid.layers.mse_loss(predictions2, m_out_train_p) # fluid.layers.assign(predictions,predictions2) out, out1, out2 = nets_ema['mapping_network'].finetune(z_train_p, y_train_p) d_avg_cost = fluid.layers.mse_loss(out, m_out_train_p) + fluid.layers.mse_loss(out1, m_out_train_1p) + fluid.layers.mse_loss( out2, m_out_train_2p) d_avg_cost.backward() d_optimizer.minimize(d_avg_cost) nets_ema['mapping_network'].clear_gradients() if ii % 99 == 0: print("d_avg_cost", d_avg_cost.numpy()) nets_ema['mapping_network'].eval() m_out_t = nets_ema['mapping_network'](torch.Tensor(z), torch.LongTensor(y)).numpy() print(ii, "torch m result:", m_out.detach().numpy().mean(), m_out_t.mean()) nets_ema['mapping_network'].train() y_train = np.random.randint(0, 2, batch_size).astype("int32") z_train = np.random.randn(16 * batch_size).astype("float32").reshape(batch_size, -1) m_out_train, m_out_train_1, m_out_train_2 = mapping_network.finetune(pytorch.FloatTensor(z_train), pytorch.LongTensor(y_train)) z_train_p = torch.Tensor(z_train) y_train_p = torch.LongTensor(y_train) m_out_train_p = torch.Tensor(m_out_train.detach().numpy()) if np.abs(m_out.detach().numpy().mean() - m_out_t.mean()) < 1e-5: break nets_ema['mapping_network'].eval() g_out_t = nets_ema['generator'](torch.Tensor(x), torch.Tensor(s)).numpy() m_out_t = nets_ema['mapping_network'](torch.Tensor(z), torch.LongTensor(y)).numpy() s_out_t = nets_ema['style_encoder'](torch.Tensor(x), torch.LongTensor(y)).numpy() print("torch g result:", g_out.mean().detach().numpy(), g_out_t.mean()) print("torch s result:", s_out.mean().detach().numpy(), s_out_t.mean()) print("torch m result:", m_out.mean().detach().numpy(), m_out_t.mean()) nets_ema_state_dict['generator'] = generator_ema.state_dict() nets_ema_state_dict['mapping_network'] = mapping_network_ema.state_dict() nets_ema_state_dict['style_encoder'] = style_encoder_ema.state_dict() torch.save(nets_ema_state_dict, "../../../starganv2_paddle/expr/checkpoints/celeba_hq/100000_nets_ema.ckpt") if args.w_hpf > 0: fan = FAN(fname_pretrained="../../../starganv2_paddle/expr/checkpoints/wing.pdparams").eval() nets.fan = fan nets_ema.fan = fan return nets, nets_ema