def predict(): device = "cuda:0" if torch.cuda.is_available() else "cpu" df = pd.read_csv(config.TEST_PATH, header=None) dataset = PicDataset(df.loc[:, 1:]) preds = np.zeros((len(dataset), 256)) for i in range(5): temp = np.zeros((len(dataset), 256)) model = DNN() model.load_state_dict(torch.load(f'./models/model_{i}.bin')) model.to(device) model.eval() for j in range(len(dataset)): x, _ = dataset[j] x = x.to(device) y = model(x) temp[j, :] = y.detach().cpu().numpy() preds += temp preds /= 5 df = pd.DataFrame(np.concatenate([np.arange(1, 921).reshape(-1, 1), preds], axis=1), columns=np.arange(257)) df[0] = df[0].astype('int') df.to_csv('./predictions.csv', index=False, header=False)
def train(args, config, io): train_loader, validation_loader = get_loader(args, config) device = torch.device("cuda" if args.cuda else "cpu") # print(len(train_loader), len(validation_loader)) #Try to load models model = DNN(args).to(device) """if device == torch.device("cuda"): model = nn.DataParallel(model)""" if args.model_path != "": model.load_state_dict(torch.load(args.model_path)) # for para in list(model.parameters())[:-5]: # para.requires_grad=False # print(model) if args.use_sgd: # print("Use SGD") opt = optim.SGD(model.parameters(), lr=args.lr * 100, momentum=args.momentum, weight_decay=1e-4) else: # print("Use Adam") opt = optim.Adam(model.parameters(), lr=args.lr, weight_decay=1e-4) """opt = optim.Adam([ {'params': list(model.parameters())[:-1], 'lr':args.lr/50, 'weight_decay': 1e-4}, {'params': list(model.parameters())[-1], 'lr':args.lr, 'weight_decay': 1e-4} ]) """ scheduler = CosineAnnealingLR(opt, args.epochs, eta_min=args.lr) criterion = nn.MSELoss() best_test_loss = 9999999. for epoch in range(args.epochs): startTime = time.time() #################### # Train #################### train_loss = 0.0 train_dis = 0.0 count = 0.0 model.train() for data, label in train_loader: data, label = data.to(device), label.to(device) data = drop(jitter(data, device), device) # data = jitter(data, device, delta=0.05) batch_size = data.shape[0] logits = model(data) loss = criterion(logits, label) opt.zero_grad() loss.backward() opt.step() dis = distance(logits, label) count += batch_size train_loss += loss.item() * batch_size train_dis += dis.item() * batch_size scheduler.step() outstr = 'Train %d, loss: %.6f, distance: %.6f' % ( epoch, train_loss * 1.0 / count, train_dis * 1.0 / count) io.cprint(outstr) #################### # Evaluation #################### test_loss = 0.0 test_dis = 0.0 count = 0.0 model.eval() with torch.no_grad(): for data, label in validation_loader: data, label = data.to(device), label.to(device) batch_size = data.shape[0] logits = model(data) loss = criterion(logits, label) dis = distance(logits, label) count += batch_size test_loss += loss.item() * batch_size test_dis += dis.item() * batch_size outstr = 'Test %d, loss: %.6f, distance: %.6f' % ( epoch, test_loss * 1.0 / count, test_dis * 1.0 / count) io.cprint(outstr) if test_loss <= best_test_loss: best_test_loss = test_loss torch.save(model.state_dict(), 'checkpoints/%s/models/model.t7' % args.exp_name) torch.save(model, (config.root + config.model_path)) io.cprint('Time: %.3f sec' % (time.time() - startTime))
model = DNN(x.size(1)).to(device) total_param = sum(p.numel() for p in model.parameters()) trainable_param = sum(p.numel() for p in model.parameters() if p.requires_grad) print("{} parameters with {} trainable".format(total_param, trainable_param), flush=True) print("\nStart training...", flush=True) train_dataset1 = TwitterDataset(x, train_y1) train_loader1 = torch.utils.data.DataLoader(dataset=train_dataset1, batch_size=BATCH, shuffle=True, num_workers=4) train_model(train_loader1, model, device, LR) print("\nStart testing...", flush=True) test_x = [ "today is a good day , but it is hot", "today is hot , but it is a good day" ] test_x = [i.split() for i in test_x] x = torch.zeros(2, len(word_index)) for i in range(2): for w in test_x[i]: x[i][word_index[w]] += 1 model.load_state_dict(torch.load("model_bow.pkl")) model.eval() x = x.to(device, dtype=torch.float) print(model(x).cpu())
'./segan_data_out/20200422_0713/checkpoints/state-20.pkl') #from collections import OrderedDict state_dict = checkpoint['DNN'] #for k, v in state_dict.items(): # name = k[7:] # remove `module.` # v = v.cpu().numpy() # np.savetxt(os.path.join(save_dir,name), v, newline="\n") # print(name, v.shape) scaler_path_input = os.path.join(scaler_dir, "scaler_input.p") scaler_input = pickle.load(open(scaler_path_input, 'rb')) scaler_path_label = os.path.join(scaler_dir, "scaler_label.p") scaler_label = pickle.load(open(scaler_path_label, 'rb')) test_audio_path = './dr1_fdac1_sx214.wav' mode = 'magnitude' n_concat = 11 model.load_state_dict(state_dict) #if torch.cuda.is_available(): # model.cuda() test_audio, _ = read_audio(test_audio_path) test_spec = calc_sp(test_audio, mode) mixed_complx_x = calc_sp(test_audio, mode='complex') plt.matshow(test_spec.T) plt.savefig("noisy.png") n_window = 512 n_overlap = 256 ham_win = np.hamming(n_window) #print(test_spec) test_spec = log_sp(test_spec) n_pad = (n_concat - 1) // 2 test_spec_padding = pad_with_border(test_spec, n_pad) test_spec_padding_concated = concate_seq(test_spec_padding, n_pad)
import os import torch import time import torch.nn.functional as F from args import args from model import DNN from utils import cos_sim from processing import BuildExamples # 加载模型 MODEL_PATH = os.path.join(args.get('data_path'), 'model/embedded_adam_0.001_19.model') print(MODEL_PATH) model = DNN(vocab_size=5265, embedding_size=200, hidden_size=512) model.load_state_dict(torch.load(MODEL_PATH, map_location=torch.device('cpu'))) # 加载字典 corpus = BuildExamples() corpus.load_vocabulary(path=os.path.join(args.get('data_path'), 'vocabulary.txt')) MAX_LEN = args.get('max_len', 20) def get_hidden_state(sentence): line = sentence.strip().split() line = ['bos'] + line + ['eos'] sentence2id = torch.LongTensor([corpus.words2id.get(item, 0) for item in line]).view(1, -1) with torch.no_grad(): last_hidden_state, _ = model(sentence2id)
def train(args, config, io): train_loader, validation_loader, unlabelled_loader = get_loader( args, config) device = torch.device("cuda" if args.cuda else "cpu") #Try to load models model = DNN(args).to(device) ema_model = DNN(args).to(device) for param in ema_model.parameters(): param.detach_() if device == torch.device("cuda"): model = nn.DataParallel(model) ema_model = nn.DataParallel(ema_model) if args.model_path != "": model.load_state_dict(torch.load(args.model_path)) ema_model.load_state_dict(torch.load(args.model_path)) if args.use_sgd: print("Use SGD") opt = optim.SGD(model.parameters(), lr=args.lr * 100, momentum=args.momentum, weight_decay=1e-4) else: print("Use Adam") opt = optim.Adam(model.parameters(), lr=args.lr, weight_decay=1e-4) scheduler = CosineAnnealingLR(opt, args.epochs, eta_min=args.lr) criterion = nn.MSELoss() consistency_criterion = nn.MSELoss() best_test_loss = 9999999. global_step = 0 for epoch in range(args.epochs): startTime = time.time() #################### # Train #################### train_loss = 0.0 count = 0.0 model.train() ema_model.train() i = -1 for (data, label), (u, _) in zip(cycle(train_loader), unlabelled_loader): i = i + 1 if data.shape[0] != u.shape[0]: bt_size = np.minimum(data.shape[0], u.shape[0]) data = data[0:bt_size] label = label[0:bt_size] u = u[0:bt_size] data, label, u = data.to(device), label.to(device), u.to(device) batch_size = data.shape[0] logits = model(data) class_loss = criterion(logits, label) u_student = jitter(u, device) u_teacher = jitter(u, device) logits_unlabeled = model(u_student) ema_logits_unlabeled = ema_model(u_teacher) ema_logits_unlabeled = Variable(ema_logits_unlabeled.detach().data, requires_grad=False) consistency_loss = consistency_criterion(logits_unlabeled, ema_logits_unlabeled) if epoch < args.consistency_rampup_starts: consistency_weight = 0.0 else: consistency_weight = get_current_consistency_weight( args, args.final_consistency, epoch, i, len(unlabelled_loader)) consistency_loss = consistency_weight * consistency_loss loss = class_loss + consistency_loss opt.zero_grad() loss.backward() opt.step() global_step += 1 # print(global_step) update_ema_variables(model, ema_model, args.ema_decay, global_step) count += batch_size train_loss += loss.item() * batch_size scheduler.step() outstr = 'Train %d, loss: %.6f' % (epoch, train_loss * 1.0 / count) io.cprint(outstr) #################### # Evaluation #################### test_loss = 0.0 count = 0.0 model.eval() ema_model.eval() for data, label in validation_loader: data, label = data.to(device), label.to(device) batch_size = data.shape[0] logits = ema_model(data) loss = criterion(logits, label) count += batch_size test_loss += loss.item() * batch_size outstr = 'Test %d, loss: %.6f' % (epoch, test_loss * 1.0 / count) io.cprint(outstr) if test_loss <= best_test_loss: best_test_loss = test_loss torch.save(ema_model.state_dict(), 'checkpoints/%s/models/model.t7' % args.exp_name) torch.save(ema_model, (config.root + config.model_path)) io.cprint('Time: %.3f sec' % (time.time() - startTime))
def main(): print('> Starting execution...') # Training settings parser = argparse.ArgumentParser(description='PyTorch MNIST Example') group = parser.add_mutually_exclusive_group() group.add_argument('--fit', action='store_true', help='fit the tuned model on digits 0-4') group.add_argument('--transfer', action='store_true', help='train a pretrained model on digits 5-9') parser.add_argument('--batch-size', type=int, default=256, metavar='N', help='input batch size for training (default: 256)') parser.add_argument('--epochs', type=int, default=50, metavar='E', help='number of epochs to train (default: 50)') parser.add_argument('--lr', type=float, default=1e-3, metavar='L', help='learning rate (default: 1e-3)') parser.add_argument('--early-stopping', type=int, default=7, metavar='E', help='early stopping (default: 7 epochs)') parser.add_argument( '--size', type=int, default=100, metavar='S', help='size of the training data for transfer learning (default: 100)') parser.add_argument('--seed', type=int, default=23, metavar='S', help='random seed (default: 23)') args = parser.parse_args() use_cuda = torch.cuda.is_available() # use cuda if available device = torch.device("cuda" if use_cuda else "cpu") torch.manual_seed(args.seed) # random seed print('> Loading MNIST data') train_set = datasets.MNIST(MNIST_DATA_DIR, train=True, download=True, transform=transforms.Compose([ transforms.ToTensor(), transforms.Normalize((0.1307, ), (0.3081, )) ])) test_set = datasets.MNIST(MNIST_DATA_DIR, train=False, download=True, transform=transforms.Compose([ transforms.ToTensor(), transforms.Normalize((0.1307, ), (0.3081, )) ])) train_digits_04 = np.where(train_set.train_labels < 5)[0] train_digits_59 = np.where(train_set.train_labels > 4)[0] test_digits_04 = np.where(test_set.test_labels < 5)[0] test_digits_59 = np.where(test_set.test_labels > 4)[0] if args.fit: # Training the tuned model on digits 0-4 print('> Training a new model on MNIST digits 0-4') X_train_04, y_train_04, X_valid_04, y_valid_04 = data_to_numpy( train_set, test_set, INPUT_DIM, train_digits_04, test_digits_04) torch.manual_seed(args.seed) print('> Initializing the model') model = DNN(INPUT_DIM, OUTPUT_DIM, HIDDEN_DIM, batch_norm=True) model.apply(init_he_normal) # He initialization model = model.to(device) criterion = nn.CrossEntropyLoss() optimizer = optim.Adam(model.parameters(), lr=args.lr) print('> Training the model') model, _, _ = train_model(model, device, X_train_04, y_train_04, criterion, optimizer, X_valid=X_valid_04, y_valid=y_valid_04, batch_size=args.batch_size, n_epochs=args.epochs, early_stopping=args.early_stopping) print(f'> Saving the model state at {MODEL_04_PATH}') torch.save(model.state_dict(), MODEL_04_PATH) elif args.transfer: # Transfer learning print( '> Training a model on MNIST digits 5-9 from a pretrained model for digits 0-4' ) if os.path.isfile(MODEL_04_PATH): print('> Loading the pretrained model') model = DNN(INPUT_DIM, OUTPUT_DIM, HIDDEN_DIM, batch_norm=True).to(device) model.load_state_dict(torch.load(MODEL_04_PATH)) for param in model.parameters(): param.requires_grad = False # Parameters of newly constructed modules have requires_grad=True by default model.fc4 = nn.Linear(HIDDEN_DIM, HIDDEN_DIM) model.fc5 = nn.Linear(HIDDEN_DIM, HIDDEN_DIM) model.out = nn.Linear(HIDDEN_DIM, OUTPUT_DIM) print('> Using saved model state') else: print( '> Model state file is not found, fit a model before the transfer learning' ) print('> Stopping execution') return X_train_59, y_train_59, X_valid_59, y_valid_59 = data_to_numpy( train_set, test_set, INPUT_DIM, train_digits_59[:args.size], test_digits_59) # fixing the issues with labels y_train_59 = y_train_59 - 5 y_valid_59 = y_valid_59 - 5 criterion = nn.CrossEntropyLoss() optimizer = optim.Adam(model.parameters(), lr=args.lr) print('> Training the model') model, _, _ = train_model(model, device, X_train_59, y_train_59, criterion, optimizer, X_valid=X_valid_59, y_valid=y_valid_59, batch_size=args.batch_size, n_epochs=args.epochs, early_stopping=args.early_stopping) print(f'> Saving the model state at {MODEL_59_PATH}') torch.save(model.state_dict(), MODEL_59_PATH) else: print('> Incorrect mode, try either `--fit` or `--transfer`') print('> Stopping execution')