def predict(): device = "cuda:0" if torch.cuda.is_available() else "cpu" df = pd.read_csv(config.TEST_PATH, header=None) dataset = PicDataset(df.loc[:, 1:]) preds = np.zeros((len(dataset), 256)) for i in range(5): temp = np.zeros((len(dataset), 256)) model = DNN() model.load_state_dict(torch.load(f'./models/model_{i}.bin')) model.to(device) model.eval() for j in range(len(dataset)): x, _ = dataset[j] x = x.to(device) y = model(x) temp[j, :] = y.detach().cpu().numpy() preds += temp preds /= 5 df = pd.DataFrame(np.concatenate([np.arange(1, 921).reshape(-1, 1), preds], axis=1), columns=np.arange(257)) df[0] = df[0].astype('int') df.to_csv('./predictions.csv', index=False, header=False)
def run(): df = pd.read_csv(config.TRAIN_PATH) kfold = KFold(n_splits=5, random_state=config.SEED, shuffle=True) fold_losses = [] for i, (train_idx, val_idx) in enumerate(kfold.split(df)): print("-------------------------------------------------------") print(f"Training fold {i}") print("-------------------------------------------------------") train = df.iloc[train_idx] validation = df.iloc[val_idx] train_dataset = PicDataset(train) train_data_loader = torch.utils.data.DataLoader( train_dataset, batch_size=config.BATCH_SIZE ) val_dataset = PicDataset(validation) val_data_loader = torch.utils.data.DataLoader( val_dataset, batch_size=config.BATCH_SIZE ) device = 'cuda:0' if torch.cuda.is_available() else "cpu" model = DNN() model.to(device) optimizer = torch.optim.Adam(model.parameters(), lr=config.LR) loss = 0 for _ in range(config.EPOCHS): engine.train_fn(train_data_loader, model, optimizer, device) loss = engine.eval_fn(val_data_loader, model, device) print(f"Loss on fold {i} is {loss}") fold_losses.append(loss) torch.save(model.state_dict(), f'./models/model_{i}.bin') print(f"Average loss on cross validation is {sum(fold_losses) / 5}")
if args['weighted']: weight = corpus.load_json( os.path.join(source_path, 'cross_entropy_loss_weight.json')) weight = torch.FloatTensor(list(weight.values())).to(device) data = sorted(corpus.examples.get('seq'), key=lambda x: len(x), reverse=True) vocab_size = len(corpus.words2id) logging.info('vocabulary size: {}'.format(vocab_size)) model = DNN(vocab_size=vocab_size, embedding_size=200, hidden_size=512, embedding=embedding) model.to(device) loss_function = nn.CrossEntropyLoss(weight=weight) optimizer = torch.optim.SGD(model.parameters(), lr=0.01, momentum=0.9) # optimizer = torch.optim.Adam(model.parameters(), lr=0.001) model.train() total_data = len(data) batch_size = args['batch_size'] total_step = math.ceil(total_data / batch_size) last_training_loss = 1000000000000 for epoch in range(args.get('epoch')): start = 0 training_loss = 0
# create folder for generated data gen_data_path = os.path.join(out_path, gen_data_fdr) if not os.path.exists(gen_data_path): os.makedirs(gen_data_path) if not os.path.exists(scaler_dir): os.makedirs(scaler_dir) print('here') # create folder for model checkpoints checkpoint_path = os.path.join(out_path, checkpoint_fdr) if not os.path.exists(checkpoint_path): os.makedirs(checkpoint_path) #Your statements here model = DNN(input_size, hidden_size, out_size) model = torch.nn.DataParallel(model.to(device), device_ids=use_devices) #print(model) #save_dir = "FPGA" checkpoint = torch.load( './segan_data_out/20200422_0713/checkpoints/state-20.pkl') #from collections import OrderedDict state_dict = checkpoint['DNN'] #for k, v in state_dict.items(): # name = k[7:] # remove `module.` # v = v.cpu().numpy() # np.savetxt(os.path.join(save_dir,name), v, newline="\n") # print(name, v.shape) scaler_path_input = os.path.join(scaler_dir, "scaler_input.p") scaler_input = pickle.load(open(scaler_path_input, 'rb')) scaler_path_label = os.path.join(scaler_dir, "scaler_label.p") scaler_label = pickle.load(open(scaler_path_label, 'rb'))
def main(): print('> Starting execution...') # Training settings parser = argparse.ArgumentParser(description='PyTorch MNIST Example') group = parser.add_mutually_exclusive_group() group.add_argument('--fit', action='store_true', help='fit the tuned model on digits 0-4') group.add_argument('--transfer', action='store_true', help='train a pretrained model on digits 5-9') parser.add_argument('--batch-size', type=int, default=256, metavar='N', help='input batch size for training (default: 256)') parser.add_argument('--epochs', type=int, default=50, metavar='E', help='number of epochs to train (default: 50)') parser.add_argument('--lr', type=float, default=1e-3, metavar='L', help='learning rate (default: 1e-3)') parser.add_argument('--early-stopping', type=int, default=7, metavar='E', help='early stopping (default: 7 epochs)') parser.add_argument( '--size', type=int, default=100, metavar='S', help='size of the training data for transfer learning (default: 100)') parser.add_argument('--seed', type=int, default=23, metavar='S', help='random seed (default: 23)') args = parser.parse_args() use_cuda = torch.cuda.is_available() # use cuda if available device = torch.device("cuda" if use_cuda else "cpu") torch.manual_seed(args.seed) # random seed print('> Loading MNIST data') train_set = datasets.MNIST(MNIST_DATA_DIR, train=True, download=True, transform=transforms.Compose([ transforms.ToTensor(), transforms.Normalize((0.1307, ), (0.3081, )) ])) test_set = datasets.MNIST(MNIST_DATA_DIR, train=False, download=True, transform=transforms.Compose([ transforms.ToTensor(), transforms.Normalize((0.1307, ), (0.3081, )) ])) train_digits_04 = np.where(train_set.train_labels < 5)[0] train_digits_59 = np.where(train_set.train_labels > 4)[0] test_digits_04 = np.where(test_set.test_labels < 5)[0] test_digits_59 = np.where(test_set.test_labels > 4)[0] if args.fit: # Training the tuned model on digits 0-4 print('> Training a new model on MNIST digits 0-4') X_train_04, y_train_04, X_valid_04, y_valid_04 = data_to_numpy( train_set, test_set, INPUT_DIM, train_digits_04, test_digits_04) torch.manual_seed(args.seed) print('> Initializing the model') model = DNN(INPUT_DIM, OUTPUT_DIM, HIDDEN_DIM, batch_norm=True) model.apply(init_he_normal) # He initialization model = model.to(device) criterion = nn.CrossEntropyLoss() optimizer = optim.Adam(model.parameters(), lr=args.lr) print('> Training the model') model, _, _ = train_model(model, device, X_train_04, y_train_04, criterion, optimizer, X_valid=X_valid_04, y_valid=y_valid_04, batch_size=args.batch_size, n_epochs=args.epochs, early_stopping=args.early_stopping) print(f'> Saving the model state at {MODEL_04_PATH}') torch.save(model.state_dict(), MODEL_04_PATH) elif args.transfer: # Transfer learning print( '> Training a model on MNIST digits 5-9 from a pretrained model for digits 0-4' ) if os.path.isfile(MODEL_04_PATH): print('> Loading the pretrained model') model = DNN(INPUT_DIM, OUTPUT_DIM, HIDDEN_DIM, batch_norm=True).to(device) model.load_state_dict(torch.load(MODEL_04_PATH)) for param in model.parameters(): param.requires_grad = False # Parameters of newly constructed modules have requires_grad=True by default model.fc4 = nn.Linear(HIDDEN_DIM, HIDDEN_DIM) model.fc5 = nn.Linear(HIDDEN_DIM, HIDDEN_DIM) model.out = nn.Linear(HIDDEN_DIM, OUTPUT_DIM) print('> Using saved model state') else: print( '> Model state file is not found, fit a model before the transfer learning' ) print('> Stopping execution') return X_train_59, y_train_59, X_valid_59, y_valid_59 = data_to_numpy( train_set, test_set, INPUT_DIM, train_digits_59[:args.size], test_digits_59) # fixing the issues with labels y_train_59 = y_train_59 - 5 y_valid_59 = y_valid_59 - 5 criterion = nn.CrossEntropyLoss() optimizer = optim.Adam(model.parameters(), lr=args.lr) print('> Training the model') model, _, _ = train_model(model, device, X_train_59, y_train_59, criterion, optimizer, X_valid=X_valid_59, y_valid=y_valid_59, batch_size=args.batch_size, n_epochs=args.epochs, early_stopping=args.early_stopping) print(f'> Saving the model state at {MODEL_59_PATH}') torch.save(model.state_dict(), MODEL_59_PATH) else: print('> Incorrect mode, try either `--fit` or `--transfer`') print('> Stopping execution')