def predict():
device = "cuda:0" if torch.cuda.is_available() else "cpu"
df = pd.read_csv(config.TEST_PATH, header=None)
dataset = PicDataset(df.loc[:, 1:])
preds = np.zeros((len(dataset), 256))
for i in range(5):
temp = np.zeros((len(dataset), 256))
model = DNN()
model.load_state_dict(torch.load(f'./models/model_{i}.bin'))
model.to(device)
model.eval()
for j in range(len(dataset)):
x, _ = dataset[j]
x = x.to(device)
y = model(x)
temp[j, :] = y.detach().cpu().numpy()
preds += temp
preds /= 5
df = pd.DataFrame(np.concatenate([np.arange(1, 921).reshape(-1, 1), preds], axis=1), columns=np.arange(257))
df[0] = df[0].astype('int')
df.to_csv('./predictions.csv', index=False, header=False)
def run():
df = pd.read_csv(config.TRAIN_PATH)
kfold = KFold(n_splits=5, random_state=config.SEED, shuffle=True)
fold_losses = []
for i, (train_idx, val_idx) in enumerate(kfold.split(df)):
print("-------------------------------------------------------")
print(f"Training fold {i}")
print("-------------------------------------------------------")
train = df.iloc[train_idx]
validation = df.iloc[val_idx]
train_dataset = PicDataset(train)
train_data_loader = torch.utils.data.DataLoader(
train_dataset,
batch_size=config.BATCH_SIZE
)
val_dataset = PicDataset(validation)
val_data_loader = torch.utils.data.DataLoader(
val_dataset,
batch_size=config.BATCH_SIZE
)
device = 'cuda:0' if torch.cuda.is_available() else "cpu"
model = DNN()
model.to(device)
optimizer = torch.optim.Adam(model.parameters(), lr=config.LR)
loss = 0
for _ in range(config.EPOCHS):
engine.train_fn(train_data_loader, model, optimizer, device)
loss = engine.eval_fn(val_data_loader, model, device)
print(f"Loss on fold {i} is {loss}")
fold_losses.append(loss)
torch.save(model.state_dict(), f'./models/model_{i}.bin')
print(f"Average loss on cross validation is {sum(fold_losses) / 5}")
if args['weighted']:
weight = corpus.load_json(
os.path.join(source_path, 'cross_entropy_loss_weight.json'))
weight = torch.FloatTensor(list(weight.values())).to(device)
data = sorted(corpus.examples.get('seq'), key=lambda x: len(x), reverse=True)
vocab_size = len(corpus.words2id)
logging.info('vocabulary size: {}'.format(vocab_size))
model = DNN(vocab_size=vocab_size,
embedding_size=200,
hidden_size=512,
embedding=embedding)
model.to(device)
loss_function = nn.CrossEntropyLoss(weight=weight)
optimizer = torch.optim.SGD(model.parameters(), lr=0.01, momentum=0.9)
# optimizer = torch.optim.Adam(model.parameters(), lr=0.001)
model.train()
total_data = len(data)
batch_size = args['batch_size']
total_step = math.ceil(total_data / batch_size)
last_training_loss = 1000000000000
for epoch in range(args.get('epoch')):
start = 0
training_loss = 0
# create folder for generated data
gen_data_path = os.path.join(out_path, gen_data_fdr)
if not os.path.exists(gen_data_path):
os.makedirs(gen_data_path)
if not os.path.exists(scaler_dir):
os.makedirs(scaler_dir)
print('here')
# create folder for model checkpoints
checkpoint_path = os.path.join(out_path, checkpoint_fdr)
if not os.path.exists(checkpoint_path):
os.makedirs(checkpoint_path)
#Your statements here
model = DNN(input_size, hidden_size, out_size)
model = torch.nn.DataParallel(model.to(device), device_ids=use_devices)
#print(model)
#save_dir = "FPGA"
checkpoint = torch.load(
'./segan_data_out/20200422_0713/checkpoints/state-20.pkl')
#from collections import OrderedDict
state_dict = checkpoint['DNN']
#for k, v in state_dict.items():
# name = k[7:] # remove `module.`
# v = v.cpu().numpy()
# np.savetxt(os.path.join(save_dir,name), v, newline="\n")
# print(name, v.shape)
scaler_path_input = os.path.join(scaler_dir, "scaler_input.p")
scaler_input = pickle.load(open(scaler_path_input, 'rb'))
scaler_path_label = os.path.join(scaler_dir, "scaler_label.p")
scaler_label = pickle.load(open(scaler_path_label, 'rb'))
def main():
print('> Starting execution...')
# Training settings
parser = argparse.ArgumentParser(description='PyTorch MNIST Example')
group = parser.add_mutually_exclusive_group()
group.add_argument('--fit',
action='store_true',
help='fit the tuned model on digits 0-4')
group.add_argument('--transfer',
action='store_true',
help='train a pretrained model on digits 5-9')
parser.add_argument('--batch-size',
type=int,
default=256,
metavar='N',
help='input batch size for training (default: 256)')
parser.add_argument('--epochs',
type=int,
default=50,
metavar='E',
help='number of epochs to train (default: 50)')
parser.add_argument('--lr',
type=float,
default=1e-3,
metavar='L',
help='learning rate (default: 1e-3)')
parser.add_argument('--early-stopping',
type=int,
default=7,
metavar='E',
help='early stopping (default: 7 epochs)')
parser.add_argument(
'--size',
type=int,
default=100,
metavar='S',
help='size of the training data for transfer learning (default: 100)')
parser.add_argument('--seed',
type=int,
default=23,
metavar='S',
help='random seed (default: 23)')
args = parser.parse_args()
use_cuda = torch.cuda.is_available() # use cuda if available
device = torch.device("cuda" if use_cuda else "cpu")
torch.manual_seed(args.seed) # random seed
print('> Loading MNIST data')
train_set = datasets.MNIST(MNIST_DATA_DIR,
train=True,
download=True,
transform=transforms.Compose([
transforms.ToTensor(),
transforms.Normalize((0.1307, ), (0.3081, ))
]))
test_set = datasets.MNIST(MNIST_DATA_DIR,
train=False,
download=True,
transform=transforms.Compose([
transforms.ToTensor(),
transforms.Normalize((0.1307, ), (0.3081, ))
]))
train_digits_04 = np.where(train_set.train_labels < 5)[0]
train_digits_59 = np.where(train_set.train_labels > 4)[0]
test_digits_04 = np.where(test_set.test_labels < 5)[0]
test_digits_59 = np.where(test_set.test_labels > 4)[0]
if args.fit:
# Training the tuned model on digits 0-4
print('> Training a new model on MNIST digits 0-4')
X_train_04, y_train_04, X_valid_04, y_valid_04 = data_to_numpy(
train_set, test_set, INPUT_DIM, train_digits_04, test_digits_04)
torch.manual_seed(args.seed)
print('> Initializing the model')
model = DNN(INPUT_DIM, OUTPUT_DIM, HIDDEN_DIM, batch_norm=True)
model.apply(init_he_normal) # He initialization
model = model.to(device)
criterion = nn.CrossEntropyLoss()
optimizer = optim.Adam(model.parameters(), lr=args.lr)
print('> Training the model')
model, _, _ = train_model(model,
device,
X_train_04,
y_train_04,
criterion,
optimizer,
X_valid=X_valid_04,
y_valid=y_valid_04,
batch_size=args.batch_size,
n_epochs=args.epochs,
early_stopping=args.early_stopping)
print(f'> Saving the model state at {MODEL_04_PATH}')
torch.save(model.state_dict(), MODEL_04_PATH)
elif args.transfer:
# Transfer learning
print(
'> Training a model on MNIST digits 5-9 from a pretrained model for digits 0-4'
)
if os.path.isfile(MODEL_04_PATH):
print('> Loading the pretrained model')
model = DNN(INPUT_DIM, OUTPUT_DIM, HIDDEN_DIM,
batch_norm=True).to(device)
model.load_state_dict(torch.load(MODEL_04_PATH))
for param in model.parameters():
param.requires_grad = False
# Parameters of newly constructed modules have requires_grad=True by default
model.fc4 = nn.Linear(HIDDEN_DIM, HIDDEN_DIM)
model.fc5 = nn.Linear(HIDDEN_DIM, HIDDEN_DIM)
model.out = nn.Linear(HIDDEN_DIM, OUTPUT_DIM)
print('> Using saved model state')
else:
print(
'> Model state file is not found, fit a model before the transfer learning'
)
print('> Stopping execution')
return
X_train_59, y_train_59, X_valid_59, y_valid_59 = data_to_numpy(
train_set, test_set, INPUT_DIM, train_digits_59[:args.size],
test_digits_59)
# fixing the issues with labels
y_train_59 = y_train_59 - 5
y_valid_59 = y_valid_59 - 5
criterion = nn.CrossEntropyLoss()
optimizer = optim.Adam(model.parameters(), lr=args.lr)
print('> Training the model')
model, _, _ = train_model(model,
device,
X_train_59,
y_train_59,
criterion,
optimizer,
X_valid=X_valid_59,
y_valid=y_valid_59,
batch_size=args.batch_size,
n_epochs=args.epochs,
early_stopping=args.early_stopping)
print(f'> Saving the model state at {MODEL_59_PATH}')
torch.save(model.state_dict(), MODEL_59_PATH)
else:
print('> Incorrect mode, try either `--fit` or `--transfer`')
print('> Stopping execution')
