def predict():
device = "cuda:0" if torch.cuda.is_available() else "cpu"
df = pd.read_csv(config.TEST_PATH, header=None)
dataset = PicDataset(df.loc[:, 1:])
preds = np.zeros((len(dataset), 256))
for i in range(5):
temp = np.zeros((len(dataset), 256))
model = DNN()
model.load_state_dict(torch.load(f'./models/model_{i}.bin'))
model.to(device)
model.eval()
for j in range(len(dataset)):
x, _ = dataset[j]
x = x.to(device)
y = model(x)
temp[j, :] = y.detach().cpu().numpy()
preds += temp
preds /= 5
df = pd.DataFrame(np.concatenate([np.arange(1, 921).reshape(-1, 1), preds], axis=1), columns=np.arange(257))
df[0] = df[0].astype('int')
df.to_csv('./predictions.csv', index=False, header=False)
def train(args, config, io):
train_loader, validation_loader = get_loader(args, config)
device = torch.device("cuda" if args.cuda else "cpu")
# print(len(train_loader), len(validation_loader))
#Try to load models
model = DNN(args).to(device)
"""if device == torch.device("cuda"):
model = nn.DataParallel(model)"""
if args.model_path != "":
model.load_state_dict(torch.load(args.model_path))
# for para in list(model.parameters())[:-5]:
# para.requires_grad=False
# print(model)
if args.use_sgd:
# print("Use SGD")
opt = optim.SGD(model.parameters(),
lr=args.lr * 100,
momentum=args.momentum,
weight_decay=1e-4)
else:
# print("Use Adam")
opt = optim.Adam(model.parameters(), lr=args.lr, weight_decay=1e-4)
"""opt = optim.Adam([
{'params': list(model.parameters())[:-1], 'lr':args.lr/50, 'weight_decay': 1e-4},
{'params': list(model.parameters())[-1], 'lr':args.lr, 'weight_decay': 1e-4}
])
"""
scheduler = CosineAnnealingLR(opt, args.epochs, eta_min=args.lr)
criterion = nn.MSELoss()
best_test_loss = 9999999.
for epoch in range(args.epochs):
startTime = time.time()
####################
# Train
####################
train_loss = 0.0
train_dis = 0.0
count = 0.0
model.train()
for data, label in train_loader:
data, label = data.to(device), label.to(device)
data = drop(jitter(data, device), device)
# data = jitter(data, device, delta=0.05)
batch_size = data.shape[0]
logits = model(data)
loss = criterion(logits, label)
opt.zero_grad()
loss.backward()
opt.step()
dis = distance(logits, label)
count += batch_size
train_loss += loss.item() * batch_size
train_dis += dis.item() * batch_size
scheduler.step()
outstr = 'Train %d, loss: %.6f, distance: %.6f' % (
epoch, train_loss * 1.0 / count, train_dis * 1.0 / count)
io.cprint(outstr)
####################
# Evaluation
####################
test_loss = 0.0
test_dis = 0.0
count = 0.0
model.eval()
with torch.no_grad():
for data, label in validation_loader:
data, label = data.to(device), label.to(device)
batch_size = data.shape[0]
logits = model(data)
loss = criterion(logits, label)
dis = distance(logits, label)
count += batch_size
test_loss += loss.item() * batch_size
test_dis += dis.item() * batch_size
outstr = 'Test %d, loss: %.6f, distance: %.6f' % (
epoch, test_loss * 1.0 / count, test_dis * 1.0 / count)
io.cprint(outstr)
if test_loss <= best_test_loss:
best_test_loss = test_loss
torch.save(model.state_dict(),
'checkpoints/%s/models/model.t7' % args.exp_name)
torch.save(model, (config.root + config.model_path))
io.cprint('Time: %.3f sec' % (time.time() - startTime))
model = DNN(x.size(1)).to(device)
total_param = sum(p.numel() for p in model.parameters())
trainable_param = sum(p.numel() for p in model.parameters()
if p.requires_grad)
print("{} parameters with {} trainable".format(total_param,
trainable_param),
flush=True)
print("\nStart training...", flush=True)
train_dataset1 = TwitterDataset(x, train_y1)
train_loader1 = torch.utils.data.DataLoader(dataset=train_dataset1,
batch_size=BATCH,
shuffle=True,
num_workers=4)
train_model(train_loader1, model, device, LR)
print("\nStart testing...", flush=True)
test_x = [
"today is a good day , but it is hot",
"today is hot , but it is a good day"
]
test_x = [i.split() for i in test_x]
x = torch.zeros(2, len(word_index))
for i in range(2):
for w in test_x[i]:
x[i][word_index[w]] += 1
model.load_state_dict(torch.load("model_bow.pkl"))
model.eval()
x = x.to(device, dtype=torch.float)
print(model(x).cpu())
'./segan_data_out/20200422_0713/checkpoints/state-20.pkl')
#from collections import OrderedDict
state_dict = checkpoint['DNN']
#for k, v in state_dict.items():
# name = k[7:] # remove `module.`
# v = v.cpu().numpy()
# np.savetxt(os.path.join(save_dir,name), v, newline="\n")
# print(name, v.shape)
scaler_path_input = os.path.join(scaler_dir, "scaler_input.p")
scaler_input = pickle.load(open(scaler_path_input, 'rb'))
scaler_path_label = os.path.join(scaler_dir, "scaler_label.p")
scaler_label = pickle.load(open(scaler_path_label, 'rb'))
test_audio_path = './dr1_fdac1_sx214.wav'
mode = 'magnitude'
n_concat = 11
model.load_state_dict(state_dict)
#if torch.cuda.is_available():
# model.cuda()
test_audio, _ = read_audio(test_audio_path)
test_spec = calc_sp(test_audio, mode)
mixed_complx_x = calc_sp(test_audio, mode='complex')
plt.matshow(test_spec.T)
plt.savefig("noisy.png")
n_window = 512
n_overlap = 256
ham_win = np.hamming(n_window)
#print(test_spec)
test_spec = log_sp(test_spec)
n_pad = (n_concat - 1) // 2
test_spec_padding = pad_with_border(test_spec, n_pad)
test_spec_padding_concated = concate_seq(test_spec_padding, n_pad)
import os
import torch
import time
import torch.nn.functional as F
from args import args
from model import DNN
from utils import cos_sim
from processing import BuildExamples
# 加载模型
MODEL_PATH = os.path.join(args.get('data_path'), 'model/embedded_adam_0.001_19.model')
print(MODEL_PATH)
model = DNN(vocab_size=5265, embedding_size=200, hidden_size=512)
model.load_state_dict(torch.load(MODEL_PATH, map_location=torch.device('cpu')))
# 加载字典
corpus = BuildExamples()
corpus.load_vocabulary(path=os.path.join(args.get('data_path'), 'vocabulary.txt'))
MAX_LEN = args.get('max_len', 20)
def get_hidden_state(sentence):
line = sentence.strip().split()
line = ['bos'] + line + ['eos']
sentence2id = torch.LongTensor([corpus.words2id.get(item, 0) for item in line]).view(1, -1)
with torch.no_grad():
last_hidden_state, _ = model(sentence2id)
def train(args, config, io):
train_loader, validation_loader, unlabelled_loader = get_loader(
args, config)
device = torch.device("cuda" if args.cuda else "cpu")
#Try to load models
model = DNN(args).to(device)
ema_model = DNN(args).to(device)
for param in ema_model.parameters():
param.detach_()
if device == torch.device("cuda"):
model = nn.DataParallel(model)
ema_model = nn.DataParallel(ema_model)
if args.model_path != "":
model.load_state_dict(torch.load(args.model_path))
ema_model.load_state_dict(torch.load(args.model_path))
if args.use_sgd:
print("Use SGD")
opt = optim.SGD(model.parameters(),
lr=args.lr * 100,
momentum=args.momentum,
weight_decay=1e-4)
else:
print("Use Adam")
opt = optim.Adam(model.parameters(), lr=args.lr, weight_decay=1e-4)
scheduler = CosineAnnealingLR(opt, args.epochs, eta_min=args.lr)
criterion = nn.MSELoss()
consistency_criterion = nn.MSELoss()
best_test_loss = 9999999.
global_step = 0
for epoch in range(args.epochs):
startTime = time.time()
####################
# Train
####################
train_loss = 0.0
count = 0.0
model.train()
ema_model.train()
i = -1
for (data, label), (u, _) in zip(cycle(train_loader),
unlabelled_loader):
i = i + 1
if data.shape[0] != u.shape[0]:
bt_size = np.minimum(data.shape[0], u.shape[0])
data = data[0:bt_size]
label = label[0:bt_size]
u = u[0:bt_size]
data, label, u = data.to(device), label.to(device), u.to(device)
batch_size = data.shape[0]
logits = model(data)
class_loss = criterion(logits, label)
u_student = jitter(u, device)
u_teacher = jitter(u, device)
logits_unlabeled = model(u_student)
ema_logits_unlabeled = ema_model(u_teacher)
ema_logits_unlabeled = Variable(ema_logits_unlabeled.detach().data,
requires_grad=False)
consistency_loss = consistency_criterion(logits_unlabeled,
ema_logits_unlabeled)
if epoch < args.consistency_rampup_starts:
consistency_weight = 0.0
else:
consistency_weight = get_current_consistency_weight(
args, args.final_consistency, epoch, i,
len(unlabelled_loader))
consistency_loss = consistency_weight * consistency_loss
loss = class_loss + consistency_loss
opt.zero_grad()
loss.backward()
opt.step()
global_step += 1
# print(global_step)
update_ema_variables(model, ema_model, args.ema_decay, global_step)
count += batch_size
train_loss += loss.item() * batch_size
scheduler.step()
outstr = 'Train %d, loss: %.6f' % (epoch, train_loss * 1.0 / count)
io.cprint(outstr)
####################
# Evaluation
####################
test_loss = 0.0
count = 0.0
model.eval()
ema_model.eval()
for data, label in validation_loader:
data, label = data.to(device), label.to(device)
batch_size = data.shape[0]
logits = ema_model(data)
loss = criterion(logits, label)
count += batch_size
test_loss += loss.item() * batch_size
outstr = 'Test %d, loss: %.6f' % (epoch, test_loss * 1.0 / count)
io.cprint(outstr)
if test_loss <= best_test_loss:
best_test_loss = test_loss
torch.save(ema_model.state_dict(),
'checkpoints/%s/models/model.t7' % args.exp_name)
torch.save(ema_model, (config.root + config.model_path))
io.cprint('Time: %.3f sec' % (time.time() - startTime))
def main():
print('> Starting execution...')
# Training settings
parser = argparse.ArgumentParser(description='PyTorch MNIST Example')
group = parser.add_mutually_exclusive_group()
group.add_argument('--fit',
action='store_true',
help='fit the tuned model on digits 0-4')
group.add_argument('--transfer',
action='store_true',
help='train a pretrained model on digits 5-9')
parser.add_argument('--batch-size',
type=int,
default=256,
metavar='N',
help='input batch size for training (default: 256)')
parser.add_argument('--epochs',
type=int,
default=50,
metavar='E',
help='number of epochs to train (default: 50)')
parser.add_argument('--lr',
type=float,
default=1e-3,
metavar='L',
help='learning rate (default: 1e-3)')
parser.add_argument('--early-stopping',
type=int,
default=7,
metavar='E',
help='early stopping (default: 7 epochs)')
parser.add_argument(
'--size',
type=int,
default=100,
metavar='S',
help='size of the training data for transfer learning (default: 100)')
parser.add_argument('--seed',
type=int,
default=23,
metavar='S',
help='random seed (default: 23)')
args = parser.parse_args()
use_cuda = torch.cuda.is_available() # use cuda if available
device = torch.device("cuda" if use_cuda else "cpu")
torch.manual_seed(args.seed) # random seed
print('> Loading MNIST data')
train_set = datasets.MNIST(MNIST_DATA_DIR,
train=True,
download=True,
transform=transforms.Compose([
transforms.ToTensor(),
transforms.Normalize((0.1307, ), (0.3081, ))
]))
test_set = datasets.MNIST(MNIST_DATA_DIR,
train=False,
download=True,
transform=transforms.Compose([
transforms.ToTensor(),
transforms.Normalize((0.1307, ), (0.3081, ))
]))
train_digits_04 = np.where(train_set.train_labels < 5)[0]
train_digits_59 = np.where(train_set.train_labels > 4)[0]
test_digits_04 = np.where(test_set.test_labels < 5)[0]
test_digits_59 = np.where(test_set.test_labels > 4)[0]
if args.fit:
# Training the tuned model on digits 0-4
print('> Training a new model on MNIST digits 0-4')
X_train_04, y_train_04, X_valid_04, y_valid_04 = data_to_numpy(
train_set, test_set, INPUT_DIM, train_digits_04, test_digits_04)
torch.manual_seed(args.seed)
print('> Initializing the model')
model = DNN(INPUT_DIM, OUTPUT_DIM, HIDDEN_DIM, batch_norm=True)
model.apply(init_he_normal) # He initialization
model = model.to(device)
criterion = nn.CrossEntropyLoss()
optimizer = optim.Adam(model.parameters(), lr=args.lr)
print('> Training the model')
model, _, _ = train_model(model,
device,
X_train_04,
y_train_04,
criterion,
optimizer,
X_valid=X_valid_04,
y_valid=y_valid_04,
batch_size=args.batch_size,
n_epochs=args.epochs,
early_stopping=args.early_stopping)
print(f'> Saving the model state at {MODEL_04_PATH}')
torch.save(model.state_dict(), MODEL_04_PATH)
elif args.transfer:
# Transfer learning
print(
'> Training a model on MNIST digits 5-9 from a pretrained model for digits 0-4'
)
if os.path.isfile(MODEL_04_PATH):
print('> Loading the pretrained model')
model = DNN(INPUT_DIM, OUTPUT_DIM, HIDDEN_DIM,
batch_norm=True).to(device)
model.load_state_dict(torch.load(MODEL_04_PATH))
for param in model.parameters():
param.requires_grad = False
# Parameters of newly constructed modules have requires_grad=True by default
model.fc4 = nn.Linear(HIDDEN_DIM, HIDDEN_DIM)
model.fc5 = nn.Linear(HIDDEN_DIM, HIDDEN_DIM)
model.out = nn.Linear(HIDDEN_DIM, OUTPUT_DIM)
print('> Using saved model state')
else:
print(
'> Model state file is not found, fit a model before the transfer learning'
)
print('> Stopping execution')
return
X_train_59, y_train_59, X_valid_59, y_valid_59 = data_to_numpy(
train_set, test_set, INPUT_DIM, train_digits_59[:args.size],
test_digits_59)
# fixing the issues with labels
y_train_59 = y_train_59 - 5
y_valid_59 = y_valid_59 - 5
criterion = nn.CrossEntropyLoss()
optimizer = optim.Adam(model.parameters(), lr=args.lr)
print('> Training the model')
model, _, _ = train_model(model,
device,
X_train_59,
y_train_59,
criterion,
optimizer,
X_valid=X_valid_59,
y_valid=y_valid_59,
batch_size=args.batch_size,
n_epochs=args.epochs,
early_stopping=args.early_stopping)
print(f'> Saving the model state at {MODEL_59_PATH}')
torch.save(model.state_dict(), MODEL_59_PATH)
else:
print('> Incorrect mode, try either `--fit` or `--transfer`')
print('> Stopping execution')