def cosine_annealing(step, total_steps, lr_max, lr_min):
return lr_min + (lr_max -
lr_min) * 0.5 * (1 + np.cos(step / total_steps * np.pi))
scheduler = torch.optim.lr_scheduler.LambdaLR(
optimizer,
lr_lambda=lambda step: cosine_annealing(
step,
args.epochs * len(train_loader),
1, # since lr_lambda computes multiplicative factor
1e-6 / args.learning_rate))
adversary = attacks.PGD(epsilon=8. / 255, num_steps=10,
step_size=2. / 255).cuda()
# /////////////// Training ///////////////
def train():
net.train() # enter train mode
loss_avg = 0.0
for bx, by in train_loader:
curr_batch_size = bx.size(0)
by_prime = torch.cat(
(torch.zeros(bx.size(0)), torch.ones(bx.size(0)),
2 * torch.ones(bx.size(0)), 3 * torch.ones(bx.size(0))),
0).long()
bx = bx.numpy()
# use torch.rot90 in later versions of pytorch
adv_examples = []
for data, target in self.test_data:
data, target = data.to(self.device), target.to(self.device)
init_pred, perturbed_data, final_pred = self.attack.generate(
data, epsilon, y=target)
target = target.view((20, 1))
correct += torch.mm((init_pred == target).t(),
(init_pred == final_pred)).item()
eval_step_no += 1
if eval_steps is not None and eval_steps_no == eval_steps:
break
final_acc = correct / float(total_examples)
print("Epsilon: {}\tTest Accuracy = {} / {} = {}".format(
epsilon, correct, total_examples, final_acc))
return final_acc, adv_examples
if __name__ == '__main__':
epsilons = [0, .05, .2, .25, .3]
model, test_loader, device = create_model(20)
attack = attacks.FGSM(model, device)
testd = Test_Attack(attack, test_loader, device, epsilons)
testd.test()
attack = attacks.PGD(model, device)
testd = Test_Attack(attack, test_loader, device, epsilons)
testd.test()
def main():
device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
# parse command line
parser = opts_parser()
options = parser.parse_args()
modelfile = options.modelfile
cfg = {}
print(options.vars)
for fn in options.vars:
cfg.update(config.parse_config_file(fn))
cfg.update(config.parse_variable_assignments(options.var))
sample_rate = cfg['sample_rate']
frame_len = cfg['frame_len']
fps = cfg['fps']
mel_bands = cfg['mel_bands']
mel_min = cfg['mel_min']
mel_max = cfg['mel_max']
blocklen = cfg['blocklen']
batchsize = cfg['batchsize']
bin_nyquist = frame_len // 2 + 1
bin_mel_max = bin_nyquist * 2 * mel_max // sample_rate
# prepare dataset
datadir = os.path.join(os.path.dirname(__file__), os.path.pardir,
'datasets', options.dataset)
meanstd_file = os.path.join(os.path.dirname(__file__),
'%s_meanstd.npz' % options.dataset)
if (options.input_type == 'audio'):
dataloader = DatasetLoader(options.dataset,
options.cache_spectra,
datadir,
input_type=options.input_type)
batches = dataloader.prepare_audio_batches(sample_rate, frame_len, fps,
blocklen, batchsize)
else:
dataloader = DatasetLoader(options.dataset,
options.cache_spectra,
datadir,
input_type=options.input_type)
batches = dataloader.prepare_batches(sample_rate, frame_len, fps,
mel_bands, mel_min, mel_max,
blocklen, batchsize)
validation_data = DatasetLoader(options.dataset,
'../ismir2015/experiments/mel_data/',
datadir,
dataset_split='valid',
input_type='mel_spects')
mel_spects_val, labels_val = validation_data.prepare_batches(
sample_rate,
frame_len,
fps,
mel_bands,
mel_min,
mel_max,
blocklen,
batchsize,
batch_data=False)
mdl = model.CNNModel(model_type=options.model_type,
input_type=options.input_type,
is_zeromean=False,
sample_rate=sample_rate,
frame_len=frame_len,
fps=fps,
mel_bands=mel_bands,
mel_min=mel_min,
mel_max=mel_max,
bin_mel_max=bin_mel_max,
meanstd_file=meanstd_file,
device=device)
mdl = mdl.to(device)
#Setting up learning rate and learning rate parameters
initial_eta = cfg['initial_eta']
eta_decay = cfg['eta_decay']
momentum = cfg['momentum']
eta_decay_every = cfg.get('eta_decay_every', 1)
eta = initial_eta
#set up loss
criterion = torch.nn.BCELoss()
#set up optimizer
optimizer = torch.optim.SGD(mdl.parameters(),
lr=eta,
momentum=momentum,
nesterov=True)
scheduler = torch.optim.lr_scheduler.StepLR(optimizer,
step_size=eta_decay_every,
gamma=eta_decay)
#set up optimizer
writer = SummaryWriter(os.path.join(modelfile, 'runs'))
epochs = cfg['epochs']
epochsize = cfg['epochsize']
batches = iter(batches)
#conditions to save model
best_val_loss = 100000.
best_val_error = 1.
for epoch in range(epochs):
# - Initialize certain parameters that are used to monitor training
err = 0
total_norm = 0
loss_accum = 0
mdl.train(True)
# - Compute the L-2 norm of the gradients
for p in mdl.parameters():
if p.grad is not None:
param_norm = p.grad.data.norm(2)
total_norm += param_norm.item()**2
total_norm = total_norm**(1. / 2)
# - Start the training for this epoch
for batch in progress(range(epochsize),
min_delay=0.5,
desc='Epoch %d/%d: Batch ' %
(epoch + 1, epochs)):
data = next(batches)
if (options.input_type == 'audio' or options.input_type == 'stft'):
input_data = data[0]
else:
input_data = np.transpose(data[0][:, :, :, np.newaxis],
(0, 3, 1, 2))
labels = data[1][:, np.newaxis].astype(np.float32)
#map labels to make them softer
if not options.adversarial_training:
labels = (0.02 + 0.96 * labels)
optimizer.zero_grad()
if (options.adversarial_training):
mdl.train(False)
if (options.input_type == 'stft'):
input_data_adv = attacks.PGD(
mdl,
torch.from_numpy(input_data).to(device),
target=torch.from_numpy(labels).to(device),
eps=cfg['eps'],
step_size=cfg['eps_iter'],
iterations=cfg['nb_iter'],
use_best=True,
random_start=True,
clip_min=0,
clip_max=1e8).cpu().detach().numpy()
else:
input_data_adv = attacks.PGD(
mdl,
torch.from_numpy(input_data).to(device),
target=torch.from_numpy(labels).to(device),
eps=cfg['eps'],
step_size=cfg['eps_iter'],
iterations=cfg['nb_iter'],
use_best=True,
random_start=True).cpu().detach().numpy()
mdl.train(True)
optimizer.zero_grad()
outputs = mdl(torch.from_numpy(input_data_adv).to(device))
else:
optimizer.zero_grad()
outputs = mdl(torch.from_numpy(input_data).to(device))
#input(outputs.size())
#input(mdl.conv(torch.from_numpy(input_data).to(device)).cpu().detach().numpy().shape)
loss = criterion(outputs, torch.from_numpy(labels).to(device))
loss.backward()
optimizer.step()
print(loss.item())
loss_accum += loss.item()
# - Compute validation loss and error if desired
if options.validate:
mdl.input_type = 'mel_spects'
from eval import evaluate
mdl.train(False)
val_loss = 0
preds = []
labs = []
max_len = fps
num_iter = 0
for spect, label in zip(mel_spects_val, labels_val):
num_excerpts = len(spect) - blocklen + 1
excerpts = np.lib.stride_tricks.as_strided(
spect,
shape=(num_excerpts, blocklen, spect.shape[1]),
strides=(spect.strides[0], spect.strides[0],
spect.strides[1]))
# - Pass mini-batches through the network and concatenate results
for pos in range(0, num_excerpts, batchsize):
input_data = np.transpose(
excerpts[pos:pos + batchsize, :, :, np.newaxis],
(0, 3, 1, 2))
#if (pos+batchsize>num_excerpts):
# label_batch = label[blocklen//2+pos:blocklen//2+num_excerpts,
# np.newaxis].astype(np.float32)
#else:
# label_batch = label[blocklen//2+pos:blocklen//2+pos+batchsize,
# np.newaxis].astype(np.float32)
if (pos + batchsize > num_excerpts):
label_batch = label[pos:num_excerpts,
np.newaxis].astype(np.float32)
else:
label_batch = label[pos:pos + batchsize,
np.newaxis].astype(np.float32)
pred = mdl(torch.from_numpy(input_data).to(device))
e = criterion(pred,
torch.from_numpy(label_batch).to(device))
preds = np.append(preds, pred[:, 0].cpu().detach().numpy())
labs = np.append(labs, label_batch)
val_loss += e.item()
num_iter += 1
mdl.input_type = options.input_type
print("Validation loss: %.3f" % (val_loss / num_iter))
_, results = evaluate(preds, labs)
print("Validation error: %.3f" % (1 - results['accuracy']))
if (1 - results['accuracy'] < best_val_error):
torch.save(mdl.state_dict(),
os.path.join(modelfile, 'model.pth'))
best_val_loss = val_loss / num_iter
best_val_error = 1 - results['accuracy']
print('New saved model', best_val_loss, best_val_error)
#Update the learning rate
scheduler.step()
print('Training Loss per epoch', loss_accum / epochsize)
# - Save parameters for examining
writer.add_scalar('Training Loss', loss_accum / epochsize, epoch)
writer.add_scalar('Validation loss', val_loss / num_iter, epoch)
writer.add_scalar('Gradient norm', total_norm, epoch)
writer.add_scalar('Validation error', 1 - results['accuracy'])
#for param_group in optimizer.param_groups:
#print(param_group['lr'])
if not options.validate:
torch.save(mdl.state_dict(), os.path.join(modelfile, 'model.pth'))
with io.open(os.path.join(modelfile, 'model.vars'), 'w') as f:
f.writelines('%s=%s\n' % kv for kv in cfg.items())