def get_precision_recall(args, score, label, num_samples, beta=1.0, sampling='log', predicted_score=None):
'''
:param args:
:param score: anomaly scores
:param label: anomaly labels
:param num_samples: the number of threshold samples
:param beta:
:param scale:
:return:
'''
if predicted_score is not None:
score = score - torch.FloatTensor(predicted_score).squeeze().to(args.device)
maximum = score.max()
if sampling=='log':
# Sample thresholds logarithmically
# The sampled thresholds are logarithmically spaced between: math:`10 ^ {start}` and: math:`10 ^ {end}`.
th = torch.logspace(0, torch.log10(torch.tensor(maximum)), num_samples).to(args.device)
else:
# Sample thresholds equally
# The sampled thresholds are equally spaced points between: attr:`start` and: attr:`end`
th = torch.linspace(0, maximum, num_samples).to(args.device)
precision = []
recall = []
for i in range(len(th)):
anomaly = (score > th[i]).float()
idx = anomaly * 2 + label
tn = (idx == 0.0).sum().item() # tn
fn = (idx == 1.0).sum().item() # fn
fp = (idx == 2.0).sum().item() # fp
tp = (idx == 3.0).sum().item() # tp
p = tp / (tp + fp + 1e-7)
r = tp / (tp + fn + 1e-7)
if p != 0 and r != 0:
precision.append(p)
recall.append(r)
precision = torch.FloatTensor(precision)
recall = torch.FloatTensor(recall)
f1 = (1 + beta ** 2) * (precision * recall).div(beta ** 2 * precision + recall + 1e-7)
return precision, recall, f1
input_gt_var = input_gt.cuda()
for jj in range(K):
# loss[jj] = alpha * torch.mean(torch.abs(Z[k])) + torch.mean(torch.abs(E[k])) # + torch.mean(dual_gap(torch.mm(A_tensor.t(), L[k]), alpha)) + torch.mean(dual_gap(L[k], 1)) + torch.mean(L[k] * input_bs_var))
loss[jj] += alpha * Z[jj].abs().sum(dim=0).mean(0) + (
input_bs_var - model.A.mm(Z[jj])
).abs().sum(dim=0).mean(
) # + torch.mean(dual_gap(torch.mm(A_tensor.t(), L[k]), alpha)) + torch.mean(dual_gap(L[k], 1)) + torch.mean(L[k] * input_bs_var))
mse_value[jj] = mse_value[jj] + F.mse_loss(
255 * input_gt_var.cuda(), 255 * torch.mm(A_tensor, Z[jj]))
if use_learned and use_safeguard:
sg_count[jj] += count[jj]
loss = loss / (n_test // batch_size)
mse_value = mse_value / (n_test // batch_size)
psnr = -10 * torch.log10(mse_value) + torch.tensor(48.131).cuda()
if use_learned and use_safeguard:
sg_pct = sg_count / float(n_test)
for jj in range(K):
if (psnr_value < psnr[jj]):
psnr_value = psnr[jj].cpu().item()
for jjj in range(n_test // batch_size):
input_bs = X_ts[:, jjj * batch_size:(jjj + 1) * batch_size]
input_bs = torch.from_numpy(input_bs)
# input_bs_var = torch.autograd.Variable(input_bs.cuda())
input_bs_var = input_bs.cuda()
if use_learned and use_safeguard:
Z, E, L, count = model(input_bs_var)
else:
[Z, E, L] = model(input_bs_var)
best_pic[:, jjj * batch_size:(jjj + 1) * batch_size] = (
def train(self,
opts,
dloader,
criterion,
l1_init,
l1_dec_step,
l1_dec_epoch,
log_freq,
va_dloader=None,
device='cpu'):
""" Train the SEGAN """
# create writer
self.writer = SummaryWriter(os.path.join(opts.save_path, 'train'))
if opts.opt == 'rmsprop':
Gopt = optim.RMSprop(self.G.parameters(), lr=opts.g_lr)
elif opts.opt == 'adam':
Gopt = optim.Adam(self.G.parameters(),
lr=opts.g_lr,
betas=(0.5, 0.9))
else:
raise ValueError('Unrecognized optimizer {}'.format(opts.opt))
# attach opts to models so that they are saved altogether in ckpts
self.G.optim = Gopt
# Build savers for end of epoch, storing up to 3 epochs each
eoe_g_saver = Saver(self.G,
opts.save_path,
max_ckpts=3,
optimizer=self.G.optim,
prefix='EOE_G-')
num_batches = len(dloader)
l2_weight = l1_init
iteration = 1
timings = []
evals = {}
noisy_evals = {}
noisy_samples = None
clean_samples = None
z_sample = None
patience = opts.patience
best_val_obj = np.inf
# acumulator for exponential avg of valid curve
acum_val_obj = 0
G = self.G
for iteration in range(1, opts.epoch * len(dloader) + 1):
beg_t = timeit.default_timer()
uttname, clean, noisy, slice_idx = self.sample_dloader(
dloader, device)
bsz = clean.size(0)
Genh = self.infer_G(noisy, clean)
Gopt.zero_grad()
if self.l1_loss:
loss = F.l1_loss(Genh, clean)
else:
loss = F.mse_loss(Genh, clean)
loss.backward()
Gopt.step()
end_t = timeit.default_timer()
timings.append(end_t - beg_t)
beg_t = timeit.default_timer()
if noisy_samples is None:
noisy_samples = noisy[:20, :, :].contiguous()
clean_samples = clean[:20, :, :].contiguous()
if z_sample is None and not G.no_z:
# capture sample now that we know shape after first
# inference
z_sample = G.z[:20, :, :].contiguous()
print('z_sample size: ', z_sample.size())
z_sample = z_sample.to(device)
if iteration % log_freq == 0:
# POWER Loss (not used to backward) -----------------------------------
# make stft of gtruth
clean_stft = torch.stft(clean.squeeze(1),
n_fft=min(clean.size(-1), self.n_fft),
hop_length=160,
win_length=320,
normalized=True)
clean_mod = torch.norm(clean_stft, 2, dim=3)
clean_mod_pow = 10 * torch.log10(clean_mod**2 + 10e-20)
Genh_stft = torch.stft(Genh.detach().squeeze(1),
n_fft=min(Genh.size(-1), self.n_fft),
hop_length=160,
win_length=320,
normalized=True)
Genh_mod = torch.norm(Genh_stft, 2, dim=3)
Genh_mod_pow = 10 * torch.log10(Genh_mod**2 + 10e-20)
pow_loss = F.l1_loss(Genh_mod_pow, clean_mod_pow)
log = 'Iter {}/{} ({} bpe) g_l2_loss:{:.4f}, ' \
'pow_loss: {:.4f}, ' \
''.format(iteration,
len(dloader) * opts.epoch,
len(dloader),
loss.item(),
pow_loss.item())
log += 'btime: {:.4f} s, mbtime: {:.4f} s' \
''.format(timings[-1],
np.mean(timings))
print(log)
self.writer.add_scalar('g_l2/l1_loss', loss.item(), iteration)
self.writer.add_scalar('G_pow_loss', pow_loss.item(),
iteration)
self.writer.add_histogram('clean_mod_pow',
clean_mod_pow.cpu().data,
iteration,
bins='sturges')
self.writer.add_histogram('Genh_mod_pow',
Genh_mod_pow.cpu().data,
iteration,
bins='sturges')
self.writer.add_histogram('Gz',
Genh.cpu().data,
iteration,
bins='sturges')
self.writer.add_histogram('clean',
clean.cpu().data,
iteration,
bins='sturges')
self.writer.add_histogram('noisy',
noisy.cpu().data,
iteration,
bins='sturges')
if hasattr(G, 'skips'):
for skip_id, alpha in G.skips.items():
skip = alpha['alpha']
if skip.skip_type == 'alpha':
self.writer.add_histogram(
'skip_alpha_{}'.format(skip_id),
skip.skip_k.data,
iteration,
bins='sturges')
# get D and G weights and plot their norms by layer and global
def model_weights_norm(model, total_name):
total_GW_norm = 0
for k, v in model.named_parameters():
if 'weight' in k:
W = v.data
W_norm = torch.norm(W)
self.writer.add_scalar('{}_Wnorm'.format(k),
W_norm, iteration)
total_GW_norm += W_norm
self.writer.add_scalar('{}_Wnorm'.format(total_name),
total_GW_norm, iteration)
#model_weights_norm(G, 'Gtotal')
#model_weights_norm(D, 'Dtotal')
if not opts.no_train_gen:
#canvas_w = self.G(noisy_samples, z=z_sample)
self.gen_train_samples(clean_samples,
noisy_samples,
z_sample,
iteration=iteration)
if va_dloader is not None:
if len(noisy_evals) == 0:
sd, nsd = self.evaluate(opts,
va_dloader,
log_freq,
do_noisy=True)
self.writer.add_scalar('noisy_SD', nsd, iteration)
else:
sd = self.evaluate(opts,
va_dloader,
log_freq,
do_noisy=False)
self.writer.add_scalar('Genh_SD', sd, iteration)
print('Eval SD: {:.3f} dB, NSD: {:.3f} dB'.format(sd, nsd))
if sd < best_val_obj:
self.G.save(self.save_path, iteration, True)
best_val_obj = sd
if iteration % len(dloader) == 0:
# save models in end of epoch with EOE savers
self.G.save(self.save_path, iteration, saver=eoe_g_saver)
def __call__(img1, img2):
mse = torch.mean((img1 - img2)**2)
return 20 * torch.log10(255.0 / torch.sqrt(mse))
for i, target_image in enumerate(target_list):
print("target {} : {}".format(i,np.array(target_image).shape))
for i,input_image in enumerate(input):
print("input {}: {}".format(i,np.array(input_image).shape))
"""
input, target = input.to(device), target.to(device)
# train Discriminator
Dis_optimizer.zero_grad()
lr_generated = Gen_Model(input)
lr_discriminated = Dis_Model(lr_generated)
hr_discriminated = Dis_Model(target)
Mseloss_temp = content_criterion(lr_generated, target)
PSNR_temp = 10 * torch.log10(1 / Mseloss_temp)
total_PSNR_train += PSNR_temp
dis_adversarial_loss = adversal_criterion(
lr_discriminated,
torch.zeros_like(lr_discriminated)) + adversal_criterion(
hr_discriminated, torch.ones_like(hr_discriminated))
dis_adversarial_loss.backward()
Dis_optimizer.step()
Dis_loss_total += float(torch.mean(hr_discriminated))
# if grad_clip is not None:
# torch.utils.clip_gradient
# train Generator
def test_log10(x, y):
c = torch.log10(torch.add(x, y))
return c
def PSNRmetrics(x, y, model):
y_pred = model(x)
mse = nn.MSELoss()((y_pred + 1) / 2, (y + 1) / 2)
return 10 * torch.log10(1 / mse)
def test(model, dataloader, logger, log_interval, print_string='test'):
'''
Runs a test on data insider dataloader
Parameters
----------
model : pytorch neural network module
Generic NN.
dataloader : pytorch dataloader
data to run tests on.
logger : python logger
For writing info.
log_interval : int
For deciding how often to print onto log file.
Returns
-------
y_pred : linear tensor
linear array of predicted values
y_true : linear tensor
linear array of actual labels
y_logits : tensor
tensor of original output logits
'''
model.eval()
gpu = torch.device("cuda")
cpu = torch.device("cpu")
y_true_s = None
y_pred_s = None
all_logits_s = None
y_true_v = None
y_pred_v = None
all_logits_v = None
with torch.no_grad():
for batch_id, minibatch in enumerate(dataloader):
if batch_id % log_interval == 0:
logger.info(('\tTesting ' + print_string +
' Minibatch: %4d' % batch_id))
#x0 = minibatch[0].to(gpu) # index in orig data (unused)
x1 = minibatch[1].to(gpu) # encoded_tweets_h
x2 = minibatch[2].to(gpu) # token_type_ids_h
x3 = minibatch[3].to(gpu) # attention_mask_h
x4 = minibatch[4].to(gpu) # encoded_tweets_t
x5 = minibatch[5].to(gpu) # token_type_ids_t
x6 = minibatch[6].to(gpu) # attention_mask_t
x7 = minibatch[7].float() # followers_head
x8 = minibatch[8].float() # followers_tail
x9 = minibatch[9].float() # interaction_type_num
x7 = torch.log10(x7.to(gpu) +
0.1) # log to scale the numbers down to earth
x8 = torch.log10(x8.to(gpu) +
0.1) # log to scale the numbers down to earth
x9 = x9.to(gpu)
y_s = minibatch[10].to(gpu) # true label 4 stance class
y_v = minibatch[11].to(gpu) # viral_score
outputs = model(input_ids_h=x1,
token_type_ids_h=x2,
attention_mask_h=x3,
input_ids_t=x4,
token_type_ids_t=x5,
attention_mask_t=x6,
followers_head=x7,
followers_tail=x8,
int_type_num=x9,
task='multi')
logits_s = outputs[0]
logits_v = outputs[1]
if y_true_s is None: # for handling 1st minibatch
y_true_s = y_s.clone().to(cpu) # shape=(n,)
y_true_v = y_v.clone().to(cpu) # shape=(n,)
idx_s = logits_s.argmax(
1) # for finding index of max value for stance
idx_v = logits_v.argmax(
1) # for finding index of max value for viral
y_pred_s = idx_s.clone().to(cpu) # copy the index to cpu
y_pred_v = idx_v.clone().to(cpu) # copy the index to cpu
all_logits_s = logits_s.clone().to(
cpu) # copy the stance logits
all_logits_v = logits_v.clone().to(
cpu) # copy the viral logits
else: # for all other minibatches
y_true_s = torch.cat((y_true_s, y_s.clone().to(cpu)), 0)
y_true_v = torch.cat((y_true_v, y_v.clone().to(cpu)), 0)
idx_s = logits_s.argmax(1) # for finding index of max value
idx_v = logits_v.argmax(1) # for finding index of max value
y_pred_s = torch.cat((y_pred_s, idx_s.clone().to(cpu)), 0)
y_pred_v = torch.cat((y_pred_v, idx_v.clone().to(cpu)), 0)
all_logits_s = torch.cat(
(all_logits_s, logits_s.clone().to(cpu)), 0)
all_logits_v = torch.cat(
(all_logits_v, logits_v.clone().to(cpu)), 0)
return [
y_pred_s, y_pred_v, y_true_s, y_true_v, all_logits_s, all_logits_v
] # all have shape of (n,) except logits (n, num_class)
histograms_dic[hist_name+'i'] = []
print("Higher Order Sudo-RM-RF: {} - {} || Epoch: {}/{}".format(
experiment.get_key(), experiment.get_tags(), i+1, hparams['n_epochs']))
model.train()
for data in tqdm(generators['train'], desc='Training'):
opt.zero_grad()
m1wavs = data[0].cuda()
clean_wavs = data[1].cuda()
dataset_indexes = data[2].cuda()
# if hparams['max_abs_snr'] > 0.:
# clean_wavs = mix_with_random_snr(clean_wavs, hparams['max_abs_snr'])
histograms_dic['tr_input_snr'] += (10. * torch.log10(
(clean_wavs[:, 0] ** 2).sum(-1) / (1e-8 + (
clean_wavs[:, 1] ** 2).sum(-1)))).tolist()
# # # Online mixing over samples of the batch. (This might cause to get
# # # utterances from the same speaker but it's highly improbable).
# energies = torch.sum(clean_wavs**2, dim=-1, keepdim=True)
# new_s1 = clean_wavs[:, 0, :]
# new_s2 = clean_wavs[torch.randperm(hparams['batch_size']), 1, :]
# new_s2 = new_s2 * torch.sqrt(energies[:, 1] /
# (new_s2**2).sum(-1, keepdims=True))
#
# m1wavs = normalize_tensor_wav(new_s1 + new_s2)
# clean_wavs[:, 0, :] = normalize_tensor_wav(new_s1)
# clean_wavs[:, 1, :] = normalize_tensor_wav(new_s2)
# # ===============================================
Testgt, maskEnhanced_test = gt1_t
maskInput_test = Variable(maskInput_test.type(Tensor_gpu))
maskEnhanced_test = Variable(maskEnhanced_test.type(Tensor_gpu))
realInput_test = Variable(input_test.type(Tensor_gpu))
realEnhanced_test = Variable(Testgt.type(Tensor_gpu))
fakeEnhanced_test = generatorX(realInput_test)
test_loss = criterion( realEnhanced_test*maskEnhanced_test,torch.clamp(fakeEnhanced_test,0,1)*maskInput_test )
#psnr is okey because answers are from zero to one...we should check clamping in between (?)
psnr = psnr + 10 * torch.log10(1 / (test_loss))
d_test_loss = d_test_loss + torch.mean(discriminatorY(fakeEnhanced_test))-torch.mean(discriminatorY(realEnhanced_test))
psnrAvg = psnr/(j+1)
d_test_lossAvg = d_test_loss /(j+1)
print("Loss loss: %f" % test_loss)
print("DLoss loss: %f" % d_test_lossAvg)
print("PSNR Avg: %f" % (psnrAvg ))
f = open("./models/dtest_loss_trailing.txt", "a+")
f.write("dtest_loss_Avg: %f" % ( d_test_lossAvg ))
f.close()
f = open("./models/psnr_Score_trailing.txt", "a+")
f.write("PSNR Avg: %f" % (psnrAvg ))
f.close()
writer1.add_scalar('PSNR test',psnrAvg,batches_done)
def main(args):
##################################################################################################################
# Setup
##################################################################################################################
# -- Device handling (CPU, GPU)
if args.device is None:
device = torch.device('cpu')
else:
os.environ["CUDA_VISIBLE_DEVICES"] = str(args.device)
device = torch.device('cuda:0')
torch.cuda.set_device(0)
# Seed
random.seed(args.test_seed)
np.random.seed(args.test_seed)
torch.manual_seed(args.test_seed)
# Load XP config
xp_config = load_json(os.path.join(args.xp_dir, 'params.json'))
xp_config.device = device
xp_config.data_dir = args.data_dir
xp_config.xp_dir = args.xp_dir
xp_config.nt_pred = args.nt_pred
xp_config.n_object = 1
##################################################################################################################
# Load test data
##################################################################################################################
print('Loading data...')
test_dataset = load_dataset(xp_config, train=False)
test_loader = DataLoader(test_dataset,
batch_size=args.batch_size,
pin_memory=True)
swap_dataset = SwapDataset(False,
args.data_dir,
xp_config.nt_cond,
seq_len=xp_config.nt_cond + args.nt_pred)
swap_loader = DataLoader(swap_dataset,
batch_size=args.batch_size,
pin_memory=True)
nc = 3
size = 64
##################################################################################################################
# Load model
##################################################################################################################
print('Loading model...')
sep_net = build_model(xp_config)
##################################################################################################################
# Eval
##################################################################################################################
print('Generating samples...')
torch.set_grad_enabled(False)
swap_iterator = iter(swap_loader)
nt_test = xp_config.nt_cond + args.nt_pred
gt_swap = []
content_swap = []
cond_swap = []
target_swap = []
results = defaultdict(list)
# Evaluation is done by batch
for batch in tqdm(test_loader, ncols=80, desc='evaluation'):
# Data
x_cond, x_target, _, x_gt_swap = next(swap_iterator)
x_gt_swap = x_gt_swap.to(device)
x_cond = x_cond.to(device)
# Extraction of S
_, _, s_codes, _ = sep_net.get_forecast(x_cond, nt_test)
# Content swap
x_swap_cond, x_swap_target = batch
x_swap_cond = x_swap_cond.to(device)
x_swap_target = x_swap_target.to(device)
x_swap_cond_byte = x_cond.cpu().mul(255).byte()
x_swap_target_byte = x_swap_target.cpu().mul(255).byte()
cond_swap.append(x_swap_cond_byte.permute(0, 1, 3, 4, 2))
target_swap.append(x_swap_target_byte.permute(0, 1, 3, 4, 2))
x_swap_pred = sep_net.get_forecast(x_swap_cond,
nt_test,
init_s_code=s_codes[:, 0])[0]
x_swap_pred = x_swap_pred[:, xp_config.nt_cond:]
content_swap.append(x_swap_pred.cpu().mul(255).byte().permute(
0, 1, 3, 4, 2))
gt_swap.append(x_gt_swap[:, 0].cpu().mul(255).byte().permute(
0, 1, 3, 4, 2))
# Pixelwise quantitative eval
x_gt_swap = x_gt_swap.view(-1, xp_config.n_object, args.nt_pred, nc,
size, size).to(device)
metrics_batch = {'mse': [], 'psnr': [], 'ssim': []}
for j, reordering in enumerate(
itertools.permutations(range(xp_config.n_object))):
mse = torch.mean(F.mse_loss(x_swap_pred,
x_gt_swap[:, j],
reduction='none'),
dim=[3, 4])
metrics_batch['mse'].append(mse.mean(2).mean(1).cpu())
metrics_batch['psnr'].append(
10 * torch.log10(1 / mse).mean(2).mean(1).cpu())
metrics_batch['ssim'].append(
_ssim_wrapper(x_swap_pred, x_gt_swap[:,
j]).mean(2).mean(1).cpu())
# Compute metrics for best samples and register
results['mse'].append(
torch.min(torch.stack(metrics_batch['mse']), 0)[0])
results['psnr'].append(
torch.max(torch.stack(metrics_batch['psnr']), 0)[0])
results['ssim'].append(
torch.max(torch.stack(metrics_batch['ssim']), 0)[0])
##################################################################################################################
# Print results
##################################################################################################################
print('\n')
print('Results:')
for name in results.keys():
res = torch.cat(results[name]).numpy()
results[name] = res
print(name, res.mean(), '+/-', 1.960 * res.std() / np.sqrt(len(res)))
##################################################################################################################
# Save samples
##################################################################################################################
np.savez_compressed(os.path.join(args.xp_dir, 'results_swap.npz'),
**results)
np.savez_compressed(os.path.join(args.xp_dir, 'content_swap_gt.npz'),
gt_swap=torch.cat(gt_swap).numpy())
np.savez_compressed(os.path.join(args.xp_dir, 'content_swap_test.npz'),
content_swap=torch.cat(content_swap).numpy())
np.savez_compressed(os.path.join(args.xp_dir, 'cond_swap_test.npz'),
cond_swap=torch.cat(cond_swap).numpy())
np.savez_compressed(os.path.join(args.xp_dir, 'target_swap_test.npz'),
target_swap=torch.cat(target_swap).numpy())
def get_psnr2(img1, img2, PIXEL_MAX=1.0):
mse_ = torch.mean((img1 - img2)**2)
return 10 * torch.log10(PIXEL_MAX / mse_)
def calc_psnr_pt(img1, img2, data_range, *args, **kwargs):
err = F.mse_loss(img1, img2)
psnr = 10 * torch.log10((data_range**2) / err)
return psnr
def compute_error(x, y):
Ps = torch.norm(x)
Pn = torch.norm(x-y)
return 20*torch.log10(Ps/Pn)
def PSNR(self, img1, img2):
return 10 * torch.log10(self.max_pixel**2 / self.MSELOSS(img1, img2))
def calculate_psnr_torch(img1, img2):
SE_map = (1. * img1 - img2)**2
cur_MSE = torch.mean(SE_map)
return 20 * torch.log10(1. / torch.sqrt(cur_MSE))
def tensoramp_to_db(self, x):
return 20.0 * torch.log10(torch.clamp(x, min=1e-5))
def test_single_example(model,
datalen,
dataloader,
logger,
log_interval,
index=-1,
show=True):
model.eval()
gpu = torch.device("cuda")
cpu = torch.device("cpu")
if index == -1: # generate a random number
index = np.random.randint(0, datalen)
batchsize = dataloader.batch_size
inter_batch_idx = index // batchsize
intra_batch_idx = index % batchsize
with torch.no_grad():
for batch_id, minibatch in enumerate(dataloader):
if inter_batch_idx == batch_id:
#x0 = minibatch[0].to(gpu) # index in orig data (unused)
x1 = minibatch[1].to(gpu) # encoded_tweets_h
x2 = minibatch[2].to(gpu) # token_type_ids_h
x3 = minibatch[3].to(gpu) # attention_mask_h
x4 = minibatch[4].to(gpu) # encoded_tweets_t
x5 = minibatch[5].to(gpu) # token_type_ids_t
x6 = minibatch[6].to(gpu) # attention_mask_t
x7 = minibatch[7].float() # followers_head
x8 = minibatch[8].float() # followers_tail
x9 = minibatch[9].float() # interaction_type_num
x7 = torch.log10(x7.to(gpu) +
0.1) # log to scale the numbers down to earth
x8 = torch.log10(x8.to(gpu) +
0.1) # log to scale the numbers down to earth
x9 = x9.to(gpu)
y_true_s = minibatch[10] # true label 4 stance class
y_true_v = minibatch[11] # viral_score
outputs = model(input_ids_h=x1,
token_type_ids_h=x2,
attention_mask_h=x3,
input_ids_t=x4,
token_type_ids_t=x5,
attention_mask_t=x6,
followers_head=x7,
followers_tail=x8,
int_type_num=x9,
task='multi')
logits_s = outputs[0].to(cpu)
logits_v = outputs[1].to(cpu)
y_pred_s = logits_s.argmax(1)
y_pred_v = logits_v.argmax(1)
y_true_s = y_true_s[intra_batch_idx].item()
y_pred_s = y_pred_s[intra_batch_idx].item()
y_true_v = y_true_v[intra_batch_idx].item()
y_pred_v = y_pred_v[intra_batch_idx].item()
if show:
text_head = x1.to(cpu)[
intra_batch_idx] # get correct row of data for head
text_tail = x4.to(cpu)[
intra_batch_idx] # get correct row of data for tail
text_head = tokenizer.decode(text_head.tolist(
)) # convert head to list, then decode to text
text_tail = tokenizer.decode(text_tail.tolist(
)) # convert tail to list, then decode to text
text_head = text_head.replace(
' <pad>', '') # remove padding before printing
text_tail = text_tail.replace(
' <pad>', '') # remove padding before printing
logger.info('Original tweet index: ' + str(index))
logger.info('Original head tweet: ' + text_head)
logger.info('Original tail tweet: ' + text_tail)
logger.info(
'Original Stance Label: ' +
tokenizer_v2.convert_label_num2string(y_true_s, 4))
logger.info(
'Predicted Stance Label: ' +
tokenizer_v2.convert_label_num2string(y_pred_s, 4))
logger.info('Original Viral Label: ' + str(y_true_v))
logger.info('Predicted Viral Label: ' + str(y_pred_v))
return [y_true_s, y_pred_s, y_true_v, y_pred_v, index]
def psnr(x1, x2, max_val=1.0):
return 20*torch.log10(max_val/torch.sqrt(torch.mean((x1-x2)**2)))
def calc_psnr(img1, img2):
return 10. * torch.log10(1. / torch.mean((img1 - img2)**2))
def main():
model_file = 'model_epoch9.pth'
batch_size = 4
batch_idx = 2
orig_freq = 44100
target_freq = 16000
seconds = 5
n_fft = 512
win_length = 512
hop_length = 128
freq_bins, spec_time, _ = torch.stft(torch.Tensor(seconds * target_freq),
n_fft, hop_length, win_length).shape
stft = lambda x: torch.stft(x.reshape(x.shape[:-1].numel(), seconds *
target_freq),
n_fft,
hop_length,
win_length,
window=torch.hann_window(n_fft)).reshape(
*x.shape[:-1], freq_bins, spec_time, 2)
istft = lambda X: torchaudio.functional.istft(
X.reshape(X.shape[:-3].numel(), freq_bins, spec_time, 2),
n_fft,
hop_length,
win_length,
window=torch.hann_window(n_fft)).reshape(*X.shape[:-3], waveform_length
)
comp_mul = lambda X, Y: torch.stack((
X.unbind(-1)[0] * Y.unbind(-1)[0] - X.unbind(-1)[1] * Y.unbind(-1)[1],
X.unbind(-1)[0] * Y.unbind(-1)[1] + X.unbind(-1)[1] * Y.unbind(-1)[0]),
dim=-1)
dataset = DSD100('/Volumes/Buffalo 2TB/Datasets/DSD100', 'Test',
seconds * orig_freq)
dataloader = torch.utils.data.DataLoader(dataset,
batch_size=batch_size,
num_workers=8,
shuffle=False)
transforms = [
MixTransform([(0, 1, 2), 3, (0, 1, 2, 3)]),
lambda x: x.reshape(x.shape[0] * 3, seconds * orig_freq),
torchaudio.transforms.Resample(orig_freq, target_freq),
lambda x: torch.stft(
x, n_fft, hop_length, win_length, window=torch.hann_window(n_fft)),
lambda x: x.reshape(x.shape[0] // 3, 3, freq_bins, spec_time, 2),
]
def transform(x):
for t in transforms:
x = t(x)
return x
model = ChimeraMagPhasebook(freq_bins, spec_time, 2, 20, N=600)
model.load_state_dict(torch.load(model_file))
misi_layer = MisiNetwork(n_fft, hop_length, win_length, 1)
#batch = transform(next(iter(dataloader)))
batch = transform(dataset[list(
range(batch_idx * batch_size, (batch_idx + 1) * batch_size))])
S = batch[:, :2, :, :, :]
X = batch[:, 2, :, :, :]
X_abs = torch.sqrt(torch.sum(X**2, dim=-1))
X_phase = X / X_abs.clamp(min=1e-9).unsqueeze(-1)
x = torchaudio.functional.istft(X,
n_fft,
hop_length,
win_length,
window=torch.hann_window(n_fft))
_, (com, ) = model(torch.log10(X_abs.clamp(min=1e-9)), outputs=['com'])
com = com.detach()
Shat = comp_mul(com, X.unsqueeze(1))
shat = misi_layer(Shat, x)
s = torchaudio.functional.istft(S.reshape(batch_size * 2, freq_bins,
spec_time, 2),
n_fft,
hop_length,
win_length,
window=torch.hann_window(n_fft)).reshape(
batch_size, 2, seconds * target_freq)
print(eval_snr(shat, s))
print(eval_si_sdr(shat, s))
shat = shat.transpose(0, 1).reshape(2, batch_size * seconds * target_freq)
s = s.transpose(0, 1).reshape(2, batch_size * seconds * target_freq)
for i_channel, (_s, _shat) in enumerate(zip(s, shat)):
torchaudio.save(f's_{i_channel}.wav', _s, target_freq)
torchaudio.save(f'shat_{i_channel}.wav', _shat, target_freq)
def dynamic_range_compression_torch(x, C=1, clip_val=1e-5):
return torch.log10(torch.clamp(x, min=clip_val) * C)
def psnr(i1, i2):
mse = torch.mean((i1 - i2)**2)
if mse == 0:
return 100
PIXEL_MAX = 1
return 20 * torch.log10(PIXEL_MAX / torch.sqrt(mse))
def predict(input_filepath, file_chunks, output_filepath, model_path,
batch_size, num_workers, rank, device_id):
transducer_model, hidden_size, gru_layers, prev_ite = \
ModelHandler.load_simple_model_for_training(model_path,
input_channels=ImageSizeOptions.IMAGE_CHANNELS,
image_features=ImageSizeOptions.IMAGE_HEIGHT,
seq_len=ImageSizeOptions.SEQ_LENGTH,
num_classes=ImageSizeOptions.TOTAL_LABELS)
transducer_model.eval()
transducer_model = transducer_model.eval()
# create output file
output_filename = output_filepath + "pepper_prediction_" + str(
device_id) + ".hdf"
prediction_data_file = DataStore(output_filename, mode='w')
# data loader
input_data = SequenceDataset(input_filepath, file_chunks)
data_loader = DataLoader(input_data,
batch_size=batch_size,
shuffle=False,
num_workers=num_workers)
torch.cuda.set_device(device_id)
transducer_model.to(device_id)
transducer_model.eval()
transducer_model = DistributedDataParallel(transducer_model,
device_ids=[device_id])
if rank == 0:
progress_bar = tqdm(
total=len(data_loader),
ncols=100,
leave=False,
position=rank,
desc="GPU #" + str(device_id),
)
with torch.no_grad():
for contig, contig_start, contig_end, chunk_id, images, position, index in data_loader:
sys.stderr.flush()
images = images.type(torch.FloatTensor)
hidden = torch.zeros(images.size(0), 2 * TrainOptions.GRU_LAYERS,
TrainOptions.HIDDEN_SIZE)
prediction_base_tensor = torch.zeros(
(images.size(0), images.size(1),
ImageSizeOptions.TOTAL_LABELS))
images = images.to(device_id)
hidden = hidden.to(device_id)
prediction_base_tensor = prediction_base_tensor.to(device_id)
for i in range(0, ImageSizeOptions.SEQ_LENGTH,
TrainOptions.WINDOW_JUMP):
if i + TrainOptions.TRAIN_WINDOW > ImageSizeOptions.SEQ_LENGTH:
break
chunk_start = i
chunk_end = i + TrainOptions.TRAIN_WINDOW
# chunk all the data
image_chunk = images[:, chunk_start:chunk_end]
# run inference
output_base, hidden = transducer_model(image_chunk, hidden)
# now calculate how much padding is on the top and bottom of this chunk so we can do a simple
# add operation
top_zeros = chunk_start
bottom_zeros = ImageSizeOptions.SEQ_LENGTH - chunk_end
# do softmax and get prediction
# we run a softmax a padding to make the output tensor compatible for adding
inference_layers = nn.Sequential(
nn.Softmax(dim=2),
nn.ZeroPad2d((0, 0, top_zeros, bottom_zeros)))
inference_layers = inference_layers.to(device_id)
# run the softmax and padding layers
base_prediction = inference_layers(output_base).to(device_id)
# now simply add the tensor to the global counter
prediction_base_tensor = torch.add(prediction_base_tensor,
base_prediction)
del inference_layers
torch.cuda.empty_cache()
base_values, base_labels = torch.max(prediction_base_tensor, 2)
# this part is for the phred score calculation
counts = torch.ones(
(base_values.size(0),
base_values.size(1) - 2 * ImageSizeOptions.SEQ_OVERLAP))
top_ones = nn.ZeroPad2d(
(ImageSizeOptions.SEQ_OVERLAP, ImageSizeOptions.SEQ_OVERLAP))
counts = top_ones(counts) + 1
base_values = base_labels.cpu().numpy()
phred_score = -10 * torch.log10(1.0 - (base_values / counts))
phred_score[phred_score == float('inf')] = 100
predicted_base_labels = base_labels.cpu().numpy()
phred_score = phred_score.cpu().numpy()
for i in range(images.size(0)):
prediction_data_file.write_prediction(
contig[i], contig_start[i], contig_end[i], chunk_id[i],
position[i], index[i], predicted_base_labels[i],
phred_score[i])
if rank == 0:
progress_bar.update(1)
if rank == 0:
progress_bar.close()
def snr_loss(est, ref, eps=1e-9):
error_element = est - ref
error = torch.sum(error_element**2, dim=-1)
power_ref = torch.sum(ref**2, dim=-1)
return 10. * torch.log10(error + eps + 0.001 * power_ref)
def train(self,
opts,
dloader,
criterion,
l1_init,
l1_dec_step,
l1_dec_epoch,
log_freq,
va_dloader=None,
device='cpu'):
""" Train the SEGAN """
# create writer
self.writer = SummaryWriter(os.path.join(opts.save_path, 'train'))
# Build the optimizers
Gopt, Dopt = self.build_optimizers(opts)
# attach opts to models so that they are saved altogether in ckpts
self.G.optim = Gopt
self.D.optim = Dopt
# Build savers for end of epoch, storing up to 3 epochs each
eoe_g_saver = Saver(self.G,
opts.save_path,
max_ckpts=3,
optimizer=self.G.optim,
prefix='EOE_G-')
eoe_d_saver = Saver(self.D,
opts.save_path,
max_ckpts=3,
optimizer=self.D.optim,
prefix='EOE_D-')
num_batches = len(dloader)
l1_weight = l1_init
iteration = 1
timings = []
evals = {}
noisy_evals = {}
noisy_samples = None
clean_samples = None
z_sample = None
patience = opts.patience
best_val_obj = np.inf
for iteration in range(1, opts.epoch * len(dloader) + 1):
beg_t = timeit.default_timer()
uttname, clean, noisy, slice_idx = self.sample_dloader(
dloader, device)
bsz = clean.size(0)
# grads
Dopt.zero_grad()
D_in = torch.cat((clean, noisy), dim=1)
d_real, _ = self.infer_D(clean, noisy)
rl_lab = torch.ones(d_real.size()).cuda()
if self.vanilla_gan:
cost = F.binary_cross_entropy_with_logits
else:
cost = F.mse_loss
d_real_loss = cost(d_real, rl_lab)
Genh = self.infer_G(noisy, clean)
fake = Genh.detach()
d_fake, _ = self.infer_D(fake, noisy)
fk_lab = torch.zeros(d_fake.size()).cuda()
d_fake_loss = cost(d_fake, fk_lab)
d_weight = 0.5 # count only d_fake and d_real
d_loss = d_fake_loss + d_real_loss
if self.misalign_pair:
clean_shuf = list(torch.chunk(clean, clean.size(0), dim=0))
shuffle(clean_shuf)
clean_shuf = torch.cat(clean_shuf, dim=0)
d_fake_shuf, _ = self.infer_D(clean, clean_shuf)
d_fake_shuf_loss = cost(d_fake_shuf, fk_lab)
d_weight = 1 / 3 # count 3 components now
d_loss += d_fake_shuf_loss
if self.interf_pair:
# put interferring squared signals with random amplitude and
# freq as fake signals mixed with clean data
# TODO: Beware with hard-coded values! possibly improve this
freqs = [250, 1000, 4000]
amps = [0.01, 0.05, 0.1, 1]
bsz = clean.size(0)
squares = []
t = np.linspace(0, 2, 32000)
for _ in range(bsz):
f_ = random.choice(freqs)
a_ = random.choice(amps)
sq = a_ * signal.square(2 * np.pi * f_ * t)
sq = sq[:clean.size(-1)].reshape((1, -1))
squares.append(torch.FloatTensor(sq))
squares = torch.cat(squares, dim=0).unsqueeze(1)
if clean.is_cuda:
squares = squares.to('cuda')
interf = clean + squares
d_fake_inter, _ = self.infer_D(interf, noisy)
d_fake_inter_loss = cost(d_fake_inter, fk_lab)
d_weight = 1 / 4 # count 4 components in d loss now
d_loss += d_fake_inter_loss
d_loss = d_weight * d_loss
d_loss.backward()
Dopt.step()
Gopt.zero_grad()
d_fake_, _ = self.infer_D(Genh, noisy)
g_adv_loss = cost(d_fake_, torch.ones(d_fake_.size()).cuda())
# POWER Loss -----------------------------------
# make stft of gtruth
clean_stft = torch.stft(clean.squeeze(1),
n_fft=min(clean.size(-1), self.n_fft),
hop_length=160,
win_length=320,
normalized=True)
clean_mod = torch.norm(clean_stft, 2, dim=3)
clean_mod_pow = 10 * torch.log10(clean_mod**2 + 10e-20)
Genh_stft = torch.stft(Genh.squeeze(1),
n_fft=min(Genh.size(-1), self.n_fft),
hop_length=160,
win_length=320,
normalized=True)
Genh_mod = torch.norm(Genh_stft, 2, dim=3)
Genh_mod_pow = 10 * torch.log10(Genh_mod**2 + 10e-20)
pow_loss = self.pow_weight * F.l1_loss(Genh_mod_pow, clean_mod_pow)
G_cost = g_adv_loss + pow_loss
if l1_weight > 0:
# look for additive files to build batch mask
mask = torch.zeros(bsz, 1, Genh.size(2))
if opts.cuda:
mask = mask.to('cuda')
for utt_i, uttn in enumerate(uttname):
if 'additive' in uttn:
mask[utt_i, 0, :] = 1.
den_loss = l1_weight * F.l1_loss(Genh * mask, clean * mask)
G_cost += den_loss
else:
den_loss = torch.zeros(1)
G_cost.backward()
Gopt.step()
end_t = timeit.default_timer()
timings.append(end_t - beg_t)
beg_t = timeit.default_timer()
if noisy_samples is None:
noisy_samples = noisy[:20, :, :].contiguous()
clean_samples = clean[:20, :, :].contiguous()
if z_sample is None and not self.G.no_z:
# capture sample now that we know shape after first
# inference
z_sample = self.G.z[:20, :, :].contiguous()
print('z_sample size: ', z_sample.size())
z_sample = z_sample.to(device)
if iteration % log_freq == 0:
log = 'Iter {}/{} ({} bpe) d_loss:{:.4f}, ' \
'g_loss: {:.4f}, pow_loss: {:.4f}, ' \
'den_loss: {:.4f} ' \
''.format(iteration,
len(dloader) * opts.epoch,
len(dloader),
d_loss.item(),
G_cost.item(),
pow_loss.item(),
den_loss.item())
log += 'btime: {:.4f} s, mbtime: {:.4f} s' \
''.format(timings[-1],
np.mean(timings))
print(log)
self.writer.add_scalar('D_loss', d_loss.item(), iteration)
self.writer.add_scalar('G_loss', G_cost.item(), iteration)
self.writer.add_scalar('G_adv_loss', g_adv_loss.item(),
iteration)
self.writer.add_scalar('G_pow_loss', pow_loss.item(),
iteration)
self.writer.add_histogram('clean_mod_pow',
clean_mod_pow.cpu().data,
iteration,
bins='sturges')
self.writer.add_histogram('Genh_mod_pow',
Genh_mod_pow.cpu().data,
iteration,
bins='sturges')
self.writer.add_histogram('Gz',
Genh.cpu().data,
iteration,
bins='sturges')
self.writer.add_histogram('clean',
clean.cpu().data,
iteration,
bins='sturges')
self.writer.add_histogram('noisy',
noisy.cpu().data,
iteration,
bins='sturges')
if hasattr(self.G, 'skips'):
for skip_id, alpha in self.G.skips.items():
skip = alpha['alpha']
if skip.skip_type == 'alpha':
self.writer.add_histogram(
'skip_alpha_{}'.format(skip_id),
skip.skip_k.data,
iteration,
bins='sturges')
# get D and G weights and plot their norms by layer and global
def model_weights_norm(model, total_name):
total_GW_norm = 0
for k, v in model.named_parameters():
if 'weight' in k:
W = v.data
W_norm = torch.norm(W)
self.writer.add_scalar('{}_Wnorm'.format(k),
W_norm, iteration)
total_GW_norm += W_norm
self.writer.add_scalar('{}_Wnorm'.format(total_name),
total_GW_norm, iteration)
model_weights_norm(self.G, 'Gtotal')
model_weights_norm(self.D, 'Dtotal')
if not opts.no_train_gen:
self.gen_train_samples(clean_samples,
noisy_samples,
z_sample,
iteration=iteration)
# BEWARE: There is no evaluation in Whisper SEGAN (WSEGAN)
# TODO: Perhaps add some MCD/F0 RMSE metric
if iteration % len(dloader) == 0:
# save models in end of epoch with EOE savers
self.G.save(self.save_path, iteration, saver=eoe_g_saver)
self.D.save(self.save_path, iteration, saver=eoe_d_saver)
val_results['batch_sizes'] += 1.
lr = val_lr
hr = val_hr
lr = lr.to(device)
hr = hr.to(device)
val_hr_bicubic = val_hr_bicubic.to(device)
sr = generator(lr)
batch_mse = content_criterion(sr, hr)
batch_mse_b = content_criterion(val_hr_bicubic, hr)
val_results['mse'] += batch_mse
val_results['mse_b'] += batch_mse_b
batch_ssim = 1
batch_ssim_b = 1
val_results['ssim'] += batch_ssim
val_results['ssim_b'] += batch_ssim_b
val_results['psnr'] = 10 * torch.log10(1 / (val_results['mse'] / val_results['batch_sizes']))
val_results['psnr_b'] = 10 * torch.log10(1 / (val_results['mse_b'] / val_results['batch_sizes']))
val_bar.set_description(
desc='[converting LR images to SR images] PSNR: %.4f SSIM: %.4f MSE: %.4f PSNR_b: %.4f SSIM_b: %.4f MSE_b: %.4f' % (
val_results['psnr'], val_results['ssim'] / val_results['batch_sizes'], val_results['mse'] / val_results['batch_sizes'], val_results['psnr_b'], val_results['ssim_b'] / val_results['batch_sizes'], val_results['mse_b'] / val_results['batch_sizes']))
# Train the GAN with content + adversary loss:
optim_generator = optim.Adam(generator.parameters(), lr=1e-4)
optim_discriminator = optim.Adam(discriminator.parameters(), lr=1e-4)
train_log = {'lossD':[],'lossG':[], 'lossG_adv':[],'lossG_content':[]}
print ('Stage 2: Train the GAN ...')
stats = {'num_batches':0,'lossG':0,'lossD':0,'lossG_adv':0,'lossG_content':0}
for epoch in range(args.num_epochs):
train_bar = tqdm.tqdm(train_loader)
def forward(self, wav: torch.tensor, eps: float = 1e-7) -> torch.tensor:
# apply mel spectrogram
mel = self.melfunc(wav)
# to log-space
return torch.log10(mel + eps)
def get_psnr(input, target):
"""Computes peak signal-to-noise ratio."""
return 10 * torch.log10(1 / F.mse_loss(input, target))
def calc_PSNR(x1, x2):
x1 = x1 * 255.0
x2 = x2 * 255.0
mse = F.mse_loss(x1, x2)
psnr = -10 * torch.log10(mse) + torch.tensor(48.131)
return psnr
def train(model,
train_dl,
dev_dl,
logger,
log_interval,
epochs,
loss_fn_s,
loss_fn_v,
optimizer,
plotfile,
modelfile,
epochs_giveup=10,
task='multi',
mtt_weight=1.0):
'''
all the params needed are straightforward. except for plotfile, modelfile, ep
model : pytorch neural network model
self explanatory
train_dl : training dataloader
self explanatory
dev_dl : dev set dataloader
self explanatory
logger : python logger
self explanatory
log_interval : int
how many epochs before printing progress
epochs : int
max number of epochs to run
loss_fn_s : pytorch loss function
loss function for stance
loss_fn_v : pytorch loss function
loss function for viral
optimizer : pytorch optimizer
self explanatory
plotfile : string
filename to save plot to
modelfile : string
filename to save model params to
epochs_giveup : int
if this number of epochs pass w/o any improvements to f1 score, give up.
task : string
what task to train on. "multi", "stance" or "viral"
mtt_weight : float
relative weight of viral : stance loss. defaults to 1
Returns
-------
f1 metric stance : tuple
precisions[0:3], recalls[0:3], f1[0:3], supports[0:3]
f1 metric viral : tuple
precisions[0:1], recalls[0:1], f1[0:1], supports[0:1]
accuracy stance : float
self explanatory
accuracy viral : float
self explanatory
message to print : a string to print later
self explanatory
'''
losses_v = []
losses_s = []
losses = []
loss_horz = []
dev_losses_v = []
dev_losses_s = []
dev_losses = []
dev_loss_horz = []
dev_f1_scores_v = []
dev_f1_scores_s = []
dev_f1_scores = []
dev_f1_horz = []
best_f1 = -1
epochs_since_best = 0
gpu = torch.device("cuda")
cpu = torch.device("cpu")
for epoch in range(epochs):
model.train() # set model into training mode
for batch_id, minibatch in enumerate(train_dl):
if batch_id % log_interval == 0:
logger.info(
('\tEPOCH: %3d\tMiniBatch: %4d' % (epoch, batch_id)))
#x0 = minibatch[0].to(gpu) # index in orig data (unused)
x1 = minibatch[1].to(gpu) # encoded_tweets_h
x2 = minibatch[2].to(gpu) # token_type_ids_h
x3 = minibatch[3].to(gpu) # attention_mask_h
x4 = minibatch[4].to(gpu) # encoded_tweets_t
x5 = minibatch[5].to(gpu) # token_type_ids_t
x6 = minibatch[6].to(gpu) # attention_mask_t
x7 = minibatch[7].float() # followers_head
x8 = minibatch[8].float() # followers_tail
x9 = minibatch[9].float() # interaction_type_num
x7 = torch.log10(x7.to(gpu) +
0.1) # log to scale the numbers down to earth
x8 = torch.log10(x8.to(gpu) +
0.1) # log to scale the numbers down to earth
x9 = x9.to(gpu)
y_s = minibatch[10].to(gpu) # true label 4 stance class
y_v = minibatch[11].to(gpu) # viral_score
#print(x7.dtype)
#print(x8.dtype)
#print(x9.dtype)
outputs = model(input_ids_h=x1,
token_type_ids_h=x2,
attention_mask_h=x3,
input_ids_t=x4,
token_type_ids_t=x5,
attention_mask_t=x6,
followers_head=x7,
followers_tail=x8,
int_type_num=x9,
task=task)
logits_s = outputs[0]
logits_v = outputs[1]
if task == 'stance':
loss_v = 0
loss_s = loss_fn_s(logits_s, y_s) # calculate the stance loss
losses_s.append(loss_s.item()) # archive the loss
loss = loss_s
elif task == 'viral':
loss_s = 0
loss_v = loss_fn_v(logits_v, y_v) # calculate the viral loss
losses_v.append(loss_v.item()) # archive the loss
loss = loss_v
elif task == 'multi':
loss_s = loss_fn_s(logits_s, y_s) # calculate the stance loss
losses_s.append(loss_s.item()) # archive the loss
loss_v = loss_fn_v(logits_v, y_v) # calculate the viral loss
losses_v.append(loss_v.item()) # archive the loss
loss = loss_s + mtt_weight * loss_v # sum the losses
loss = loss / (1 + mtt_weight)
else:
err_string = 'task not found : ' + task
logger.info(err_string)
raise Exception(err_string)
loss.backward() # backward prop
optimizer.step() # step the gradients once
optimizer.zero_grad() # clear gradients before next step
loss_value = loss.item() # get value of total loss
losses.append(loss_value) # archive the total loss
# ===================================================================
# not needed to actually free up memory, cauz the procedure exits
# these variables are not returned, so not problematic
# del x1,x2,x3,x4,x5,x6,x7,x8,x9, y_s, y_v
# del loss, outputs, logits_s, logits_v, loss_s, loss_v
# gc.collect()
# ===================================================================
if len(loss_horz) == 0:
loss_horz.append(0)
else:
loss_horz.append(len(loss_horz))
model.eval() # change back to eval mode
results = test(model=model,
dataloader=dev_dl,
logger=logger,
log_interval=log_interval,
print_string='dev')
y_pred_s = results[0]
y_pred_v = results[1]
y_true_s = results[2]
y_true_v = results[3]
logits_s = results[4]
logits_v = results[5]
dev_loss_s = loss_fn_s(logits_s.to(gpu), y_true_s.to(gpu))
dev_loss_v = loss_fn_v(logits_v.to(gpu), y_true_v.to(gpu))
#dev_loss_s = loss_fn_s(logits_s, y_true_s)
#dev_loss_v = loss_fn_v(logits_v, y_true_v)
dev_loss_value_s = dev_loss_s.to(cpu).item()
dev_loss_value_v = dev_loss_v.to(cpu).item()
dev_loss_value = (dev_loss_value_s +
mtt_weight * dev_loss_value_v) / (1 + mtt_weight)
dev_losses_s.append(dev_loss_value_s)
dev_losses_v.append(dev_loss_value_v)
dev_losses.append(dev_loss_value)
dev_loss_horz.append(loss_horz[-1])
f1_metrics_s = f1_help(
y_true_s,
y_pred_s, # calculate f1 scores for stance
average=None, # dont set to calculate for all
labels=[0, 1, 2, 3]) # number of classes
f1_metrics_v = f1_help(
y_true_v,
y_pred_v, # calculate f1 scores for viral
average=None, # dont set to calculate for all
labels=[0, 1]) # number of classes
prec_s, recall_s, f1s_s, supp_s = f1_metrics_s
prec_v, recall_v, f1s_v, supp_v = f1_metrics_v
acc_s = calculate_acc(y_pred_s, y_true_s)
acc_v = calculate_acc(y_pred_v, y_true_v)
msg_s = f1_metrics_msg_stance(prec_s, recall_s, f1s_s, supp_s, acc_s)
msg_v = f1_metrics_msg_viral(prec_v, recall_v, f1s_v, supp_v, acc_v)
logger.info(msg_s + msg_v)
f1_score_s = sum(f1s_s) / len(f1s_s)
f1_score_v = sum(f1s_v) / len(f1s_v)
if task == 'stance':
f1_score = f1_score_s
elif task == 'viral':
f1_score = f1_score_v
else:
f1_score = (f1_score_s + mtt_weight * f1_score_v) / (1 +
mtt_weight)
dev_f1_scores_s.append(f1_score_s)
dev_f1_scores_v.append(f1_score_v)
dev_f1_scores.append(f1_score)
dev_f1_horz.append(epoch)
epochs_since_best += 1
if f1_score > best_f1: # if best f1 score is reached
logger.info('Best results so far. Saving model...')
best_f1 = f1_score # store best score
epochs_since_best = 0 # reset the epochs counter
torch.save(model.state_dict(), modelfile) # save model
if epochs_since_best >= epochs_giveup:
logger.info('No improvements in F1 for %d epochs' %
epochs_since_best)
break # stop training if no improvements for too long
state = torch.load(modelfile) # reload best model
model.load_state_dict(state)
del state
fig, axes = plt.subplots(2, 1)
ax0 = axes[0]
ax1 = axes[1]
if task in ['viral', 'multi']:
ax0.scatter(dev_loss_horz, dev_losses_v, label='viral_dev')
ax1.scatter(dev_f1_horz, dev_f1_scores_v, label='viral_dev')
if task in ['stance', 'multi']:
ax0.scatter(dev_loss_horz, dev_losses_s, label='stance_dev')
ax1.scatter(dev_f1_horz, dev_f1_scores_s, label='stance_dev')
ax0.scatter(dev_loss_horz, dev_losses, label='dev_loss')
ax0.scatter(loss_horz, losses, label='train_loss')
ax1.scatter(dev_f1_horz, dev_f1_scores, label='obj')
#if task in ['viral','multi']: ax0.scatter(dev_loss_horz, dev_losses_v, label='viral')
#if task in ['stance','multi']: ax0.scatter(dev_loss_horz, dev_losses_s, label='stance')
#if task=='multi': ax0.scatter(dev_loss_horz, dev_losses, label='multi')
#ax0.scatter(dev_loss_horz, dev_losses)
ax0.set_ylabel('Training, dev losses')
ax0.set_xlabel('Minibatch')
ax0.legend()
ax0.grid(True)
ax0.set_yscale('log')
#if task in ['viral','multi']: ax1.scatter(dev_f1_horz, dev_f1_scores_v, label='viral')
#if task in ['stance','multi']: ax1.scatter(dev_f1_horz, dev_f1_scores_s, label='stance')
#if task=='multi': ax1.scatter(dev_f1_horz, dev_f1_scores, label='multi')
ax1.legend()
ax1.set_ylabel('Dev F1 score')
ax1.set_xlabel('Epoch')
ax1.grid(True)
plt.tight_layout()
time.sleep(1)
fig.savefig(plotfile)
return [f1_metrics_s, f1_metrics_v, acc_s, acc_v, msg_s + msg_v]
def PSNR(recon,clean):
m = torch.max(clean)
return 10*torch.log10(m**2/MSE(recon,clean))
def log_magnitude(self, x):
x = F.relu(x)
x = 20 * torch.log10(1 + x)
return x
