def one_epoch(args, model, optimizer, epoch, testing, dicts, model_dir,
META_TEST):
"""
Wrapper to do a training epoch and test on dev
"""
if not testing:
set_grad_enabled(True)
losses, unseen_code_inds = train(args, model, optimizer, epoch, dicts)
loss = np.mean(losses)
print("epoch loss: " + str(loss))
else:
loss = np.nan
if model.lmbda > 0:
#still need to get unseen code inds
print("getting set of codes not in training set")
c2ind = dicts['c2ind']
unseen_code_inds = set(dicts['ind2c'].keys())
num_labels = len(dicts['ind2c'])
with open(args.data_path, 'r') as f:
r = csv.reader(f)
#header
next(r)
for row in r:
unseen_code_inds = unseen_code_inds.difference(
set([c2ind[c] for c in row[3].split(';') if c != '']))
print("num codes not in train set: %d" % len(unseen_code_inds))
else:
unseen_code_inds = set()
fold = 'test' if args.version == 'mimic2' else 'dev'
if epoch == args.n_epochs - 1:
print("last epoch: testing on test and train sets")
testing = True
if not META_TEST:
#test on dev
metrics = test(args, model, epoch, fold, unseen_code_inds, dicts,
model_dir, testing)
else:
metrics = defaultdict(float)
if testing or epoch == args.n_epochs - 1:
print("\nevaluating on test")
metrics_te = test(args, model, epoch, "test", unseen_code_inds, dicts,
model_dir, True)
else:
metrics_te = defaultdict(float)
fpr_te = defaultdict(lambda: [])
tpr_te = defaultdict(lambda: [])
metrics_tr = {'loss': loss}
metrics_all = (metrics, metrics_te, metrics_tr)
return metrics_all
def backward(ctx, z_grad, log_s_grad):
F = ctx.F
z, spect, speaker_ids, audio_out = ctx.saved_tensors
audio_0_out, audio_1_out = audio_out.chunk(2, 1)
audio_0_out, audio_1_out = audio_0_out.contiguous(
), audio_1_out.contiguous()
dza, dzb = z_grad.chunk(2, 1)
dza, dzb = dza.contiguous(), dzb.contiguous()
with set_grad_enabled(True):
audio_0 = audio_0_out
audio_0.requires_grad = True
log_s, t = F(audio_0, spect, speaker_ids)
with torch.no_grad():
s = torch.exp(log_s).half(
) # exp not implemented for fp16 therefore this is cast to fp32 by Nvidia/Apex
audio_1 = (audio_1_out -
t) / s # s is fp32 therefore audio_1 is cast to fp32.
z.storage().resize_(reduce(mul, audio_1.shape) * 2) # z is fp16
if z.dtype == torch.float16: # if z is fp16, cast audio_0 and audio_1 back to fp16.
torch.cat((audio_0.half(), audio_1.half()), 1,
out=z) #fp16 # .contiguous()
else:
torch.cat((audio_0, audio_1), 1, out=z) #fp32 # .contiguous()
#z.copy_(xout) # .detach()
with set_grad_enabled(True):
param_list = [audio_0] + list(F.parameters())
if ctx.needs_input_grad[1]:
param_list += [spect]
if ctx.needs_input_grad[2]:
param_list += [speaker_ids]
dtsdxa, *dw = grad(torch.cat((log_s, t), 1),
param_list,
grad_outputs=torch.cat(
(dzb * audio_1 * s + log_s_grad, dzb), 1))
dxa = dza + dtsdxa
dxb = dzb * s
dx = torch.cat((dxa, dxb), 1)
if ctx.needs_input_grad[1]:
*dw, dy = dw
else:
dy = None
if ctx.needs_input_grad[2]:
*dw, ds = dw
else:
ds = None
return (dx, dy, ds, None) + tuple(dw)
def backward(ctx, x_grad, log_s_grad):
F = ctx.F
audio_out, spect, speaker_ids, z = ctx.saved_tensors
audio_0, audio_1 = z.chunk(2, 1)
audio_0, audio_1 = audio_0.contiguous(), audio_1.contiguous()
dxa, dxb = x_grad.chunk(2, 1)
dxa, dxb = dxa.contiguous(), dxb.contiguous()
with set_grad_enabled(True):
audio_0_out = audio_0
audio_0_out.requires_grad = True
log_s, t = F(audio_0_out, spect, speaker_ids)
s = log_s.exp()
with torch.no_grad():
audio_1_out = audio_1 * s + t
audio_out.storage().resize_(reduce(mul, audio_1_out.shape) * 2)
torch.cat((audio_0_out, audio_1_out), 1, out=audio_out)
#audio_out.copy_(zout)
with set_grad_enabled(True):
param_list = [audio_0_out] + list(F.parameters())
if ctx.needs_input_grad[1]:
param_list += [spect]
if ctx.needs_input_grad[2]:
param_list += [speaker_ids]
dtsdza, *dw = grad(
torch.cat((-log_s, -t / s), 1),
param_list,
grad_outputs=torch.cat(
(dxb * audio_1_out / s.detach() + log_s_grad, dxb), 1))
dza = dxa + dtsdza
dzb = dxb / s.detach()
dz = torch.cat((dza, dzb), 1)
if ctx.needs_input_grad[1]:
*dw, dy = dw
else:
dy = None
if ctx.needs_input_grad[2]:
*dw, ds = dw
else:
ds = None
return (dz, dy, ds, None) + tuple(dw)
def backward(ctx, z_grad, log_s_grad):
F = ctx.F
z, spect, speaker_ids, audio_out = ctx.saved_tensors
audio_0_out, audio_1_out = audio_out.chunk(2, 1)
audio_0_out, audio_1_out = audio_0_out.contiguous(
), audio_1_out.contiguous()
dza, dzb = z_grad.chunk(2, 1)
dza, dzb = dza.contiguous(), dzb.contiguous()
with set_grad_enabled(True):
audio_0 = audio_0_out
audio_0.requires_grad = True
log_s, t = F(audio_0, spect, speaker_ids)
with torch.no_grad():
s = log_s.exp()
audio_1 = (audio_1_out - t) / s
z.storage().resize_(reduce(mul, audio_1.shape) * 2)
torch.cat((audio_0, audio_1), 1, out=z) #fp32 # .contiguous()
#torch.cat((audio_0.half(), audio_1.half()), 1, out=z)#fp16 # .contiguous()
#z.copy_(xout) # .detach()
with set_grad_enabled(True):
param_list = [audio_0] + list(F.parameters())
if ctx.needs_input_grad[1]:
param_list += [spect, speaker_ids]
dtsdxa, *dw = grad(torch.cat((log_s, t), 1),
param_list,
grad_outputs=torch.cat(
(dzb * audio_1 * s + log_s_grad, dzb), 1))
dxa = dza + dtsdxa
dxb = dzb * s
dx = torch.cat((dxa, dxb), 1)
if ctx.needs_input_grad[1]:
*dw, dy = dw
else:
dy = None
if ctx.needs_input_grad[2]:
*dw, ds = dw
else:
ds = None
return (dx, dy, ds, None) + tuple(dw)
def backward(ctx, x_grad, log_s_grad):
F = ctx.F
z, y, x = ctx.saved_tensors
xa, xb = x.chunk(2, 1)
xa, xb = xa.contiguous(), xb.contiguous()
dxa, dxb = x_grad.chunk(2, 1)
dxa, dxb = dxa.contiguous(), dxb.contiguous()
with set_grad_enabled(True):
za = xa
za.requires_grad = True
log_s, t = F(za, y)
s = log_s.exp()
with torch.no_grad():
zb = xb * s + t
z.storage().resize_(reduce(mul, zb.shape) * 2)
torch.cat((za, zb), 1, out=z)
#z.copy_(zout)
with set_grad_enabled(True):
param_list = [za] + list(F.parameters())
if ctx.needs_input_grad[1]:
param_list += [y]
dtsdza, *dw = grad(torch.cat((-log_s, -t / s), 1),
param_list,
grad_outputs=torch.cat(
(dxb * zb / s.detach() + log_s_grad, dxb),
1))
dza = dxa + dtsdza
dzb = dxb / s.detach()
dz = torch.cat((dza, dzb), 1)
if ctx.needs_input_grad[1]:
*dw, dy = dw
else:
dy = None
return (dz, dy, None) + tuple(dw)
def backward(ctx, z_grad, log_s_grad):
F = ctx.F
x, y, z = ctx.saved_tensors
za, zb = z.chunk(2, 1)
za, zb = za.contiguous(), zb.contiguous()
dza, dzb = z_grad.chunk(2, 1)
dza, dzb = dza.contiguous(), dzb.contiguous()
with set_grad_enabled(True):
xa = za
xa.requires_grad = True
log_s, t = F(xa, y)
with torch.no_grad():
s = log_s.exp()
xb = (zb - t) / s
x.storage().resize_(reduce(mul, xb.shape) * 2)
torch.cat((xa, xb), 1, out=x) # .contiguous()
#x.copy_(xout) # .detach()
with set_grad_enabled(True):
param_list = [xa] + list(F.parameters())
if ctx.needs_input_grad[1]:
param_list += [y]
dtsdxa, *dw = grad(torch.cat((log_s, t), 1),
param_list,
grad_outputs=torch.cat(
(dzb * xb * s + log_s_grad, dzb), 1))
dxa = dza + dtsdxa
dxb = dzb * s
dx = torch.cat((dxa, dxb), 1)
if ctx.needs_input_grad[1]:
*dw, dy = dw
else:
dy = None
return (dx, dy, None) + tuple(dw)
def test(args, model, epoch, fold, code_inds, dicts, model_dir, testing):
"""
Testing loop.
Returns metrics
"""
filename = args.data_path.replace('train', fold)
print('file for evaluation: %s' % filename)
num_labels = len(dicts['ind2c'])
#initialize stuff for saving attention samples
if args.samples:
tp_file = open('%s/tp_%s_examples_%d.txt' % (model_dir, fold, epoch),
'w')
fp_file = open('%s/fp_%s_examples_%d.txt' % (model_dir, fold, epoch),
'w')
window_size = model.conv.weight.data.size()[2]
y, yhat, yhat_raw, hids, losses = [], [], [], [], []
ind2w, w2ind, ind2c, c2ind = dicts['ind2w'], dicts['w2ind'], dicts[
'ind2c'], dicts['c2ind']
desc_embed = model.lmbda > 0
if desc_embed and len(code_inds) > 0:
unseen_code_vecs(model, code_inds, dicts, args.gpu)
model.eval()
gen = datasets.data_generator(filename,
dicts,
1,
num_labels,
desc_embed=desc_embed,
version=args.version)
for batch_idx, tup in tqdm(enumerate(gen)):
data, target, hadm_ids, _, descs = tup
set_grad_enabled(False)
data, target = Variable(torch.LongTensor(data)), Variable(
torch.FloatTensor(target))
if args.gpu:
data = data.cuda()
target = target.cuda()
model.zero_grad()
if desc_embed:
desc_data = descs
else:
desc_data = None
#get an attention sample for 2% of batches
get_attn = args.samples and (np.random.rand() < 0.02 or
(fold == 'test' and testing))
output, loss, alpha = model(data,
target,
desc_data=desc_data,
get_attention=get_attn)
output = torch.sigmoid(output)
output = output.data.cpu().numpy()
losses.append(loss.item())
target_data = target.data.cpu().numpy()
if get_attn and args.samples:
interpret.save_samples(data,
output,
target_data,
alpha,
window_size,
epoch,
tp_file,
fp_file,
dicts=dicts)
#save predictions, target, hadm ids
yhat_raw.append(output)
output = np.round(output)
y.append(target_data)
yhat.append(output)
hids.extend(hadm_ids)
#close files if needed
if args.samples:
tp_file.close()
fp_file.close()
y = np.concatenate(y, axis=0)
yhat = np.concatenate(yhat, axis=0)
yhat_raw = np.concatenate(yhat_raw, axis=0)
#write the predictions
preds_file = persistence.write_preds(yhat, model_dir, hids, fold, ind2c,
yhat_raw)
#get metrics
k = 5 if num_labels == 50 else [8, 15]
metrics = evaluation.all_metrics(yhat, y, k=k, yhat_raw=yhat_raw)
evaluation.print_metrics(metrics)
metrics['loss_%s' % fold] = np.mean(losses)
return metrics
def refine_mesh(self, mesh, c=None):
''' Refines the predicted mesh. An additional optimization
step to correct the constructed face vertices and normals with the
network prediction (logit and gradient)
Args:
mesh (trimesh object): predicted mesh
occ_hat (tensor): predicted occupancy grid
c (tensor): latent conditioned code c
'''
autograd.set_grad_enabled(True)
self.model.eval()
# Vertex parameter
v0 = torch.FloatTensor(mesh.vertices).to(self.device)
v = torch.nn.Parameter(v0.clone())
# Faces of mesh
faces = torch.LongTensor(mesh.faces)
# detach c; otherwise graph needs to be retained
# caused by new Pytorch version?
if c is not None:
c = c.detach()
# Start optimization
optimizer = optim.RMSprop([v], lr=1e-5)
# Dataset
ds_faces = TensorDataset(faces)
dataloader = DataLoader(ds_faces,
batch_size=self.refine_max_faces,
shuffle=True)
# We updated the refinement algorithm to subsample faces; this is
# usefull when using a high extraction resolution / when working on
# small GPUs
it_r = 0
while it_r < self.refinement_step:
for f_it in dataloader:
f_it = f_it[0].to(self.device)
optimizer.zero_grad()
# Loss
face_vertex = v[f_it]
eps = np.random.dirichlet((0.5, 0.5, 0.5), size=f_it.shape[0])
eps = torch.FloatTensor(eps).to(self.device)
face_point = (face_vertex * eps[:, :, None]).sum(dim=1)
face_v1 = face_vertex[:, 1, :] - face_vertex[:, 0, :]
face_v2 = face_vertex[:, 2, :] - face_vertex[:, 1, :]
face_normal = torch.cross(face_v1, face_v2)
face_normal = face_normal / \
(face_normal.norm(dim=1, keepdim=True) + 1e-10)
face_value = torch.cat([
self.model.decode(p_split, c).sdf for p_split in
torch.split(face_point.unsqueeze(0), 20000, dim=1)
],
dim=1)
normal_target = -autograd.grad(
[face_value.sum()], [face_point], create_graph=True)[0]
normal_target = \
normal_target / \
(normal_target.norm(dim=1, keepdim=True) + 1e-10)
loss_target = face_value.pow(2).mean()
loss_normal = \
(face_normal - normal_target).pow(2).sum(dim=1).mean()
loss = loss_target + 0.01 * loss_normal
# Update
loss.backward()
optimizer.step()
# Update it_r
it_r += 1
if it_r >= self.refinement_step:
break
mesh.vertices = v.data.cpu().numpy()
return mesh
# both sizes are (100, 115k)
best_distances, indices = torch.topk(joint_distances, k=K, dim=0)
# print(best_distances.shape, indices.shape)
# size (100, 115k)
best_indices = torch.gather(joint_indices, dim=0, index=indices)
# print(best_indices.shape)
return best_indices, best_distances
if __name__ == "__main__":
if not osp.exists(FEATURES_DIR):
os.makedirs(FEATURES_DIR)
set_grad_enabled(False)
""" Iterates through all train files. """
test_data = np.load(FEATURES_TEST_FILE)
test_names, test_vectors = test_data["images"], test_data["features"]
print(test_names)
print(test_vectors)
print("first vector:")
print("length", test_vectors[0].size, "non-zeros", np.count_nonzero(test_vectors[0]))
# note transpose
test_vectors = torch.tensor(test_vectors.T, dtype=torch.float).cuda(async=True)
print("test_names:", test_names.shape)
print("test_vectors:", test_vectors.shape)
