def test(epoch):
model.eval()
accuracies = []
for idx, data in tqdm.tqdm(enumerate(testloader), total=len(testloader)):
data = [Variable(d, volatile=True).cuda() for d in data]
_, [attr_pred, obj_pred, _] = model(data)
match_stats = utils.performance_stats(attr_pred, obj_pred, data)
accuracies.append(match_stats)
accuracies = zip(*accuracies)
accuracies = map(torch.mean, map(torch.cat, accuracies))
attr_acc, obj_acc, closed_acc, open_acc, objoracle_acc = accuracies
log_value('test_attr_acc', attr_acc, epoch)
log_value('test_obj_acc', obj_acc, epoch)
log_value('test_closed_acc', closed_acc, epoch)
log_value('test_open_acc', open_acc, epoch)
log_value('test_objoracle_acc', objoracle_acc, epoch)
print '(test) E: %d | A: %.3f | O: %.3f | Cl: %.3f | Op: %.4f | OrO: %.4f' % (
epoch, attr_acc, obj_acc, closed_acc, open_acc, objoracle_acc)
if epoch > 0 and epoch % args.save_every == 0:
state = {
'net': model.state_dict(),
'epoch': epoch,
}
torch.save(
state,
args.cv_dir + '/ckpt_E_%d_A_%.3f_O_%.3f_Cl_%.3f_Op_%.3f.t7' %
(epoch, attr_acc, obj_acc, closed_acc, open_acc))
def test(epoch):
agent.eval()
matches, rewards, policies = [], [], []
for batch_idx, (inputs, targets) in tqdm.tqdm(enumerate(testloader),
total=len(testloader)):
inputs, targets = Variable(
inputs, volatile=True), Variable(targets).cuda(async=True)
if not args.parallel:
inputs = inputs.cuda()
# Get the low resolution agent images
inputs_agent = inputs.clone()
inputs_agent = torch.nn.functional.interpolate(
inputs_agent, (args.lr_size, args.lr_size))
probs = F.sigmoid(
agent.forward(inputs_agent,
args.model.split('_')[1], 'lr'))
# Sample the test-time policy
policy = probs.data.clone()
policy[policy < 0.5] = 0.0
policy[policy >= 0.5] = 1.0
# Get the masked high-res image and perform inference
inputs = utils.agent_chosen_input(inputs, policy, mappings, patch_size)
preds = rnet.forward(inputs, args.model.split('_')[1], 'hr')
reward, match = utils.compute_reward(preds, targets, policy.data,
args.penalty)
matches.append(match)
rewards.append(reward)
policies.append(policy.data)
accuracy, reward, sparsity, variance, policy_set = utils.performance_stats(
policies, rewards, matches)
print('Test - Acc: %.3f | Rw: %.2E | S: %.3f | V: %.3f | #: %d' %
(accuracy, reward, sparsity, variance, len(policy_set)))
log_value('test_accuracy', accuracy, epoch)
log_value('test_reward', reward, epoch)
log_value('test_sparsity', sparsity, epoch)
log_value('test_variance', variance, epoch)
log_value('test_unique_policies', len(policy_set), epoch)
# Save the Policy Network - Classifier is fixed in this phase
agent_state_dict = agent.module.state_dict(
) if args.parallel else agent.state_dict()
state = {
'agent': agent_state_dict,
'epoch': epoch,
'reward': reward,
'acc': accuracy
}
torch.save(
state,
args.cv_dir + '/ckpt_E_%d_A_%.3f_R_%.2E' % (epoch, accuracy, reward))
def test(epoch):
agent.eval()
rewards, metrics, policies, set_labels = [], [], [], []
for batch_idx, (inputs, targets) in tqdm.tqdm(enumerate(testloader),
total=len(testloader)):
inputs = Variable(inputs, volatile=True)
if not args.parallel:
inputs = inputs.cuda()
# Actions by the Policy Network
probs = F.sigmoid(agent(inputs))
# Sample the policy from the agents output
policy = probs.data.clone()
policy[policy < 0.5] = 0.0
policy[policy >= 0.5] = 1.0
policy = Variable(policy)
offset_fd, offset_cd = utils.read_offsets(targets, num_actions)
reward = utils.compute_reward(offset_fd, offset_cd, policy.data,
args.beta, args.sigma)
metrics, set_labels = utils.get_detected_boxes(policy, targets,
metrics, set_labels)
rewards.append(reward)
policies.append(policy.data)
true_positives, pred_scores, pred_labels = [
np.concatenate(x, 0) for x in list(zip(*metrics))
]
precision, recall, AP, f1, ap_class = utils_detector.ap_per_class(
true_positives, pred_scores, pred_labels, set_labels)
reward, sparsity, variance, policy_set = utils.performance_stats(
policies, rewards)
print('Test - AP: %.3f | AR : %.3f' % (AP[0], recall.mean()))
print('Test - Rw: %.2E | S: %.3f | V: %.3f | #: %d' %
(reward, sparsity, variance, len(policy_set)))
log_value('test_reward', reward, epoch)
log_value('test_AP', AP[0], epoch)
log_value('test_AR', recall.mean(), epoch)
log_value('test_sparsity', sparsity, epoch)
log_value('test_variance', variance, epoch)
log_value('test_unique_policies', len(policy_set), epoch)
# save the model --- agent
agent_state_dict = agent.module.state_dict(
) if args.parallel else agent.state_dict()
state = {
'agent': agent_state_dict,
'epoch': epoch,
'reward': reward,
}
torch.save(state, args.cv_dir + '/ckpt_E_%d_R_%.2E' % (epoch, reward))
def evaluate_svms():
attr_clfs = [
pickle.load(open('%s/svm/attr_%d' % (args.data_dir, attr)))
for attr in range(len(dataset.attrs))
]
obj_clfs = [
pickle.load(open('%s/svm/obj_%d' % (args.data_dir, obj)))
for obj in range(len(dataset.objs))
]
# Calibrate all classifiers first
Y = [(train_attrs == attr).astype(np.int)
for attr in range(len(dataset.attrs))]
for attr in tqdm.tqdm(range(len(dataset.attrs))):
clf = attr_clfs[attr]
calibrated = CalibratedClassifierCV(clf, method='sigmoid', cv='prefit')
calibrated.fit(X_train, Y[attr])
attr_clfs[attr] = calibrated
Y = [(train_objs == obj).astype(np.int)
for obj in range(len(dataset.objs))]
for obj in tqdm.tqdm(range(len(dataset.objs))):
clf = obj_clfs[obj]
calibrated = CalibratedClassifierCV(clf, method='sigmoid', cv='prefit')
calibrated.fit(X_train, Y[obj])
obj_clfs[obj] = calibrated
# Generate all the scores
attr_scores, obj_scores = [], []
for attr in tqdm.tqdm(range(len(dataset.attrs))):
clf = attr_clfs[attr]
score = clf.predict_proba(X_test)[:, 1]
attr_scores.append(score)
attr_scores = np.vstack(attr_scores)
for obj in tqdm.tqdm(range(len(dataset.objs))):
clf = obj_clfs[obj]
score = clf.predict_proba(X_test)[:, 1]
obj_scores.append(score)
obj_scores = np.vstack(obj_scores)
attr_pred = torch.from_numpy(attr_scores).transpose(0, 1)
obj_pred = torch.from_numpy(obj_scores).transpose(0, 1)
x = [
None,
Variable(torch.from_numpy(test_attrs)).long(),
Variable(torch.from_numpy(test_objs)).long(),
Variable(torch.from_numpy(test_pairs)).long()
]
attr_pred, obj_pred, _ = utils.generate_prediction_tensors(
[attr_pred, obj_pred], dataset, x[2].data, source='classification')
attr_match, obj_match, zsl_match, gzsl_match, fixobj_match = utils.performance_stats(
attr_pred, obj_pred, x)
print attr_match.mean(), obj_match.mean(), zsl_match.mean(
), gzsl_match.mean(), fixobj_match.mean()
def train(epoch):
agent.train()
matches, rewards, rewards_baseline, policies = [], [], [], []
for batch_idx, (inputs, targets) in tqdm.tqdm(enumerate(trainloader), total=len(trainloader)):
inputs = Variable(inputs)
if not args.parallel:
inputs = inputs.cuda()
# Actions by the Agent
probs = F.sigmoid(agent.forward(inputs))
alpha_hp = np.clip(args.alpha + epoch * 0.001, 0.6, 0.95)
probs = probs*alpha_hp + (1-alpha_hp) * (1-probs)
# Sample the policies from the Bernoulli distribution characterized by agent
distr = Bernoulli(probs)
policy_sample = distr.sample()
# Test time policy - used as baseline policy in the training step
policy_map = probs.data.clone()
policy_map[policy_map<0.5] = 0.0
policy_map[policy_map>=0.5] = 1.0
policy_map = Variable(policy_map)
# Get the batch wise metrics
offset_fd, offset_cd = utils.read_offsets(targets, num_actions)
# Find the reward for baseline and sampled policy
reward_map = utils.compute_reward(offset_fd, offset_cd, , policy_map.data, args.beta, args.sigma)
reward_sample = utils.compute_reward(offset_fd, offset_cd, policy_sample.data, args.beta, args.sigma)
advantage = reward_sample.cuda().float() - reward_map.cuda().float()
# Find the loss for only the policy network
loss = -distr.log_prob(policy_sample)
loss = loss * Variable(advantage).expand_as(policy_sample)
loss = loss.mean()
optimizer.zero_grad()
loss.backward()
optimizer.step()
rewards.append(reward_sample.cpu())
rewards_baseline.append(reward_map.cpu())
policies.append(policy_sample.data.cpu())
reward, sparsity, variance, policy_set = utils.performance_stats(policies, rewards)
# Compute the Precision and Recall Performance of the Agent and Detectors
print 'Train: %d | Rw: %.2E | S: %.3f | V: %.3f | #: %d'%(epoch, reward, sparsity, variance, len(policy_set))
log_value('train_reward', reward, epoch)
log_value('train_sparsity', sparsity, epoch)
log_value('train_variance', variance, epoch)
log_value('train_baseline_reward', torch.cat(rewards_baseline, 0).mean(), epoch)
log_value('train_unique_policies', len(policy_set), epoch)
def test(epoch):
model.eval()
accuracies = []
for idx, data in tqdm.tqdm(enumerate(testloader), total=len(testloader)):
data = [Variable(d, volatile=True).cuda() for d in data]
_, [attr_pred, obj_pred, _] = model(data)
match_stats = utils.performance_stats(attr_pred, obj_pred, data)
accuracies.append(match_stats)
accuracies = zip(*accuracies)
accuracies = map(torch.mean, map(torch.cat, accuracies))
attr_acc, obj_acc, closed_acc, open_acc, objoracle_acc = accuracies
print '(test) E: %d | A: %.3f | O: %.3f | Cl: %.3f | Op: %.4f | OrO: %.4f' % (
epoch, attr_acc, obj_acc, closed_acc, open_acc, objoracle_acc)
def evaluate_tensorcompletion():
def parse_tensor(fl):
tensor = scipy.io.loadmat(fl)
nz_idx = zip(*(tensor['subs']))
composite_clfs = np.zeros(
(len(dataset.attrs), len(dataset.objs), X.shape[1]))
composite_clfs[nz_idx[0], nz_idx[1],
nz_idx[2]] = tensor['vals'].squeeze()
return composite_clfs, nz_idx, tensor['vals'].squeeze()
# see recon error
tr_file = 'tensor-completion/incomplete/%s.mat' % args.dataset
ts_file = args.completed
tr_clfs, tr_nz_idx, tr_vals = parse_tensor(tr_file)
ts_clfs, ts_nz_idx, ts_vals = parse_tensor(ts_file)
print tr_vals.min(), tr_vals.max(), tr_vals.mean()
print ts_vals.min(), ts_vals.max(), ts_vals.mean()
print 'Completed Tensor: %s' % args.completed
# see train recon error
err = 1.0 * ((tr_clfs[tr_nz_idx[0], tr_nz_idx[1], tr_nz_idx[2]] -
ts_clfs[tr_nz_idx[0], tr_nz_idx[1], tr_nz_idx[2]])**
2).sum() / (len(tr_vals))
print 'recon error:', err
# Create and scale classifiers for each pair
clfs = {}
test_pair_set = set(map(tuple, dataset.test_pairs.numpy().tolist()))
for idx, (attr, obj) in tqdm.tqdm(enumerate(dataset.pairs),
total=len(dataset.pairs)):
clf = LinearSVC(fit_intercept=False)
clf.fit(np.eye(2), [0, 1])
if (attr, obj) in test_pair_set:
X_ = X_test
Y_ = (test_attrs == attr).astype(np.int) * (test_objs
== obj).astype(np.int)
clf.coef_ = ts_clfs[attr, obj][None, :]
else:
X_ = X_train
Y_ = (train_attrs == attr).astype(np.int) * (train_objs
== obj).astype(np.int)
clf.coef_ = tr_clfs[attr, obj][None, :]
calibrated = CalibratedClassifierCV(clf, method='sigmoid', cv='prefit')
calibrated.fit(X_, Y_)
clfs[(attr, obj)] = calibrated
scores = {}
for attr, obj in tqdm.tqdm(dataset.pairs):
score = clfs[(attr, obj)].predict_proba(X_test)[:, 1]
scores[(attr, obj)] = torch.from_numpy(score).float().unsqueeze(1)
x = [
None,
Variable(torch.from_numpy(test_attrs)).long(),
Variable(torch.from_numpy(test_objs)).long(),
Variable(torch.from_numpy(test_pairs)).long()
]
attr_pred, obj_pred, _ = utils.generate_prediction_tensors(
scores, dataset, x[2].data, source='manifold')
attr_match, obj_match, zsl_match, gzsl_match, fixobj_match = utils.performance_stats(
attr_pred, obj_pred, x)
print attr_match.mean(), obj_match.mean(), zsl_match.mean(
), gzsl_match.mean(), fixobj_match.mean()
def train(epoch):
agent.train()
rnet_hr.train()
matches, rewards, rewards_baseline, policies = [], [], [], []
for batch_idx, (inputs, targets) in tqdm.tqdm(enumerate(trainloader),
total=len(trainloader)):
inputs, targets = Variable(inputs), Variable(targets).cuda(async=True)
if not args.parallel:
inputs = inputs.cuda()
# Get the low resolution agent images
inputs_agent = inputs.clone()
inputs_agent = torch.nn.functional.interpolate(
inputs_agent, (args.lr_size, args.lr_size))
probs = F.sigmoid(
agent.forward(inputs_agent,
args.model.split('_')[1], 'lr'))
probs = probs * args.alpha + (1 - probs) * (1 - args.alpha)
# sample the policies from the Bernoulli distribution characterized by agent's output
distr = Bernoulli(probs)
policy_sample = distr.sample()
# Test time policy - used as baseline policy in the training step
policy_map = probs.data.clone()
policy_map[policy_map < 0.5] = 0.0
policy_map[policy_map >= 0.5] = 1.0
# Sample the high resolution patches using the actions
inputs_map = inputs.clone()
inputs_sample = inputs.clone()
inputs_map = utils.agent_chosen_input(inputs_map, policy_map, mappings,
patch_size)
inputs_sample = utils.agent_chosen_input(inputs_sample,
policy_sample.int(), mappings,
patch_size)
# Perform inference and combine low and high resolution classifier
preds_lr = rnet_lr.forward(inputs_agent,
args.model.split('_')[1], 'lr')
preds_map = rnet_hr.forward(inputs_map, args.model.split('_')[1], 'hr')
preds_sample = rnet_hr.forward(inputs_sample,
args.model.split('_')[1], 'hr')
preds_map = preds_map + preds_lr
preds_sample = preds_sample + preds_lr
# Get the rewards for baseline and sampled policy
reward_map, match = utils.compute_reward(preds_map, targets,
policy_map.data, args.penalty)
reward_sample, _ = utils.compute_reward(preds_sample, targets,
policy_sample.data,
args.penalty)
advantage = reward_sample - reward_map
# Find the joint loss from combined classifier and agent
loss = -distr.log_prob(policy_sample).sum(
1, keepdim=True) * Variable(advantage)
loss = loss.mean()
loss += F.cross_entropy(preds_sample, targets)
optimizer.zero_grad()
loss.backward()
optimizer.step()
matches.append(match.cpu())
rewards.append(reward_sample.cpu())
rewards_baseline.append(reward_map.cpu())
policies.append(policy_sample.data.cpu())
accuracy, reward, sparsity, variance, policy_set = utils.performance_stats(
policies, rewards, matches)
print('Train: %d | Acc: %.3f | Rw: %.2E | S: %.3f | V: %.3f | #: %d' %
(epoch, accuracy, reward, sparsity, variance, len(policy_set)))
log_value('train_accuracy', accuracy, epoch)
log_value('train_reward', reward, epoch)
log_value('train_sparsity', sparsity, epoch)
log_value('train_variance', variance, epoch)
log_value('train_baseline_reward',
torch.cat(rewards_baseline, 0).mean(), epoch)
log_value('train_unique_policies', len(policy_set), epoch)
def train(epoch):
# This steps trains the policy network only
agent.train()
matches, rewards, rewards_baseline, policies = [], [], [], []
for batch_idx, (inputs, targets) in tqdm.tqdm(enumerate(trainloader),
total=len(trainloader)):
inputs, targets = Variable(inputs), Variable(targets).cuda(async=True)
if not args.parallel:
inputs = inputs.cuda()
inputs_agent = inputs.clone()
inputs_map = inputs.clone()
inputs_sample = inputs.clone()
# Run the low-res image through Policy Network
inputs_agent = torch.nn.functional.interpolate(
inputs_agent, (args.lr_size, args.lr_size))
probs = F.sigmoid(
agent.forward(inputs_agent,
args.model.split('_')[1], 'lr'))
probs = probs * args.alpha + (1 - args.alpha) * (1 - probs)
# Sample the policies from the Bernoulli distribution characterized by agent's output
distr = Bernoulli(probs)
policy_sample = distr.sample()
# Test time policy - used as baseline policy in the training step
policy_map = probs.data.clone()
policy_map[policy_map < 0.5] = 0.0
policy_map[policy_map >= 0.5] = 1.0
# Agent sampled high resolution images
inputs_map = utils.agent_chosen_input(inputs_map, policy_map, mappings,
patch_size)
inputs_sample = utils.agent_chosen_input(inputs_sample,
policy_sample.int(), mappings,
patch_size)
# Forward propagate images through the classifiers
preds_map = rnet.forward(inputs_map, args.model.split('_')[1], 'hr')
preds_sample = rnet.forward(inputs_sample,
args.model.split('_')[1], 'hr')
# Find the reward for baseline and sampled policy
reward_map, match = utils.compute_reward(preds_map, targets,
policy_map.data, args.penalty)
reward_sample, _ = utils.compute_reward(preds_sample, targets,
policy_sample.data,
args.penalty)
advantage = reward_sample.cuda().float() - reward_map.cuda().float()
# Find the loss for only the policy network
loss = -distr.log_prob(policy_sample)
loss = loss * Variable(advantage).expand_as(policy_sample)
loss = loss.mean()
optimizer.zero_grad()
loss.backward()
optimizer.step()
matches.append(match.cpu())
rewards.append(reward_sample.cpu())
rewards_baseline.append(reward_map.cpu())
policies.append(policy_sample.data.cpu())
accuracy, reward, sparsity, variance, policy_set = utils.performance_stats(
policies, rewards, matches)
print('Train: %d | Acc: %.3f | Rw: %.2E | S: %.3f | V: %.3f | #: %d' %
(epoch, accuracy, reward, sparsity, variance, len(policy_set)))
log_value('train_accuracy', accuracy, epoch)
log_value('train_reward', reward, epoch)
log_value('train_sparsity', sparsity, epoch)
log_value('train_variance', variance, epoch)
log_value('train_baseline_reward',
torch.cat(rewards_baseline, 0).mean(), epoch)
log_value('train_unique_policies', len(policy_set), epoch)
