def get_phrase_scores_cls(split=None):
cub_dataset = CUBDataset(split=split)
data_loader = DataLoader(cub_dataset, batch_size=128, shuffle=False)
model, device = cls_load_model()
model.eval()
dataset_pred_scores = []
for _, imgs, _, _ in tqdm(data_loader, desc='predicting phrase scores over images'):
with torch.no_grad():
pred_scores = model(imgs.to(device)).to('cpu')
dataset_pred_scores.append(pred_scores)
pred_scores = torch.cat(dataset_pred_scores)
return pred_scores
def rand_tune_tree(num):
cub_dataset = CUBDataset(split=None, val_ratio=0.1)
model_names = list()
best_score = 0
best_model = ''
# settings = {'criterion': ['entropy', 'gini'],
# 'splitter': ['best', 'random'],
# 'class_weight': ['none', 'balanced'],
# 'max_depth': range(10, 52, 5),
# 'min_samples_split': range(2, 101, 5),
# 'min_samples_leaf': range(1, 101, 5)}
settings = [['entropy', 'gini'], ['best', 'random'], ['none', 'balanced'],
range(2, 51, 1),
range(2, 11, 1),
range(1, 11, 1)]
while len(model_names) < num:
model_name = 'DecisionTreeClassifier'
for i, ss in enumerate(settings):
s = np.random.choice(ss)
if i < 3:
model_name += '_' + s
elif i == 3:
model_name += '_D%d' % s
elif i == 4:
model_name += '_S%d' % s
elif i == 5:
model_name += '_L%d' % s
if model_name not in model_names:
model_names.append(model_name)
print('trial %d: %s' % (len(model_names), model_name))
score = classify(cub_dataset=cub_dataset,
feat_mode='ph_cls',
model_name=model_name,
norm=False,
val_ratio=0.1)
if score > best_score:
best_score = score
best_model = model_name
print('NEW BEST!')
print(best_model, best_score)
return
def get_phrase_scores_tri(split=None):
phrase_dataset = PhraseOnlyDataset()
phrase_dataloader = DataLoader(phrase_dataset, batch_size=128, shuffle=False)
cub_dataset = CUBDataset(split=split)
img_dataloader = DataLoader(cub_dataset, batch_size=1, shuffle=False)
model, device = tri_load_model()
neg_distances = torch.zeros((len(cub_dataset), len(phrase_dataset)))
phrase_vecs = list()
with torch.no_grad():
for phrases in tqdm(phrase_dataloader, desc='getting phrase_vecs in batches'):
batch_phrase_vecs = model.lang_encoder(phrases).to('cpu')
phrase_vecs.append(batch_phrase_vecs)
phrase_vecs = torch.cat(phrase_vecs) # phrase_num x vec_dim
img_i = 0
with torch.no_grad():
for _, imgs, _, _ in tqdm(img_dataloader, desc='getting img_vecs and distances'):
img_vec = model.img_encoder(imgs.to(device)).to('cpu')[0]
for ph_i in range(len(phrase_dataset)):
ph_vec = phrase_vecs[ph_i]
neg_distances[img_i, ph_i] = - model.dist_fn(img_vec, ph_vec)
img_i += 1
return neg_distances
def predict_attributes():
dataset = CUBDataset(split=None, val_ratio=0)
data_loader = DataLoader(dataset, batch_size=64, shuffle=False)
model: AttClassifier = AttClassifier(class_num=len(dataset.att_names),
pretrained_backbone=True,
fc_dims=[
512,
],
use_feats=[2, 4])
device = 'cuda:0'
model.to(device)
model.load_state_dict(
torch.load(
'applications/fine_grained_classification/att_pred_1/checkpoints/BEST_checkpoint.pth'
))
model.eval()
dataset_pred_scores = []
for _, imgs, _, labels in tqdm(data_loader, desc='pred scores on images'):
with torch.no_grad():
pred_scores = model(imgs.to(device))
dataset_pred_scores.append(pred_scores)
pred_scores = torch.cat(
dataset_pred_scores).cpu().numpy() # img_num x class_num
np.save('applications/fine_grained_classification/att_scores.npy',
pred_scores)
print('att_scores saved.')
correct = dataset.gt_att_labels == (pred_scores > 0)
correct_train = correct[dataset.img_splits['train'], :]
acc_train = np.sum(correct_train) / np.size(correct_train)
correct_test = correct[dataset.img_splits['test'], :]
acc_test = np.sum(correct_test) / np.size(correct_test)
print('acc on train_val: %.4f; on test: %.4f' %
(acc_train, acc_test)) # 0.9658, 0.9036
return
def classify(cub_dataset=None,
val_ratio=0.1,
feat_mode='ph_cls',
model_name='LogisticRegression_lbfgs_multinomial',
norm=True,
ks=None):
if norm:
norm_str = '_norm'
else:
norm_str = ''
exp_name = '%s_%s%s_val%.1f' % (model_name, feat_mode, norm_str, val_ratio)
print(exp_name)
if cub_dataset is None or cub_dataset.val_ratio != val_ratio:
cub_dataset = CUBDataset(split=None, val_ratio=val_ratio)
labels = [img['class_label'] for img in cub_dataset.img_data_list]
def att_feat_filter_rank(att_feat):
att_vars = np.var(cub_dataset.class_att_labels, axis=0)
att_idxs_sorted = np.argsort(att_vars * -1)
att_type_str = feat_mode.split('_')[-1]
att_idxs_allowed = set()
if 's' in att_type_str:
att_idxs_allowed.update(cub_dataset.att_types['shape'])
if 'c' in att_type_str:
att_idxs_allowed.update(cub_dataset.att_types['color'])
if 'p' in att_type_str:
att_idxs_allowed.update(cub_dataset.att_types['pattern'])
att_idxs_filtered = [
i for i in att_idxs_sorted if i in att_idxs_allowed
]
att_feat = att_feat[:, att_idxs_filtered]
return att_feat
feat_start_k = 0
if feat_mode.startswith('ph_cls'):
feats = np.load(
'applications/fine_grained_classification/phrase_scores_cls.npy')
if feat_mode.endswith('sig'):
feats = 1 / (1 + np.exp(-feats))
elif feat_mode == 'ph_tri':
feats = np.sqrt(-np.load(
'applications/fine_grained_classification/phrase_scores_tri.npy'))
elif feat_mode.startswith('att_gt'):
feats = np.array([d['att_labels'] for d in cub_dataset.img_data_list])
feats = att_feat_filter_rank(feats)
elif feat_mode.startswith('att_pred'):
feats = np.load(
'applications/fine_grained_classification/att_scores.npy')
feats = att_feat_filter_rank(feats)
elif feat_mode.startswith('joint'):
modes = feat_mode.split('_')
if 'gt' in modes[1]: # att_gt
feat1 = np.array(
[d['att_labels'] for d in cub_dataset.img_data_list])
feat1 = att_feat_filter_rank(feat1)
else: # att_pred
feat1 = np.load(
'applications/fine_grained_classification/att_scores.npy')
feat1 = att_feat_filter_rank(feat1)
if 'cls' in modes[2]: # ph_cls
feat2 = np.load(
'applications/fine_grained_classification/phrase_scores_cls.npy'
)
else: # ph_tri
feat2 = np.sqrt(-np.load(
'applications/fine_grained_classification/phrase_scores_tri.npy'
))
feats = np.concatenate((feat1, feat2), axis=1)
feat_start_k = feat1.shape[1]
# print(feats.shape)
else:
raise NotImplementedError
if val_ratio > 0:
eval_split = 'val'
else:
eval_split = 'test'
test_feats = np.array(
[feats[i] for i in cub_dataset.img_splits[eval_split]])
test_labels = np.array(
[labels[i] for i in cub_dataset.img_splits[eval_split]])
train_feats = np.array([feats[i] for i in cub_dataset.img_splits['train']])
train_labels = np.array(
[labels[i] for i in cub_dataset.img_splits['train']])
# print(test_feats.shape)
# print(test_labels.shape)
# print(train_feats.shape)
# print(train_labels.shape)
f = open('applications/fine_grained_classification/logs/%s.txt' % exp_name,
'w')
score = 0
score_all = []
if ks is None:
if eval_split == 'val':
ks = [100]
elif feat_mode.startswith('ph') or feat_mode.startswith('joint'):
ks = list(range(1, 50, 5)) + list(range(51, 656, 25)) + [655]
elif feat_mode.startswith('att'):
ks = list(range(1, 50, 2)) + list(range(51, 313, 25)) + [312]
else:
raise NotImplementedError
elif ks[0] <= 0:
ks = [train_feats.shape[-1]]
model = None
for k in ks:
tic = time.time()
train_feats_k = train_feats[:, :feat_start_k + k]
test_feats_k = test_feats[:, :feat_start_k + k]
print(train_feats_k.shape)
print(test_feats_k.shape)
if norm:
train_feats_k = preprocessing.normalize(train_feats_k,
norm='l2',
axis=0)
test_feats_k = preprocessing.normalize(test_feats_k,
norm='l2',
axis=0)
if model_name.startswith('LogisticRegression'):
settings = model_name.split('_')
c = float(settings[3][1:])
model = LogisticRegression(solver=settings[1],
multi_class=settings[2],
C=c,
verbose=False,
max_iter=10000,
tol=1e-4,
n_jobs=-1)
elif model_name.startswith('DecisionTreeClassifier'):
settings = model_name.split('_')
cls_weight = settings[3]
if cls_weight == 'none':
cls_weight = None
d = int(settings[4][1:])
ss = int(settings[5][1:])
sl = int(settings[6][1:])
model = DecisionTreeClassifier(criterion=settings[1],
splitter=settings[2],
class_weight=cls_weight,
max_depth=d,
min_samples_split=ss,
min_samples_leaf=sl)
else:
raise NotImplementedError
model.fit(train_feats_k, train_labels)
score = model.score(test_feats_k, test_labels)
score_all.append((k, score))
time_stamp = time.strftime("[%Y-%m-%d %H:%M:%S]")
if model_name.startswith('DecisionTreeClassifier'):
train_score = model.score(train_feats_k, train_labels)
log_str = '%s %s: k = %d train_acc = %.3f, accuracy = %.3f; %.2f minutes' \
% (time_stamp, exp_name, k, train_score * 100, score * 100, (time.time() - tic) / 60)
else:
log_str = '%s %s: k = %d, accuracy = %.3f; %.2f minutes' \
% (time_stamp, exp_name, k, score * 100, (time.time() - tic) / 60)
print(log_str)
f.write(log_str + '\n')
np.save(
'applications/fine_grained_classification/logs/%s.npy' % exp_name,
score_all)
f.close()
return score, model
# rand_tune_tree(500)
# cub_dataset = CUBDataset(split=None, val_ratio=0.1)
# model_name = 'LogisticRegression_newton-cg_multinomial_C1'
# for norm in [False, True]:
# for feat_mode in ['joint_gt_cls', 'joint_pred_cls', 'joint_gt_tri', 'joint_pred_tri']:
# classify(cub_dataset=cub_dataset, feat_mode=feat_mode, model_name=model_name, norm=norm, val_ratio=0.1)
# for criterion in ['entropy', 'gini']:
# for splitter in ['best', 'random']:
# for cls_weight in ['none', 'balanced']:
# for d in range(10, 52, 10):
# model_name = 'DecisionTreeClassifier_%s_%s_%s_D%d' % (criterion, splitter, cls_weight, d)
# classify(cub_dataset=cub_dataset, feat_mode='ph_cls', model_name=model_name, norm=norm,
# val_ratio=0.1)
cub_dataset = CUBDataset(split=None, val_ratio=0)
model_name = 'LogisticRegression_newton-cg_multinomial_C1'
# for f in ['att_gt_', 'joint_gt_cls_', 'att_pred_', 'joint_pred_cls_']:
# ks = [0]
# if f.startswith('joint'):
# ks = [100]
# for att_types in ['s', 'c', 'p', 'cp', 'sc', 'sp', 'scp']:
# classify(cub_dataset=cub_dataset, feat_mode=f + att_types, model_name=model_name, norm=False,
# val_ratio=0, ks=ks)
# classify(cub_dataset=cub_dataset, feat_mode='ph_cls_sig', model_name=model_name, norm=False,
# val_ratio=0, ks=None)
for att_types in ['s', 'c', 'p', 'sp', 'scp', 'cp', 'sc']:
classify(cub_dataset=cub_dataset,
feat_mode='joint_gt_cls_' + att_types,
model_name=model_name,
def main(ph_num=100):
cub_dataset = CUBDataset(split=None, val_ratio=0)
classes = cub_dataset.class_names
# model_name = 'LogisticRegression_newton-cg_multinomial_C1'
# _, model = classify(cub_dataset=cub_dataset, feat_mode='ph_cls', model_name=model_name, norm=False,
# val_ratio=0, ks=[ph_num])
# weights = model.coef_ # classes x phrases
# np.save('applications/fine_grained_classification/classify_ph_weights_%d.npy' % ph_num, weights)
weights = np.load(
'applications/fine_grained_classification/classify_ph_weights_%d.npy' %
ph_num)
texture_dataset = TextureDescriptionData()
phrases = texture_dataset.phrases
# mean_v = np.mean(weights)
std_v = np.std(weights)
min_v = -1 * std_v
max_v = std_v
pos_std = weights[weights > 0]
neg_std = weights[weights < 0]
print('ready')
folder = 'applications/fine_grained_classification/ph_clouds_%d_np' % ph_num
if not os.path.exists(folder):
os.makedirs(folder)
# for cls_i, cls in enumerate(classes):
for cls_i in [17, 55, 188]:
cls = classes[cls_i]
ph_weights = weights[cls_i, :]
ph_freq_dict = {ph: abs(w) for ph, w in zip(phrases, ph_weights)}
pos_dict = {ph: w for ph, w in zip(phrases, ph_weights) if w > 0}
neg_dict = {ph: -w for ph, w in zip(phrases, ph_weights) if w < 0}
def color_fn(phrase, *args, **kwargs):
ph_i = texture_dataset.phrase_to_phid(phrase)
# v = (ph_weights[ph_i] - min_v) / (max_v - min_v)
w = ph_weights[ph_i]
if w > 0:
v = w / pos_std + 0.5
else:
v = w / neg_std + 0.5
cmap = cm.get_cmap('coolwarm')
rgba = cmap(v, bytes=True)
return rgba
red_fn = lambda *args, **kwargs: "red"
blue_fn = lambda *args, **kwargs: "blue"
# wc = WordCloud(background_color="white", color_func=color_fn, prefer_horizontal=0.9,
# height=600, width=1200, min_font_size=5, margin=4, max_words=500,
# font_path='visualizations/DIN Alternate Bold.ttf')
# wc.generate_from_frequencies(ph_freq_dict)
# wc_path = os.path.join(folder, '%s.jpg' % cls)
# print(wc_path)
# wc.to_file(wc_path)
wc = WordCloud(background_color="white",
color_func=red_fn,
prefer_horizontal=0.9,
height=600,
width=1200,
min_font_size=4,
margin=2,
max_words=500,
font_path='visualizations/DIN Alternate Bold.ttf')
wc.generate_from_frequencies(pos_dict)
wc_path = os.path.join(folder, '%s_pos.jpg' % cls)
print(wc_path)
wc.to_file(wc_path)
wc = WordCloud(background_color="white",
color_func=blue_fn,
prefer_horizontal=0.9,
height=600,
width=1200,
min_font_size=4,
margin=2,
max_words=500,
font_path='visualizations/DIN Alternate Bold.ttf')
wc.generate_from_frequencies(neg_dict)
wc_path = os.path.join(folder, '%s_neg.jpg' % cls)
print(wc_path)
wc.to_file(wc_path)
def train():
from torch.utils.tensorboard import SummaryWriter
# load configs
output_path = 'applications/fine_grained_classification/att_pred_1'
if not os.path.exists(output_path):
os.makedirs(output_path)
# make data_loader, model, criterion, optimizer
dataset = CUBDataset(split='train', val_ratio=0.1)
train_data_loader = DataLoader(dataset,
batch_size=64,
shuffle=True,
drop_last=True)
eval_dataset = copy.deepcopy(dataset)
eval_dataset.split = 'val'
eval_data_loader = DataLoader(eval_dataset, batch_size=64, shuffle=False)
model: AttClassifier = AttClassifier(class_num=len(dataset.att_names),
pretrained_backbone=True,
fc_dims=[
512,
],
use_feats=[2, 4])
model.train()
device = torch.device('cuda')
model.to(device)
# re-weight loss based on phrase frequency, more weights on positive samples
class_weights = get_class_weights(dataset).to(device)
criterion = nn.BCEWithLogitsLoss(reduction='mean',
pos_weight=class_weights)
optimizer = torch.optim.Adam(filter(lambda p: p.requires_grad,
model.parameters()),
lr=0.0001,
weight_decay=0.0001,
betas=(0.8, 0.999),
eps=1.0e-08)
lr_scheduler = torch.optim.lr_scheduler.StepLR(optimizer,
step_size=50,
gamma=0.1)
# make tensorboard writer and dirs
checkpoint_dir = os.path.join(output_path, 'checkpoints')
tb_dir = os.path.join(output_path, 'tensorboard')
tb_writer = SummaryWriter(tb_dir)
if not os.path.exists(checkpoint_dir):
os.makedirs(checkpoint_dir)
if not os.path.exists(tb_dir):
os.makedirs(tb_dir)
# training loop
step = 1
epoch = 1
loss = None
pred_labels = None
# best_eval_loss = 100
best_eval_acc = 0
best_eval_count = 0
while epoch <= 100:
lr = optimizer.param_groups[0]['lr']
for _, imgs, _, labels in train_data_loader:
imgs = imgs.to(device)
labels = labels.to(device, dtype=torch.float)
pred_labels = model(imgs)
loss = criterion(pred_labels, labels)
optimizer.zero_grad()
loss.backward()
optimizer.step()
if step <= 20:
print('[%s] epoch-%d step-%d: loss %.4f; lr %.4f' %
(time.strftime('%m/%d %H:%M:%S'), epoch, step, loss, lr))
if epoch == 1 and step == 2: # debug
eval_loss, acc = do_eval(model, eval_data_loader, device,
criterion)
print(eval_loss, acc)
step += 1
lr_scheduler.step(epoch=epoch)
tb_writer.add_scalar('train/loss', loss, epoch)
tb_writer.add_scalar('train/lr', lr, epoch)
tb_writer.add_scalar('step', step, epoch)
tb_writer.add_histogram('pred_labels', pred_labels, epoch)
eval_loss, acc = do_eval(model, eval_data_loader, device, criterion)
print(
'[%s] epoch-%d step-%d: loss %.4f; lr %.4f; eval loss %.4f, acc %.4f '
% (time.strftime('%m/%d %H:%M:%S'), epoch, step, loss, lr,
eval_loss, acc))
tb_writer.add_scalar('eval/loss', loss, epoch)
tb_writer.add_scalar('eval/acc', acc, epoch)
model.train()
# if eval_loss < best_eval_loss:
if acc > best_eval_acc:
print('EVAL: new best!')
# best_eval_loss = eval_loss
best_eval_acc = acc
best_eval_count = 0
with open(os.path.join(checkpoint_dir, 'epoch.txt'), 'w') as f:
f.write('BEST: epoch {}\n'.format(epoch))
torch.save(model.state_dict(),
os.path.join(checkpoint_dir, 'BEST_checkpoint.pth'))
else:
best_eval_count += 1
print('EVAL: since last best: %d' % best_eval_count)
if epoch % 15 == 0:
with open(os.path.join(checkpoint_dir, 'epoch.txt'), 'a') as f:
f.write('LAST: epoch {}\n'.format(epoch))
torch.save(model.state_dict(),
os.path.join(checkpoint_dir, 'LAST_checkpoint.pth'))
if best_eval_count % 10 == 0 and best_eval_count > 0:
print('EVAL: lr decay triggered')
for param_group in optimizer.param_groups:
param_group['lr'] *= 0.1
if best_eval_count % 20 == 0 and best_eval_count > 0:
print('EVAL: early stop triggered')
break
epoch += 1
tb_writer.close()
return