def main():
text_ckpt = torch.load(
'/home/zwei/Dev/AttributeNet3/TextClassificationV2/ckpts/TextCNN_googlenews_NLT_Static.pth.tar'
)
args_model = text_ckpt['args_model']
args_data = text_ckpt['args_data']
text_model = TextCNN(args_model)
model_tag2idx = args_data.tag2idx
text_model.load_state_dict(text_ckpt['state_dict'], strict=True)
vocab_idx2tag = loadpickle(
'/home/zwei/Dev/AttributeNet3/AdobeStockSelection/EmotionNetFinal/tag2idx.pkl'
)['idx2tag']
dataset = loadpickle(
'/home/zwei/Dev/AttributeNet3/AdobeStockSelection/EmotionNetFinal/CNNsplit_tagidx_36534_test.pkl'
)
text_model.eval()
emotion_tags = loadpickle(
'/home/zwei/Dev/AttributeNet3/AdobeStockSelection/EmotionNetFinal/etag2idx.pkl'
)['key2idx']
image_url_dict = loadpickle(
'/home/zwei/Dev/AttributeNet3/AdobeStockSelection/RetrieveSelected778/data_v2/dataset_image_urls.pkl'
)
new_dataset = []
for data_idx, s_data in tqdm.tqdm(enumerate(dataset), total=len(dataset)):
x_tags = [vocab_idx2tag[x] for x in s_data[1]]
x_tag_ids = []
x_tag_names = []
x_emotion_tags = []
for x_tag in x_tags:
if x_tag in model_tag2idx:
x_tag_ids.append(model_tag2idx[x_tag])
x_tag_names.append(x_tag)
if x_tag in emotion_tags:
x_emotion_tags.append(x_tag)
x_tag_ids = pad_sentences(x_tag_ids, args_model.max_len,
args_model.vocab_size + 1)
x_tag_ids = torch.LongTensor(x_tag_ids).unsqueeze(0)
predicts = F.softmax(text_model(x_tag_ids)[0],
dim=1).squeeze(0).cpu().data.numpy()
new_dataset.append([s_data[0], predicts.tolist()])
if data_idx % 50000 == 0:
image_cid = int(get_image_cid_from_url(s_data[0], location=1))
if image_cid in image_url_dict:
print("{}".format(image_url_dict[image_cid]))
print(", ".join(x_emotion_tags))
print(", ".join(x_tag_names))
print(', '.join(
'{}({:.2f})'.format(idx2emotion[i], predicts[i])
for i in range(len(predicts))))
save2pickle(
'/home/zwei/Dev/AttributeNet3/AdobeStockSelection/EmotionNetFinal/CNNsplit_distill8_test.pkl',
new_dataset)
# Copyright (c) 2019 Zijun Wei. # Licensed under the MIT License. # Author: Zijun Wei # Usage(TODO): # Email: [email protected] # Created: 13/Mar/2019 19:56 import torch from TextClassificationV2.models.TextCNN import TextCNN_NLT from CNNs.models.resnet import load_state_dict from PyUtils.pickle_utils import loadpickle if __name__ == '__main__': ckpt_file = '/home/zwei/Dev/AttributeNet3/TextClassificationV2/ckpts/TextCNN_googlenews_NLT_Static.pth.tar' ckpt_info = torch.load(ckpt_file) args_model = ckpt_info['args_model'] text_model = TextCNN_NLT(args_model) load_state_dict(text_model, ckpt_info['state_dict']) x_weight = text_model.embedding.weight[1] args_data = ckpt_info['args_data'] embed_dict = loadpickle( '/home/zwei/Dev/AttributeNet3/LanguageData/word2vec_dicts/googlenews_S_w2v_dict.pl' ) x_weight_from_data = embed_dict[args_data.idx2tag[1]] print("Done")
def main():
import argparse
parser = argparse.ArgumentParser(
description="Pytorch Image CNN training from Configure Files")
parser.add_argument(
'--config_file',
required=True,
help="This scripts only accepts parameters from Json files")
input_args = parser.parse_args()
config_file = input_args.config_file
args = parse_config(config_file)
if args.name is None:
args.name = get_stem(config_file)
torch.set_default_tensor_type('torch.FloatTensor')
# best_prec1 = 0
args.script_name = get_stem(__file__)
# current_time_str = get_date_str()
print_func = print
if args.device:
os.environ["CUDA_VISIBLE_DEVICES"] = args.device
if args.seed is not None:
random.seed(args.seed)
torch.manual_seed(args.seed)
cudnn.deterministic = True
warnings.warn('You have chosen to seed training. '
'This will turn on the CUDNN deterministic setting, '
'which can slow down your training considerably! '
'You may see unexpected behavior when restarting '
'from checkpoints.')
if args.gpu is not None:
warnings.warn('You have chosen a specific GPU. This will completely '
'disable data parallelism.')
args.distributed = args.world_size > 1
if args.distributed:
dist.init_process_group(backend=args.dist_backend,
init_method=args.dist_url,
world_size=args.world_size)
if args.pretrained:
print_func("=> using pre-trained model '{}'".format(args.arch))
model = models.__dict__[args.arch](pretrained=True,
num_classes=args.num_classes)
else:
print_func("=> creating model '{}'".format(args.arch))
model = models.__dict__[args.arch](pretrained=False,
num_classes=args.num_classes)
if args.gpu is not None:
model = model.cuda(args.gpu)
elif args.distributed:
model.cuda()
model = torch.nn.parallel.DistributedDataParallel(model)
else:
if args.arch.startswith('alexnet') or args.arch.startswith('vgg'):
model.features = torch.nn.DataParallel(model.features)
model.cuda()
else:
model = torch.nn.DataParallel(model).cuda()
# model = model.cuda()
if args.visual_model:
if os.path.isfile(args.visual_model):
print_func("=> loading checkpoint '{}'".format(args.visual_model))
checkpoint = torch.load(args.visual_model)
model.load_state_dict(checkpoint['state_dict'], strict=True)
# import collections
# if isinstance(checkpoint, collections.OrderedDict):
# load_state_dict(model, checkpoint, exclude_layers=['fc.weight', 'fc.bias'])
#
#
# else:
# load_state_dict(model, checkpoint['state_dict'], exclude_layers=['module.fc.weight', 'module.fc.bias'])
# print_func("=> loaded checkpoint '{}' (epoch {})"
# .format(args.visual_model, checkpoint['epoch']))
else:
print_func("=> no checkpoint found at '{}'".format(
args.visual_model))
return
else:
print_func(
"=> This script is for fine-tuning only, please double check '{}'".
format(args.visual_model))
print_func("Now using randomly initialized parameters!")
cudnn.benchmark = True
from PyUtils.pickle_utils import loadpickle
import numpy as np
from PublicEmotionDatasets.Emotic.constants import emotion_full_words_690 as emotion_self_words
from torchvision.datasets.folder import default_loader
# tag_wordvectors = loadpickle(args.text_embed)
text_ckpt = torch.load(args.text_ckpt)
args_textmodel = text_ckpt['args_model']
args_textdata = text_ckpt['args_data']
text_model = TextCNN_NLT(args_textmodel)
text_model.load_state_dict(text_ckpt['state_dict'], strict=True)
text_model.eval()
idx2tag = args_textdata.idx2tag
tag2idx = args_textdata.tag2idx
print_func(" => loading word2vec parameters: {}".format(args.text_ckpt))
emotic_emotion_explaintations = {}
for x_key in emotion_self_words:
x_words = emotion_self_words[x_key].split(',')
x_id = [
tag2idx[x] if x in tag2idx else args_textmodel.vocab_size + 2
for x in x_words
]
x_id_padded = x_id + [args_textmodel.vocab_size + 1
] * (args_textmodel.max_len - len(x_id))
x_id_padded = torch.LongTensor(x_id_padded).unsqueeze(0)
item = {}
item['pred'] = []
item['label'] = []
item['target_matrix'] = text_model(x_id_padded)[-2].squeeze(
0).cpu().data.numpy()[:len(x_id)]
item['description'] = x_words
emotic_emotion_explaintations[x_key] = item
val_list = loadpickle(args.val_file)
image_directory = args.data_dir
from CNNs.datasets.multilabel import get_val_simple_transform
val_transform = get_val_simple_transform()
model.eval()
import tqdm
for i, (input_image_file, target, _,
_) in tqdm.tqdm(enumerate(val_list),
desc="Evaluating Peace",
total=len(val_list)):
# measure data loading time
image_path = os.path.join(image_directory, input_image_file)
input_image = default_loader(image_path)
input_image = val_transform(input_image)
if args.gpu is not None:
input_image = input_image.cuda(args.gpu, non_blocking=True)
input_image = input_image.unsqueeze(0).cuda()
# target_idx = target.nonzero() [:,1]
# compute output
output, output_proj = model(input_image)
output_proj = output_proj.cpu().data.numpy()
target_labels = set([x[0] for x in target.most_common()])
for x_key in emotic_emotion_explaintations:
dot_product_label = cosine_similarity(
output_proj,
emotic_emotion_explaintations[x_key]['target_matrix'])[0]
pred_score = np.average(dot_product_label)
emotic_emotion_explaintations[x_key]['pred'].append(pred_score)
if x_key in target_labels:
emotic_emotion_explaintations[x_key]['label'].append(1)
else:
emotic_emotion_explaintations[x_key]['label'].append(0)
from sklearn.metrics import average_precision_score
full_AP = []
for x_key in emotic_emotion_explaintations:
full_pred = np.array(emotic_emotion_explaintations[x_key]['pred'])
full_label = np.array(emotic_emotion_explaintations[x_key]['label'])
AP = average_precision_score(full_label, full_pred)
if np.isnan(AP):
print("{} is Nan".format(x_key))
continue
full_AP.append(AP)
print("{}\t{:.4f}".format(x_key, AP * 100))
AvgAP = np.mean(full_AP)
print("Avg AP: {:.2f}".format(AvgAP * 100))
def train(train_set, val_set, args_data, args_model, args_hyper):
wordvec_type = args_hyper.w2v_type # or selftrained or googlenews
wordvec_file = '/home/zwei/Dev/AttributeNet3/LanguageData/word2vec_dicts/' \
'{}_w2v_dict.pl'.format(wordvec_type)
print("Loading from {}".format(wordvec_type))
pretrained_w2v = loadpickle(wordvec_file)
#This is only creating a vocabulary that exists in th
wv_matrix = []
words_not_found = []
for i in range(len(args_data.vocab)):
word = args_data.idx2tag[i]
if word in pretrained_w2v:
wv_matrix.append(pretrained_w2v[word])
else:
words_not_found.append(word)
# print("{} not found in dictrionary, will use random".format(word))
wv_matrix.append(
np.random.uniform(-0.01, 0.01,
args_hyper.word_dim).astype("float32"))
print("{} words were not found".format(len(words_not_found)))
# one for UNK and one for zero padding
wv_matrix.append(
np.random.uniform(-0.01, 0.01, args_hyper.word_dim).astype("float32"))
wv_matrix.append(np.zeros(args_hyper.word_dim).astype("float32"))
wv_matrix = np.array(wv_matrix)
model = TextCNN(args_model, init_wv=wv_matrix).cuda()
parameters = filter(lambda p: p.requires_grad, model.parameters())
optimizer = optim.Adadelta(parameters, args_hyper.learning_rate)
scheduler = lr_scheduler.MultiStepLR(
optimizer, [int(x) for x in args_hyper.lr_schedule.split(',')],
gamma=0.1)
max_dev_top1 = 0
max_dev_hits = 0
max_test_acc = 0
best_model = None
model.train()
report_params(model)
for e in range(args_hyper.epoch):
train_set = shuffle(train_set)
train_losses = AverageMeter()
train_top1 = AverageMeter()
train_hits = AverageMeter()
scheduler.step()
current_lr = optimizer.param_groups[0]['lr']
model.train()
for i in tqdm.tqdm(range(0, len(train_set), args_hyper.batch_size)):
batch_range = min(args_hyper.batch_size, len(train_set) - i)
# add random dropping words:
batch_x = []
for sent in train_set[i:i + batch_range]:
x_sent = sent[0]
drop_thre = 0.2
x_collected_words = []
for x_word in x_sent:
p = random.uniform(0, 1)
if p >= drop_thre:
x_collected_words.append(args_data.tag2idx[x_word])
if len(x_collected_words) >= args_model.max_len:
batch_x.append(x_collected_words[:args_model.max_len])
else:
batch_x.append(
x_collected_words + [args_model.vocab_size + 1] *
(args_model.max_len - len(x_collected_words)))
batch_y = [c[1] for c in train_set[i:i + batch_range]]
torch_x = Variable(torch.LongTensor(batch_x)).cuda()
torch_y = Variable(torch.FloatTensor(batch_y)).cuda()
model_output = model(torch_x)
pred = model_output[0]
log_softmax_output = F.log_softmax(pred, dim=1)
loss = -torch.sum(log_softmax_output * torch_y) / pred.shape[0]
optimizer.zero_grad()
loss.backward()
nn.utils.clip_grad_norm_(parameters, max_norm=args_hyper.max_norm)
optimizer.step()
pred_idx = np.argmax(pred.cpu().data.numpy(), axis=1)
train_losses.update(loss.item(), batch_range)
# # FIXME: to top 1 and top-hit
top1_batch, _, _ = multilabelTop1(pred_idx, batch_y)
train_top1.update(top1_batch, len(pred_idx))
hits_batch, _, _ = multilabelHits(pred_idx, batch_y)
train_hits.update(hits_batch, len(pred_idx))
dev_top1, dev_hits, dev_loss = test(val_set, model, args_data,
args_model)
if dev_top1 > max_dev_top1:
max_dev_top1 = dev_top1
best_model = copy.deepcopy(model)
if dev_hits > max_dev_hits:
max_dev_hits = dev_hits
print(
'epoch: {} lr: {:.6f}, dev_top1: {:.2f}, dev_hits: {:.2f}, dev_loss: {:.2f}, '
'train_top1: {:.2f}, train_hits: {:.2f}, train_loss:{:.2f}, max_dev_top1: {:.2f}, '
'max_dev_hits: {:.2f}'.format(e + 1, current_lr, dev_top1 * 100,
dev_hits * 100, dev_loss,
train_top1.avg * 100,
train_hits.avg * 100,
train_losses.avg, max_dev_top1 * 100,
max_dev_hits * 100))
print("max dev top1: {:.2f},\tmax dev hits: {:.2f}".format(
max_dev_top1, max_dev_hits))
return best_model
def main():
import argparse
parser = argparse.ArgumentParser(
description="Pytorch Image CNN training from Configure Files")
parser.add_argument(
'--config_file',
required=True,
help="This scripts only accepts parameters from Json files")
input_args = parser.parse_args()
config_file = input_args.config_file
args = parse_config(config_file)
if args.name is None:
args.name = get_stem(config_file)
torch.set_default_tensor_type('torch.FloatTensor')
best_prec1 = 0
args.script_name = get_stem(__file__)
current_time_str = get_date_str()
# if args.resume is None:
if args.save_directory is None:
save_directory = get_dir(
os.path.join(project_root, 'ckpts2', '{:s}'.format(args.name),
'{:s}-{:s}'.format(args.ID, current_time_str)))
else:
save_directory = get_dir(
os.path.join(project_root, 'ckpts2', args.save_directory))
# else:
# save_directory = os.path.dirname(args.resume)
print("Save to {}".format(save_directory))
log_file = os.path.join(save_directory,
'log-{0}.txt'.format(current_time_str))
logger = log_utils.get_logger(log_file)
log_utils.print_config(vars(args), logger)
print_func = logger.info
print_func('ConfigFile: {}'.format(config_file))
args.log_file = log_file
if args.device:
os.environ["CUDA_VISIBLE_DEVICES"] = args.device
if args.seed is not None:
random.seed(args.seed)
torch.manual_seed(args.seed)
cudnn.deterministic = True
warnings.warn('You have chosen to seed training. '
'This will turn on the CUDNN deterministic setting, '
'which can slow down your training considerably! '
'You may see unexpected behavior when restarting '
'from checkpoints.')
if args.gpu is not None:
warnings.warn('You have chosen a specific GPU. This will completely '
'disable data parallelism.')
args.distributed = args.world_size > 1
if args.distributed:
dist.init_process_group(backend=args.dist_backend,
init_method=args.dist_url,
world_size=args.world_size)
if args.pretrained:
print_func("=> using pre-trained model '{}'".format(args.arch))
visual_model = models.__dict__[args.arch](pretrained=True,
num_classes=args.num_classes)
else:
print_func("=> creating model '{}'".format(args.arch))
visual_model = models.__dict__[args.arch](pretrained=False,
num_classes=args.num_classes)
if args.freeze:
visual_model = CNN_utils.freeze_all_except_fc(visual_model)
if os.path.isfile(args.text_ckpt):
print_func("=> loading checkpoint '{}'".format(args.text_ckpt))
text_data = torch.load(args.text_ckpt)
text_model = TextCNN(text_data['args_model'])
# load_state_dict(text_model, text_data['state_dict'])
text_model.load_state_dict(text_data['state_dict'], strict=True)
text_model.eval()
print_func("=> loaded checkpoint '{}' for text classification".format(
args.text_ckpt))
args.vocab_size = text_data['args_model'].vocab_size
else:
print_func("=> no checkpoint found at '{}'".format(args.text_ckpt))
return
# text_params = {
# # "Learning_SCHEDULE": [int(i) for i in args.lr_schedule.split(',')],
# # "LEARNING_RATE": args.learning_rate,
# "MAX_SENT_LEN": args.sent_len,
# "BATCH_SIZE": args.batch_size,
# "WORD_DIM": args.word_dim,
# "FILTER_NUM": args.filter_dim,
# "VOCAB_SIZE": len(text_data['tag2idx']), # FIXME: check here!
# "CLASS_SIZE": 8,
# "tag2idx": text_data['tag2idx'],
# "idx2tag": text_data['idx2tag'],
# }
# args.vocab_size = text_params['VOCAB_SIZE']
# text_model = Text_Embed(**text_params)
# text_model.load_state_dict(text_data['model'].state_dict(), strict=False)
# # Not training Text Model, set to eval to freeze BN
# text_model.eval()
if args.gpu is not None:
visual_model = visual_model.cuda(args.gpu)
text_model = text_model.cuda((args.gpu))
elif args.distributed:
visual_model.cuda()
visual_model = torch.nn.parallel.DistributedDataParallel(visual_model)
else:
if args.arch.startswith('alexnet') or args.arch.startswith('vgg'):
visual_model.features = torch.nn.DataParallel(
visual_model.features)
visual_model.cuda()
else:
visual_model = torch.nn.DataParallel(visual_model).cuda()
text_model = torch.nn.DataParallel(text_model).cuda()
# text_model = torch.nn.DataParallel(text_model).cuda()
# define loss function (criterion) and optimizer
# # Update: here
# config = {'loss': {'type': 'simpleCrossEntropyLoss', 'args': {'param': None}}}
# criterion = get_instance(loss_funcs, 'loss', config)
# criterion = criterion.cuda(args.gpu)
criterion = nn.CrossEntropyLoss(ignore_index=-1).cuda(args.gpu)
optimizer = torch.optim.SGD(filter(lambda p: p.requires_grad,
visual_model.parameters()),
lr=args.lr,
momentum=args.momentum,
weight_decay=args.weight_decay)
if args.lr_schedule:
print_func("Using scheduled learning rate")
scheduler = lr_scheduler.MultiStepLR(
optimizer, [int(i) for i in args.lr_schedule.split(',')],
gamma=0.1)
else:
scheduler = lr_scheduler.ReduceLROnPlateau(optimizer,
'min',
patience=args.lr_patience)
# optimizer = torch.optim.SGD(model.parameters(), args.lr,
# momentum=args.momentum,
# weight_decay=args.weight_decay)
# optionally resume from a checkpoint
if args.resume:
if os.path.isfile(args.resume):
print_func("=> loading checkpoint '{}'".format(args.resume))
checkpoint = torch.load(args.resume)
import collections
if isinstance(checkpoint, collections.OrderedDict):
load_state_dict(visual_model, checkpoint)
else:
load_state_dict(visual_model, checkpoint['state_dict'])
print_func("=> loaded checkpoint '{}' (epoch {})".format(
args.resume, checkpoint['epoch']))
else:
print_func("=> no checkpoint found at '{}'".format(args.resume))
cudnn.benchmark = True
model_total_params = sum(p.numel() for p in visual_model.parameters())
model_grad_params = sum(p.numel() for p in visual_model.parameters()
if p.requires_grad)
print_func("Total Parameters: {0}\t Gradient Parameters: {1}".format(
model_total_params, model_grad_params))
# Data loading code
val_dataset = get_instance(custom_datasets, '{0}'.format(args.valloader),
args)
if val_dataset is None:
val_loader = None
else:
val_loader = torch.utils.data.DataLoader(val_dataset,
batch_size=args.batch_size,
shuffle=False,
num_workers=args.workers,
pin_memory=True,
collate_fn=none_collate)
if args.evaluate:
print_func('Validation Only')
validate(val_loader, visual_model, criterion, args, print_func)
return
else:
train_dataset = get_instance(custom_datasets,
'{0}'.format(args.trainloader), args)
if args.distributed:
train_sampler = torch.utils.data.distributed.DistributedSampler(
train_dataset)
else:
train_sampler = None
train_loader = torch.utils.data.DataLoader(
train_dataset,
batch_size=args.batch_size,
shuffle=(train_sampler is None),
num_workers=args.workers,
pin_memory=True,
sampler=train_sampler,
collate_fn=none_collate)
for epoch in range(args.start_epoch, args.epochs):
if args.distributed:
train_sampler.set_epoch(epoch)
if args.lr_schedule:
# CNN_utils.adjust_learning_rate(optimizer, epoch, args.lr)
scheduler.step()
current_lr = optimizer.param_groups[0]['lr']
print_func("Epoch: [{}], learning rate: {}".format(epoch, current_lr))
# train for one epoch
train(train_loader, visual_model, text_model, criterion, optimizer,
epoch, args, print_func)
# evaluate on validation set
if val_loader:
prec1, val_loss = validate(val_loader, visual_model, criterion,
args, print_func)
else:
prec1 = 0
val_loss = 0
# remember best prec@1 and save checkpoint
is_best = prec1 > best_prec1
best_prec1 = max(prec1, best_prec1)
CNN_utils.save_checkpoint(
{
'epoch': epoch + 1,
'arch': args.arch,
'state_dict': visual_model.state_dict(),
'best_prec1': best_prec1,
'optimizer': optimizer.state_dict(),
},
is_best,
file_directory=save_directory,
epoch=epoch)
if not args.lr_schedule:
scheduler.step(val_loss)