def test_accuracy(data_path, checkpoint_path):
model = init_model(checkpoint_path)
polyvore_json_data = load_polyvore_data(data_path)
true_count = 0
error_count = 0
for question_json in polyvore_json_data[:2000]:
try:
question_imgs = question_json["question"]
question_imgs_feature = []
for question_img_name in question_imgs:
question_img_name = question_img_name.split('_')[0] + '.png'
question_img_feature = load_image_feature(
model, question_img_name)
question_imgs_feature.append(question_img_feature)
answer_imgs = question_json["answers"]
answer_imgs_score = []
for answer_img_name in answer_imgs:
answer_img_name = answer_img_name.split('_')[0] + '.png'
answer_imgs_feature = load_image_feature(
model, answer_img_name)
answer_img_score = calculate_distance(question_imgs_feature,
answer_imgs_feature)
answer_imgs_score.append(answer_img_score)
if min(answer_imgs_score) == answer_imgs_score[0]:
true_count += 1
print(true_count)
except Exception as e:
error_count += 1
continue
# print("Test accuracy: {}".format(float(true_count)/float(len(polyvore_json_data))))
print("Test accuracy: {}".format(
float(true_count) / float(2000 - error_count)))
print(arg, getattr(args, arg))
if args.seed:
np.random.seed(args.seed)
random.seed(args.seed)
torch.manual_seed(args.seed)
if torch.cuda.is_available():
torch.cuda.manual_seed_all(args.seed)
if args.use_benchmark:
torch.backends.cudnn.benchmark = True
torch.backends.cudnn.deterministic = True
pretrained = True if args.pretrained and args.checkpoint_path is None else False
model = util.init_model(args.model_name,
num_classes=args.num_classes,
pretrained=pretrained,
pooling=args.pooling.split(","))
lr = args.lr
iteration = 0
optimizer_state = None
if args.checkpoint_path is not None and os.path.isfile(
args.checkpoint_path):
model, optimizer_state, tmp_lr, tmp_iteration = util.load_checkpoint(
args.checkpoint_path,
model,
model_name=args.model_name,
is_different_class_num=args.is_different_class_num,
not_dict_model=args.not_dict_model)
if args.load_lr and not args.not_dict_model:
lr = tmp_lr
# iteration = tmp_iteration
t = time.time()
sv, vocabf, corpusf = load_train_vocab(args)
corpus = corpus.Corpus.load_from_corpus(sv, vocabf, corpusf, args)
print "Time elapsed", time.time() - t
args.ntokens = sv.sizes
# Load the model, either from a checkpoint, or init a new one.
if args.mode == 'train':
if args.checkpoint != '':
print "Loading model checkpoint"
with open(args.checkpoint, 'rb') as f:
model = torch.load(f)
else:
model = util.init_model(args)
if args.cnn_encoder:
cnn = cnn.CNN(args)
if args.cuda:
cnn.cuda()
sigmoid = nn.Sigmoid()
criterion = nn.CrossEntropyLoss(ignore_index=PADDING_IDX)
params = list(model.parameters()) + list(cnn.parameters()) if args.cnn_encoder else list(model.parameters())
optimizer = torch.optim.Adam(params, lr=args.lr)
print model
elif args.mode in ['get_hiddens', 'generate']:
checkpoint = args.checkpoint if args.checkpoint != '' else util.get_savef(args, corpus)
args.batch_size = 1
def extract_features(args):
root_frames_dir = args.frames_dir
root_feats_dir = args.feats_dir
work = args.work
autofill = int(args.autofill)
ftype = args.type
gpu_list = args.gpu_list
class_dirs = os.listdir(root_frames_dir)
# skip a level for UCF101 dataset
for class_dir in class_dirs:
class_frames_dir = os.path.join(root_frames_dir, class_dir)
frames_dirs = os.listdir(class_frames_dir)
class_feats_dir = os.path.join(root_feats_dir, class_dir)
if not os.path.isdir(class_feats_dir):
os.makedirs(class_feats_dir)
# else:
# if autofill:
# logger.info('AUTOFILL ON: Attempting to autofill missing features.')
# frames_dirs = validate_feats.go(featsd=root_feats_dir, framesd=root_frames_dir)
# Difficulty of each job is measured by # of frames to process in each chunk.
# Can't be randomized since autofill list woudld be no longer valid.
# np.random.shuffle(frames_dirs)
work = len(frames_dirs) if not work else work
tf_img, model = init_model(args.gpu_list, args.type)
work_done = 0
while work_done != work:
frames_dirs_avail = diff_feats(class_frames_dir, class_feats_dir)
if len(frames_dirs_avail) == 0:
break
frames_dir = frames_dirs_avail.pop()
feat_filename = frames_dir.split('/')[-1] + '.npy'
video_feats_path = os.path.join(class_feats_dir, feat_filename)
if os.path.exists(video_feats_path):
logger.info(
'Features already extracted:\t{}'.format(video_feats_path))
continue
try:
frames_to_do = [
os.path.join(args.frames_dir, class_dir, frames_dir, p)
for p in os.listdir(
os.path.join(args.frames_dir, class_dir, frames_dir))
]
except Exception as e:
logger.exception(e)
continue
# Must sort so frames follow numerical order. os.listdir does not guarantee order.
frames_to_do.sort()
if len(frames_to_do) == 0:
logger.warning("Frame folder has no frames! Skipping...")
continue
# Save a flag copy
with open(video_feats_path, 'wb') as pf:
np.save(pf, [])
try:
batches = create_batches(frames_to_do,
tf_img,
logger=logger,
batch_size=args.batch_size)
except OSError as e:
logger.exception(e)
logger.warning("Corrupt image file. Skipping...")
os.remove(video_feats_path)
continue
logger.debug("Start video {}".format(work_done))
feats = process_batches(batches,
ftype,
gpu_list,
model,
logger=logger)
with open(video_feats_path, 'wb') as pf:
np.save(pf, feats)
logger.info(
'Saved complete features to {}.'.format(video_feats_path))
work_done += 1
def main(args):
if args.use_glob:
checkpoint_paths = glob.glob(args.checkpoint_paths)
checkpoint_paths.sort()
else:
checkpoint_paths = args.checkpoint_paths.split(",")
print("all checkpoints", checkpoint_paths)
if args.weights is None:
weights = [1.0] * len(checkpoint_paths)
else:
weights = [float(w) for w in args.weights.split(",")]
if not args.use_glob and len(checkpoint_paths) != len(weights):
sys.exit("weights count not matched with checkpoint_paths")
if not args.from_pkl:
model_names = args.model_names.split(":")
if len(model_names) == 1:
model_names = model_names * len(checkpoint_paths)
if len(checkpoint_paths) != len(model_names):
sys.exit("model_names count not matched with checkpoint_paths")
input_sizes = args.input_sizes.split(",")
if len(input_sizes) == 1:
input_sizes = input_sizes * len(checkpoint_paths)
if len(checkpoint_paths) != len(input_sizes):
sys.exit("input_sizes count not matched with checkpoint_paths")
new_input_sizes = []
for input_size in input_sizes:
input_size = input_size.split("x")
if len(input_size) == 1:
input_size = int(input_size[0])
else:
input_size = (int(input_size[0]), int(input_size[1]))
new_input_sizes.append(input_size)
input_sizes = new_input_sizes
if args.use_crops:
use_crops = args.use_crops.split(",")
if len(use_crops) == 1:
use_crops = use_crops * len(checkpoint_paths)
if len(checkpoint_paths) != len(use_crops):
sys.exit("use_crops count not matched with checkpoint_paths")
use_crops = [
use_crop in ['true', 'True'] for use_crop in use_crops
]
else:
use_crops = [False] * len(checkpoint_paths)
if args.poolings:
poolings = args.poolings.split(":")
if len(poolings) == 1:
poolings = [poolings[0].split(",")]
poolings = poolings * len(checkpoint_paths)
else:
if len(checkpoint_paths) != len(poolings):
sys.exit(
"poolings count not matched with checkpoint_paths")
poolings = [pooling.split(",") for pooling in poolings]
else:
poolings = [['GAP']] * len(checkpoint_paths)
if args.eval or args.save:
val_samples = get_val_samples(args.image_dir, args.label_file,
args.val_ratio)
val_samples.sort()
print("val samples", len(val_samples))
eval_logits = None
test_logits = None
if args.save:
os.makedirs(args.output_dir, exist_ok=True)
for i, checkpoint_path in enumerate(checkpoint_paths):
print(checkpoint_path, i, len(checkpoint_paths))
model = util.init_model(model_names[i],
num_classes=args.num_classes,
pretrained=False,
pooling=poolings[i])
model, _, _, _ = util.load_checkpoint(checkpoint_path,
model,
model_name=model_names[i],
is_different_class_num=False,
not_dict_model=False,
strict=not args.not_strict)
device = "cuda"
model = model.to(device)
model.eval()
input_size = input_sizes[i]
print("input size", input_size)
if args.eval or args.save:
data_loader = util.get_data_loader(
EvalDataset, [val_samples],
util.get_test_transforms(
input_size,
use_crops[i],
center_crop_ratio=args.center_crop_ratio,
use_gray=args.use_gray),
args.batch_size,
args.num_workers,
shuffle=False)
epoch_labels = []
epoch_preds = []
total_logits = None
for step, (inputs, labels) in enumerate(data_loader):
epoch_labels += list(labels.numpy())
inputs = inputs.to(device)
with torch.set_grad_enabled(False):
outputs = model(inputs)
_, preds = torch.max(outputs, 1)
epoch_preds += list(preds.cpu().numpy())
logits = F.softmax(outputs, 1)
if total_logits is None:
total_logits = logits.cpu().numpy()
else:
total_logits = np.concatenate(
[total_logits,
logits.cpu().numpy()], axis=0)
_, preds = torch.max(outputs, 1)
if step > 0 and step % args.log_step_interval == 0:
print("evaluating", step, len(data_loader))
tmp_acc = accuracy_score(epoch_labels, epoch_preds)
tmp_f1 = f1_score(epoch_labels, epoch_preds, average='macro')
print("acc: {}, f1: {}".format(tmp_acc, tmp_f1))
if args.save:
pickle.dump(
{
"logits": total_logits,
"labels": epoch_labels
},
open(
os.path.join(
args.output_dir,
os.path.basename(
os.path.dirname(checkpoint_path)) + "_" +
os.path.splitext(
os.path.basename(checkpoint_path))[0] +
"_eval.pkl"), "wb+"))
total_logits *= weights[i]
if eval_logits is None:
eval_logits = total_logits
else:
eval_logits += total_logits
if args.test or args.save:
data_loader = util.get_data_loader(
TestDataset, [args.test_dir],
util.get_test_transforms(
input_size,
use_crops[i],
center_crop_ratio=args.center_crop_ratio,
use_gray=args.use_gray,
use_pad=args.use_pad),
args.batch_size,
args.num_workers,
shuffle=False)
total_file_names = []
total_logits = None
for step, (imgs, file_names) in enumerate(data_loader):
if step > 0 and step % args.log_step_interval == 0:
print("testing", step, len(data_loader))
imgs = imgs.to(device)
total_file_names += list(file_names)
with torch.set_grad_enabled(False):
outputs = model(imgs)
logits = F.softmax(outputs, 1)
if total_logits is None:
total_logits = logits.cpu().numpy()
else:
total_logits = np.concatenate(
[total_logits,
logits.cpu().numpy()], axis=0)
if args.save:
pickle.dump(
{
"logits": total_logits,
"file_names": total_file_names
},
open(
os.path.join(
args.output_dir,
os.path.basename(
os.path.dirname(checkpoint_path)) + "_" +
os.path.splitext(
os.path.basename(checkpoint_path))[0] +
"_test.pkl"), "wb+"))
total_logits *= weights[i]
if test_logits is None:
test_logits = total_logits
else:
test_logits += total_logits
del model
if args.eval:
if args.from_pkl:
eval_logits = None
if args.use_glob:
checkpoint_paths = [
os.path.basename(p)[:-9] for p in glob.glob(
os.path.join(args.pkl_dir, "*_eval.pkl"))
]
weights = [1.0 / len(checkpoint_paths)] * len(checkpoint_paths)
print(weights)
for i, logits_file in enumerate(checkpoint_paths):
print(logits_file)
logits = pickle.load(
open(os.path.join(args.pkl_dir, logits_file + "_eval.pkl"),
"rb+"))
epoch_labels = logits['labels']
logits = logits['logits']
if eval_logits is None:
eval_logits = logits * weights[i]
else:
eval_logits += logits * weights[i]
else:
eval_logits = eval_logits # / len(checkpoint_paths)
eval_indices = []
for eval_logit in eval_logits:
index = np.argmax(eval_logit)
eval_indices.append(index)
acc = accuracy_score(epoch_labels, eval_indices)
f1 = f1_score(epoch_labels, eval_indices, average='macro')
print("last acc", acc)
print("last fq", f1)
if args.test:
if args.from_pkl:
test_logits = None
if args.use_glob:
checkpoint_paths = [
os.path.basename(p)[:-9] for p in glob.glob(
os.path.join(args.pkl_dir, "*_eval.pkl"))
]
weights = [1.0 / len(checkpoint_paths)] * len(checkpoint_paths)
print(weights)
for i, logits_file in enumerate(checkpoint_paths):
logits = pickle.load(
open(os.path.join(args.pkl_dir, logits_file + "_test.pkl"),
"rb+"))
total_file_names = logits['file_names']
logits = logits['logits']
if test_logits is None:
test_logits = logits * weights[i]
else:
test_logits += logits * weights[i]
else:
test_logits = test_logits # / len(checkpoint_paths)
if sum(weights) != 1:
test_logits += (1 - sum(weights))
test_indices = []
test_scores = []
for test_logit in test_logits:
index = np.argmax(test_logit)
score = test_logit[index]
test_indices.append(index)
test_scores.append(score)
rows = zip(total_file_names, test_indices, test_scores)
output_dir = os.path.dirname(args.csv)
if output_dir:
os.makedirs(output_dir, exist_ok=True)
with open(args.csv, "w") as f:
writer = csv.writer(f)
writer.writerow(["id", "landmark_id", "conf"])
for row in rows:
writer.writerow(row)
print("test done")
def __init__(self, args):
"""Constructs a basic PatchGAN convolutional discriminator.
Each position in the output is a score of discriminator confidence that
a 70x70 patch of the input is real.
Args:
args: Arguments passed in via the command line.
"""
super(PatchGAN, self).__init__()
norm_layer = get_norm_layer(args.norm_type)
layers = []
# Double channels for conditional GAN (concatenated src and tgt images)
num_channels = args.num_channels
layers += [
nn.Conv2d(num_channels,
args.num_channels_d,
args.kernel_size_d,
stride=2,
padding=1),
nn.LeakyReLU(0.2, True)
]
layers += [
nn.Conv2d(args.num_channels_d,
2 * args.num_channels_d,
args.kernel_size_d,
stride=2,
padding=1),
norm_layer(2 * args.num_channels_d),
nn.LeakyReLU(0.2, True)
]
layers += [
nn.Conv2d(2 * args.num_channels_d,
4 * args.num_channels_d,
args.kernel_size_d,
stride=2,
padding=1),
norm_layer(4 * args.num_channels_d),
nn.LeakyReLU(0.2, True)
]
layers += [
nn.Conv2d(4 * args.num_channels_d,
8 * args.num_channels_d,
args.kernel_size_d,
stride=1,
padding=1),
norm_layer(8 * args.num_channels_d),
nn.LeakyReLU(0.2, True)
]
layers += [
nn.Conv2d(8 * args.num_channels_d,
1,
args.kernel_size_d,
stride=1,
padding=1)
]
self.model = nn.Sequential(*layers)
init_model(self.model, init_method=args.initializer)
def __init__(self, args):
"""
Args:
args: Configuration args passed in via the command line.
"""
super(Flow2Flow, self).__init__()
self.device = 'cuda' if len(args.gpu_ids) > 0 else 'cpu'
self.gpu_ids = args.gpu_ids
self.is_training = args.is_training
self.in_channels = args.num_channels
self.out_channels = 4 ** (args.num_scales - 1) * self.in_channels
# Set up RealNVP generators (g_src: X <-> Z, g_tgt: Y <-> Z)
self.g_src = RealNVP(num_scales=args.num_scales,
in_channels=args.num_channels,
mid_channels=args.num_channels_g,
num_blocks=args.num_blocks,
un_normalize_x=True,
no_latent=False)
util.init_model(self.g_src, init_method=args.initializer)
self.g_tgt = RealNVP(num_scales=args.num_scales,
in_channels=args.num_channels,
mid_channels=args.num_channels_g,
num_blocks=args.num_blocks,
un_normalize_x=True,
no_latent=False)
util.init_model(self.g_tgt, init_method=args.initializer)
if self.is_training:
# Set up discriminators
self.d_tgt = PatchGAN(args) # Answers Q "is this tgt image real?"
self.d_src = PatchGAN(args) # Answers Q "is this src image real?"
self._data_parallel()
# Set up loss functions
self.max_grad_norm = args.clip_gradient
self.lambda_mle = args.lambda_mle
self.mle_loss_fn = RealNVPLoss()
self.gan_loss_fn = util.GANLoss(device=self.device, use_least_squares=True)
self.clamp_jacobian = args.clamp_jacobian
self.jc_loss_fn = util.JacobianClampingLoss(args.jc_lambda_min, args.jc_lambda_max)
# Set up optimizers
g_src_params = util.get_param_groups(self.g_src, args.weight_norm_l2, norm_suffix='weight_g')
g_tgt_params = util.get_param_groups(self.g_tgt, args.weight_norm_l2, norm_suffix='weight_g')
self.opt_g = torch.optim.Adam(chain(g_src_params, g_tgt_params),
lr=args.rnvp_lr,
betas=(args.rnvp_beta_1, args.rnvp_beta_2))
self.opt_d = torch.optim.Adam(chain(self.d_tgt.parameters(), self.d_src.parameters()),
lr=args.lr,
betas=(args.beta_1, args.beta_2))
self.optimizers = [self.opt_g, self.opt_d]
self.schedulers = [util.get_lr_scheduler(opt, args) for opt in self.optimizers]
# Setup image mixers
buffer_capacity = 50 if args.use_mixer else 0
self.src2tgt_buffer = util.ImageBuffer(buffer_capacity) # Buffer of generated tgt images
self.tgt2src_buffer = util.ImageBuffer(buffer_capacity) # Buffer of generated src images
else:
self._data_parallel()
# Images in flow src -> lat -> tgt
self.src = None
self.src2lat = None
self.src2tgt = None
# Images in flow tgt -> lat -> src
self.tgt = None
self.tgt2lat = None
self.tgt2src = None
# Jacobian clamping tensors
self.src_jc = None
self.tgt_jc = None
self.src2tgt_jc = None
self.tgt2src_jc = None
# Discriminator loss
self.loss_d_tgt = None
self.loss_d_src = None
self.loss_d = None
# Generator GAN loss
self.loss_gan_src = None
self.loss_gan_tgt = None
self.loss_gan = None
# Generator MLE loss
self.loss_mle_src = None
self.loss_mle_tgt = None
self.loss_mle = None
# Jacobian Clamping loss
self.loss_jc_src = None
self.loss_jc_tgt = None
self.loss_jc = None
# Generator total loss
self.loss_g = None
def __init__(self, args):
super(ResNet, self).__init__()
num_blocks = 9
num_channels = args.num_channels
num_filters = args.num_channels_g
norm_layer = get_norm_layer(args.norm_type)
use_bias = args.norm_type == 'instance'
layers = [
nn.ReflectionPad2d(3),
nn.Conv2d(num_channels,
num_filters,
kernel_size=7,
padding=0,
bias=use_bias),
norm_layer(num_filters),
nn.ReLU(inplace=True)
]
num_down = 2 # Number of times to down-sample
for i in range(num_down):
k = 2**i
layers += [
nn.Conv2d(k * num_filters,
2 * k * num_filters,
kernel_size=3,
stride=2,
padding=1,
bias=use_bias),
norm_layer(2 * k * num_filters),
nn.ReLU(inplace=True)
]
k = 2**num_down
layers += [
ResNetBlock(k * num_filters, norm_layer, use_bias=use_bias)
for _ in range(num_blocks)
]
for i in range(num_down):
k = 2**(num_down - i)
layers += [
nn.ConvTranspose2d(k * num_filters,
int(k * num_filters / 2),
kernel_size=3,
stride=2,
padding=1,
output_padding=1,
bias=use_bias),
norm_layer(int(k * num_filters / 2)),
nn.ReLU(inplace=True)
]
layers += [
nn.ReflectionPad2d(3),
nn.Conv2d(num_filters, num_channels, kernel_size=7, padding=0),
nn.Tanh()
]
self.model = nn.Sequential(*layers)
init_model(self.model, init_method=args.initializer)