from datasets.nist import nist
from datasets.dvmm import dvmm
from datasets.swapme import swapme
import numpy as np
# Set up voc_<year>_<split>
for year in ['2007', '2012']:
for split in ['train', 'val', 'trainval', 'test']:
name = 'voc_{}_{}'.format(year, split)
__sets[name] = (lambda split=split, year=year: pascal_voc(split, year))
# Set up coco_2014_<split>
for year in ['2014']:
for split in ['train', 'val', 'minival', 'valminusminival', 'trainval']:
name = 'coco_{}_{}'.format(year, split)
__sets[name] = (lambda split=split, year=year: coco(split, year))
# Set up coco_2015_<split>
for year in ['2015']:
for split in ['test', 'test-dev']:
name = 'COCO_{}_{}'.format(year, split)
__sets[name] = (lambda split=split, year=year: coco(split, year))
dvmm_path = '/vulcan/scratch/pengzhou/dataset/4cam_splc'
for split in ['dist_train', 'dist_test']:
name = split
__sets[name] = (lambda split=split: dvmm(split, 2007, dvmm_path))
dso_path = '/vulcan/scratch/pengzhou/dataset/COVERAGE'
for split in ['dist_cover_train_single', 'dist_cover_test_single']:
name = split
from datasets.pascal_voc import pascal_voc
from datasets.coco import coco
from datasets.grotoap2 import grotoap2
import numpy as np
# Set up voc_<year>_<split> using selective search "fast" mode
for year in ['2007', '2012']:
for split in ['train', 'val', 'trainval', 'test']:
name = 'voc_{}_{}'.format(year, split)
__sets[name] = (lambda split=split, year=year: pascal_voc(split, year))
# Set up coco_2014_<split>
for year in ['2014']:
for split in ['train', 'val', 'minival', 'valminusminival']:
name = 'coco_{}_{}'.format(year, split)
__sets[name] = (lambda split=split, year=year: coco(split, year))
# Set up coco_2015_<split>
for year in ['2015']:
for split in ['test', 'test-dev']:
name = 'coco_{}_{}'.format(year, split)
__sets[name] = (lambda split=split, year=year: coco(split, year))
# Set up grotoap2
for split in ['train', 'val', 'trainval', 'test']:
name = 'grotoap2_{}'.format(split)
__sets[name] = (lambda split=split: grotoap2(split))
def get_imdb(name):
"""Get an imdb (image database) by name."""
def test_scene_model(config):
output_dir = osp.join(config.model_dir, 'test_scene_model')
maybe_create(output_dir)
plt.switch_backend('agg')
transformer = image_normalize('background')
db = coco(config, 'train', transform=transformer)
pca_table = AllCategoriesTables(db)
pca_table.run_PCAs_and_build_nntables_in_feature_space()
loader = DataLoader(db,
batch_size=config.batch_size, shuffle=False,
num_workers=config.num_workers)
net = SceneModel(db)
net.eval()
for cnt, batched in enumerate(loader):
word_inds = batched['word_inds'].long()
word_lens = batched['word_lens'].long()
bg_images = batched['background'].float()
fg_inds = batched['fg_inds'].long()
gt_inds = batched['out_inds'].long()
gt_vecs = batched['out_vecs'].float()
gt_msks = batched['out_msks'].float()
fg_onehots = indices2onehots(fg_inds, config.output_vocab_size)
# inf_outs, _ = net((word_inds, word_lens, bg_images, fg_onehots))
# obj_logits, coord_logits, attri_logits, pca_vectors, enc_msks, what_wei, where_wei = inf_outs
# print('teacher forcing')
# print('obj_logits ', obj_logits.size())
# print('coord_logits ', coord_logits.size())
# print('attri_logits ', attri_logits.size())
# print('pca_vectors ', pca_vectors.size())
# if config.what_attn:
# print('what_att_logits ', what_wei.size())
# if config.where_attn > 0:
# print('where_att_logits ', where_wei.size())
# print('----------------------')
inf_outs, env = net.inference(word_inds, word_lens, -1, 0, 0, gt_inds, gt_vecs)
# inf_outs, env = net.inference(word_inds, word_lens, -1, 2.0, 0, None, None)
obj_logits, coord_logits, attri_logits, pca_vectors, enc_msks, what_wei, where_wei = inf_outs
print('scheduled sampling')
print('obj_logits ', obj_logits.size())
print('coord_logits ', coord_logits.size())
print('attri_logits ', attri_logits.size())
print('pca_vectors ', pca_vectors.size())
if config.what_attn:
print('what_att_logits ', what_wei.size())
if config.where_attn > 0:
print('where_att_logits ', where_wei.size())
print('----------------------')
sequences = env.batch_redraw(True)
for i in range(len(sequences)):
sequence = sequences[i]
image_idx = batched['image_index'][i]
name = '%03d_'%i + str(image_idx).zfill(12)
out_path = osp.join(output_dir, name+'.png')
color = cv2.imread(batched['color_path'][i], cv2.IMREAD_COLOR)
color, _, _ = create_squared_image(color)
fig = plt.figure(figsize=(32, 16))
plt.suptitle(batched['sentence'][i], fontsize=30)
for j in range(min(len(sequence), 14)):
plt.subplot(3, 5, j+1)
partially_completed_img = clamp_array(sequence[j], 0, 255).astype(np.uint8)
partially_completed_img = partially_completed_img[:,:,::-1]
plt.imshow(partially_completed_img)
plt.axis('off')
plt.subplot(3, 5, 15)
plt.imshow(color[:,:,::-1])
plt.axis('off')
fig.savefig(out_path, bbox_inches='tight')
plt.close(fig)
break
from datasets.pascal_voc import pascal_voc
from datasets.coco import coco
from datasets.detect_task import detect_task
import numpy as np
# Set up voc_<year>_<split> using selective search "fast" mode
for year in ['2007', '2012']:
for split in ['train', 'val', 'trainval', 'test']:
name = 'voc_{}_{}'.format(year, split)
__sets[name] = (lambda split=split, year=year: pascal_voc(split, year))
# Set up coco_2014_<split>
for year in ['2014']:
for split in ['train', 'val', 'minival', 'valminusminival']:
name = 'coco_{}_{}'.format(year, split)
__sets[name] = (lambda split=split, year=year: coco(split, year))
# Set up coco_2015_<split>
for year in ['2015']:
for split in ['test', 'test-dev']:
name = 'coco_{}_{}'.format(year, split)
__sets[name] = (lambda split=split, year=year: coco(split, year))
def get_imdb(name):
"""Get an imdb (image database) by name."""
if not __sets.has_key(name):
__sets[name] = (lambda task_name=name: detect_task(task_name))
return __sets[name]()
# raise KeyError('Unknown dataset: {}'.format(name))
return __sets[name]()
def test_step_by_step(config):
db = coco(config, 'train', '2017')
output_dir = osp.join(config.model_dir, 'test_step_by_step')
maybe_create(output_dir)
all_tables = AllCategoriesTables(db)
all_tables.build_nntables_for_all_categories(True)
seq_db = sequence_loader(db, all_tables)
env = simulator(db, config.batch_size, all_tables)
env.reset()
loader = DataLoader(seq_db,
batch_size=config.batch_size,
shuffle=True,
num_workers=config.num_workers)
for cnt, batched in enumerate(loader):
out_inds = batched['out_inds'].long().numpy()
out_vecs = batched['out_vecs'].float().numpy()
sequences = []
for i in range(out_inds.shape[1]):
frames = env.batch_render_to_pytorch(out_inds[:, i], out_vecs[:,
i])
sequences.append(frames)
sequences = torch.stack(sequences, dim=1)
# sequences = [tensors_to_vols(x) for x in sequences]
for i in range(len(sequences)):
sequence = sequences[i]
image_idx = batched['image_index'][i]
name = '%03d_' % i + str(image_idx).zfill(12)
out_path = osp.join(output_dir, name + '.png')
color = cv2.imread(batched['image_path'][i], cv2.IMREAD_COLOR)
color, _, _ = create_squared_image(color)
fig = plt.figure(figsize=(32, 32))
plt.suptitle(batched['sentence'][i], fontsize=30)
for j in range(min(len(sequence), 14)):
plt.subplot(4, 4, j + 1)
seq_np = sequence[j].cpu().data.numpy()
if config.use_color_volume:
partially_completed_img, _ = heuristic_collage(seq_np, 83)
else:
partially_completed_img = seq_np[:, :, -3:]
partially_completed_img = clamp_array(partially_completed_img,
0, 255).astype(np.uint8)
partially_completed_img = partially_completed_img[:, :, ::-1]
plt.imshow(partially_completed_img)
plt.axis('off')
plt.subplot(4, 4, 16)
plt.imshow(color[:, :, ::-1])
plt.axis('off')
fig.savefig(out_path, bbox_inches='tight')
plt.close(fig)
break
def test_coco_dataloader(config):
db = coco(config, 'train', '2017')
all_tables = AllCategoriesTables(db)
all_tables.build_nntables_for_all_categories(True)
sequence_db = sequence_loader(db, all_tables)
output_dir = osp.join(config.model_dir, 'test_coco_dataloader')
maybe_create(output_dir)
loader = DataLoader(sequence_db,
batch_size=config.batch_size,
shuffle=True,
num_workers=config.num_workers)
start = time()
for cnt, batched in enumerate(loader):
x = batched['background'].float()
y = batched['foreground'].float()
z = batched['negative'].float()
# x = sequence_onehot_volumn_preprocess(x, len(db.classes))
x = sequence_color_volumn_preprocess(x, len(db.classes))
y = sequence_onehot_volumn_preprocess(y, len(db.classes))
z = sequence_onehot_volumn_preprocess(z, len(db.classes))
# cv2.imwrite('mask0.png', y[0,2,:,:,-4].cpu().data.numpy())
# cv2.imwrite('mask1.png', y[1,2,:,:,-4].cpu().data.numpy())
# cv2.imwrite('mask2.png', y[2,2,:,:,-4].cpu().data.numpy())
# cv2.imwrite('mask3.png', y[3,2,:,:,-4].cpu().data.numpy())
# cv2.imwrite('label0.png', y[0,2,:,:,3].cpu().data.numpy())
# cv2.imwrite('label1.png', y[1,2,:,:,3].cpu().data.numpy())
# cv2.imwrite('label2.png', y[2,2,:,:,3].cpu().data.numpy())
# cv2.imwrite('label3.png', y[3,2,:,:,3].cpu().data.numpy())
# cv2.imwrite('color0.png', y[0,2,:,:,-3:].cpu().data.numpy())
# cv2.imwrite('color1.png', y[1,2,:,:,-3:].cpu().data.numpy())
# cv2.imwrite('color2.png', y[2,2,:,:,-3:].cpu().data.numpy())
# cv2.imwrite('color3.png', y[3,2,:,:,-3:].cpu().data.numpy())
# cv2.imwrite('bg0.png', x[0,3,:,:,9:12].cpu().data.numpy())
# cv2.imwrite('bg1.png', x[1,3,:,:,9:12].cpu().data.numpy())
# cv2.imwrite('bg2.png', x[2,3,:,:,9:12].cpu().data.numpy())
# cv2.imwrite('bg3.png', x[3,3,:,:,9:12].cpu().data.numpy())
x = (x - 128.0).permute(0, 1, 4, 2, 3)
y = (y - 128.0).permute(0, 1, 4, 2, 3)
z = (z - 128.0).permute(0, 1, 4, 2, 3)
print('background', x.size())
print('foreground', y.size())
print('negative', z.size())
print('word_inds', batched['word_inds'].size())
print('word_lens', batched['word_lens'].size())
print('fg_inds', batched['fg_inds'].size())
print('patch_inds', batched['patch_inds'].size())
print('out_inds', batched['out_inds'].size())
print('out_msks', batched['out_msks'].size())
print('foreground_resnets', batched['foreground_resnets'].size())
print('negative_resnets', batched['negative_resnets'].size())
print('foreground_resnets', batched['foreground_resnets'][0, 0])
print('negative_resnets', batched['negative_resnets'][0, 0])
print('out_msks', batched['out_msks'][0])
print('patch_inds', batched['patch_inds'][0])
plt.switch_backend('agg')
bg_images = x
fg_images = y
neg_images = z
bsize, ssize, n, h, w = bg_images.size()
bg_images = bg_images.view(bsize * ssize, n, h, w)
bg_images = tensors_to_vols(bg_images)
bg_images = bg_images.reshape(bsize, ssize, h, w, n)
bsize, ssize, n, h, w = fg_images.size()
fg_images = fg_images.view(bsize * ssize, n, h, w)
fg_images = tensors_to_vols(fg_images)
fg_images = fg_images.reshape(bsize, ssize, h, w, n)
bsize, ssize, n, h, w = neg_images.size()
neg_images = neg_images.view(bsize * ssize, n, h, w)
neg_images = tensors_to_vols(neg_images)
neg_images = neg_images.reshape(bsize, ssize, h, w, n)
for i in range(bsize):
bg_seq = bg_images[i]
fg_seq = fg_images[i]
neg_seq = neg_images[i]
image_idx = batched['image_index'][i]
fg_inds = batched['fg_inds'][i]
name = '%03d_' % i + str(image_idx).zfill(12)
out_path = osp.join(output_dir, name + '.png')
color = cv2.imread(batched['image_path'][i], cv2.IMREAD_COLOR)
color, _, _ = create_squared_image(color)
fig = plt.figure(figsize=(48, 32))
plt.suptitle(batched['sentence'][i], fontsize=30)
for j in range(min(len(bg_seq), 15)):
bg, _ = heuristic_collage(bg_seq[j], 83)
bg_mask = 255 * np.ones((bg.shape[1], bg.shape[0]))
row, col = np.where(np.sum(np.absolute(bg), -1) == 0)
bg_mask[row, col] = 0
# bg = bg_seq[j][:,:,-3:]
# bg_mask = bg_seq[j][:,:,-4]
bg_mask = np.repeat(bg_mask[..., None], 3, -1)
fg_color = fg_seq[j][:, :, -3:]
# fg_mask = fg_seq[j][:,:,fg_inds[j+1]]
fg_mask = fg_seq[j][:, :, -4]
neg_color = neg_seq[j][:, :, -3:]
# neg_mask = neg_seq[j][:,:,fg_inds[j+1]]
neg_mask = neg_seq[j][:, :, -4]
color_pair = np.concatenate((fg_color, neg_color), 1)
mask_pair = np.concatenate((fg_mask, neg_mask), 1)
mask_pair = np.repeat(mask_pair[..., None], 3, -1)
patch = np.concatenate((color_pair, mask_pair), 0)
patch = cv2.resize(patch, (bg.shape[1], bg.shape[0]))
partially_completed_img = np.concatenate((bg, bg_mask, patch),
1)
partially_completed_img = clamp_array(partially_completed_img,
0, 255).astype(np.uint8)
partially_completed_img = partially_completed_img[:, :, ::-1]
plt.subplot(4, 4, j + 1)
plt.imshow(partially_completed_img)
plt.axis('off')
plt.subplot(4, 4, 16)
plt.imshow(color[:, :, ::-1])
plt.axis('off')
fig.savefig(out_path, bbox_inches='tight')
plt.close(fig)
if cnt == 3:
break
print("Time", time() - start)
def test_puzzle_model(config):
output_dir = osp.join(config.model_dir, 'test_puzzle_model')
maybe_create(output_dir)
plt.switch_backend('agg')
db = coco(config, 'train', '2017')
all_tables = AllCategoriesTables(db)
all_tables.build_nntables_for_all_categories(True)
sequence_db = sequence_loader(db, all_tables)
loader = DataLoader(sequence_db,
batch_size=config.batch_size, shuffle=False,
num_workers=config.num_workers)
net = PuzzleModel(db)
net.eval()
for cnt, batched in enumerate(loader):
word_inds = batched['word_inds'].long()
word_lens = batched['word_lens'].long()
bg_images = batched['background'].float()
fg_images = batched['foreground'].float()
neg_images = batched['negative'].float()
fg_resnets = batched['foreground_resnets'].float()
neg_resnets = batched['negative_resnets'].float()
fg_inds = batched['fg_inds'].long()
gt_inds = batched['out_inds'].long()
gt_msks = batched['out_msks'].float()
fg_onehots = indices2onehots(fg_inds, config.output_vocab_size)
inf_outs, _, positive_feats, negative_feats = net((word_inds, word_lens, bg_images, fg_onehots, fg_images, neg_images, fg_resnets, neg_resnets))
obj_logits, coord_logits, attri_logits, patch_vectors, enc_msks, what_wei, where_wei = inf_outs
print('teacher forcing')
print('obj_logits ', obj_logits.size())
print('coord_logits ', coord_logits.size())
print('attri_logits ', attri_logits.size())
print('patch_vectors ', patch_vectors.size())
print('patch_vectors max:', torch.max(patch_vectors))
print('patch_vectors min:', torch.min(patch_vectors))
print('patch_vectors norm:', torch.norm(patch_vectors, dim=-2)[0,0,0])
print('positive_feats ', positive_feats.size())
print('negative_feats ', negative_feats.size())
if config.what_attn:
print('what_att_logits ', what_wei.size())
if config.where_attn > 0:
print('where_att_logits ', where_wei.size())
print('----------------------')
_, pred_vecs = net.collect_logits_and_vectors(inf_outs, gt_inds)
print('pred_vecs', pred_vecs.size())
print('*******************')
# # inf_outs, env = net.inference(word_inds, word_lens, -1, 0.0, 0, gt_inds, gt_vecs)
# inf_outs, env = net.inference(word_inds, word_lens, -1, 2.0, 0, None, None, all_tables)
# obj_logits, coord_logits, attri_logits, patch_vectors, enc_msks, what_wei, where_wei = inf_outs
# print('scheduled sampling')
# print('obj_logits ', obj_logits.size())
# print('coord_logits ', coord_logits.size())
# print('attri_logits ', attri_logits.size())
# print('patch_vectors ', patch_vectors.size())
# if config.what_attn:
# print('what_att_logits ', what_wei.size())
# if config.where_attn > 0:
# print('where_att_logits ', where_wei.size())
# print('----------------------')
# sequences = env.batch_redraw(True)
# for i in range(len(sequences)):
# sequence = sequences[i]
# image_idx = batched['image_index'][i]
# name = '%03d_'%i + str(image_idx).zfill(12)
# out_path = osp.join(output_dir, name+'.png')
# color = cv2.imread(batched['color_path'][i], cv2.IMREAD_COLOR)
# color, _, _ = create_squared_image(color)
# fig = plt.figure(figsize=(32, 16))
# plt.suptitle(batched['sentence'][i], fontsize=30)
# for j in range(min(len(sequence), 14)):
# plt.subplot(3, 5, j+1)
# partially_completed_img = clamp_array(sequence[j][:,:,-3:], 0, 255).astype(np.uint8)
# partially_completed_img = partially_completed_img[:,:,::-1]
# plt.imshow(partially_completed_img)
# plt.axis('off')
# plt.subplot(3, 5, 15)
# plt.imshow(color[:,:,::-1])
# plt.axis('off')
# fig.savefig(out_path, bbox_inches='tight')
# plt.close(fig)
break
from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
import os
import os.path as osp
import sys
this_dir = osp.dirname(__file__)
sys.path.insert(0, osp.join(this_dir, '../tools'))
import _init_paths
from datasets.coco import coco
from pprint import pprint
import time
imdb = coco('minival', '2014')
imdb.append_flipped_images()
print(len(imdb.roidb))
print(imdb.roidb[0].keys())
# # check image_set and roidb
# image_set = loader.image_set # split -> image_ids
# roidb = loader.roidb
# for split in ['train', 'val', 'testA', 'testB']:
# print('[%s] has %s roidb instances and %s images.' % (split, len(roidb), len(image_set[split])))
# # check appending
# print('After appending flipped...')
# loader.append_flipped_images('train')
# for split in ['train', 'val', 'testA', 'testB']:
# print('[%s] has %s roidb instances and %s images.' % (split, len(roidb), len(image_set[split])))
name = split
__sets[name] = (lambda split=split: dist_fake(split,2007,cover_path))
nist_path='/media/li/Li/NIST2016'
for split in ['dist_NIST_train_new_2', 'dist_NIST_test_new_2']:
name = split
__sets[name] = (lambda split=split: nist(split,2007,nist_path))
casia_path='/media/li/Data/CASIA'
#for split in ['casia_train_all_single', 'casia_test_all_1']:
for split in ['casia_train_all_single', 'casia_test_all_single']:
name = split
__sets[name] = (lambda split=split: casia(split,2007,casia_path))
coco_path='/media/li/Li/filter_tamper'
for split in ['coco_train_filter_single', 'coco_test_filter_single']:
name = split
__sets[name] = (lambda split=split: coco(split,2007,coco_path))
def get_imdb(name):
"""Get an imdb (image database) by name."""
if name not in __sets:
raise KeyError('Unknown dataset: {}'.format(name))
return __sets[name]()
def list_imdbs():
"""List all registered imdbs."""
return list(__sets.keys())
status = subprocess.call(cmd, shell=True)
def evaluate_detections(self, all_boxes, output_dir):
self._write_voc_results_file(all_boxes)
self._do_python_eval(output_dir)
#if self.config['matlab_eval']:
#self._do_matlab_eval(output_dir)
if self.config['cleanup']:
for cls in self._classes:
if cls == '__background__' or cls == self.classes[0]:
continue
filename = self._get_voc_results_file_template().format(cls)
#os.remove(filename)
def competition_mode(self, on):
if on:
self.config['use_salt'] = False
self.config['cleanup'] = False
else:
self.config['use_salt'] = True
self.config['cleanup'] = True
if __name__ == '__main__':
from datasets.coco import coco
d = coco('train', '2007')
res = d.roidb
from IPython import embed
embed()
def __init__(self, image_set, year, devkit_path=None, shuffled=None):
imdb.__init__(self, 'merged_' + image_set)
self._year = year
self._image_set = image_set
self._anno_set_dir = image_set
if "val" in image_set:
self._image_set_dir = "val"
if "val1" in image_set:
self._anno_set_dir = "val1"
if "val2" in image_set:
self._anno_set_dir = "val2"
elif "train" in image_set:
self._anno_set_dir = "train"
elif "test" in image_set:
self._anno_set_dir = "test"
if image_set == "train":
self.imdbs = [imagenet(image_set),\
coco(image_set, '2015'),\
cam2(image_set,'2017'),\
sun(image_set,'2012'),\
caltech(image_set,'2009'),\
kitti(image_set,'2013'),\
inria(image_set,'2005'),\
pascal_voc(image_set,'2007'),\
pascal_voc(image_set,'2012')]
elif image_set == "test":
self.imdbs = [imagenet('val'),\
coco('test-dev', '2015'),\
cam2('all','2017'),\
sun('test','2012'),\
caltech('test','2009'),\
kitti('val','2013'),\
inria('all','2005'),\
pascal_voc('test','2007')]
self.roidbs = [None for _ in range(len(self.datasets))]
for idx,imdb in enumerate(self.imdbs):
self.roidbs[idx] = get_training_roidb(imdb)
self._devkit_path = self._get_default_path() if devkit_path is None \
else devkit_path
self._data_path = os.path.join(self._devkit_path, 'VOC' + self._year)
self._classes = ('__background__', # always index 0
'voc',
'imagenet',
'caltech',
'coco',
'sun',
'kitti',
'inria',
'cam2')
self._class_to_ind = dict(zip(self.classes, xrange(self.num_classes)))
self._image_ext = '.jpg'
self._image_index = self._load_image_set_index()
# Default to roidb handler
self._roidb_handler = self.selective_search_roidb
self._salt = str(uuid.uuid4())
self._comp_id = 'comp4'
# PASCAL specific config options
self.config = {'cleanup' : True,
'use_salt' : True,
'use_diff' : False,
'matlab_eval' : False,
'rpn_file' : None,
'min_size' : 2}
assert os.path.exists(self._devkit_path), \
'VOCdevkit path does not exist: {}'.format(self._devkit_path)
assert os.path.exists(self._data_path), \
'Path does not exist: {}'.format(self._data_path)
