def __init__(
self,
dir_data,
split='train',
batch_size=4,
shuffle=False,
pin_memory=False,
nb_threads=4,
*args,
**kwargs):
self.dir_data = dir_data
self.split = split
self.batch_size = batch_size
self.shuffle = shuffle
self.pin_memory = pin_memory
self.nb_threads = nb_threads
self.sampler = None
self.collate_fn = btf.Compose([
btf.ListDictsToDictLists(),
btf.StackTensors()
])
self.nb_items = kwargs['nb_items']
self.data = torch.randn(self.nb_items, 10)
self.target = torch.zeros(self.nb_items)
self.target[:int(self.nb_items / 2)].fill_(1)
def filter_img(X, classes, nb_points):
X_new = {
'path': [],
'class_name': [],
'index': [],
'data': [],
'class_id': [],
'idx': []
}
idx = 0
for c in classes:
for i in range(len(X['index'])):
if c == X['class_name'][i]:
X_new['path'].append(X['path'][i])
X_new['class_name'].append(X['class_name'][i])
X_new['index'].append(X['index'][i])
X_new['data'].append(X['data'][i])
X_new['class_id'].append(X['class_id'][i])
X_new['idx'].append(X['idx'][i])
idx += 1
if idx >= nb_points:
break
# if idx < nb_points:
# Logger()('Warning: classe {} has {} items'.format(c, idx))
X_new['data'] = transforms.StackTensors()(X_new['data'])
return X_new
def __init__(self,
dir_data='data/mnist',
split='train',
batch_size=100,
nb_threads=1,
pin_memory=True):
self.dir_data = dir_data
self.split = split
self.batch_size = batch_size
self.nb_threads = nb_threads
self.pin_memory = pin_memory
if self.split == 'train':
is_train = True
self.shuffle = True
elif self.split == 'val':
is_train = False
self.shuffle = False
else:
raise ValueError()
self.item_tf = transforms.Compose([
transforms.ToTensor(),
transforms.Normalize((0.1307, ), (0.3081, ))
])
download = (not os.path.isdir(self.dir_data))
self.dataset = datasets.MNIST(self.dir_data,
train=is_train,
download=download,
transform=self.item_tf)
# the usual collate function for bootstrap
# to handle the (potentially nested) dict item format bellow
self.collate_fn = bootstrap_tf.Compose(
[bootstrap_tf.ListDictsToDictLists(),
bootstrap_tf.StackTensors()])
def stack():
return transforms.Compose(
[transforms.ListDictsToDictLists(),
transforms.StackTensors()])
def __init__(self,
dir_data='/local/cadene/data/vqa',
split='train',
batch_size=80,
nb_threads=4,
pin_memory=False,
shuffle=False,
nans=1000,
minwcount=10,
nlp='mcb',
proc_split='train',
samplingans=False,
has_valset=True,
has_testset=True,
has_testset_anno=False,
has_testdevset=True,
has_answers_occurence=True,
do_tokenize_answers=False):
super(AbstractVQA, self).__init__(dir_data=dir_data,
split=split,
batch_size=batch_size,
nb_threads=nb_threads,
pin_memory=pin_memory,
shuffle=shuffle)
self.nans = nans
self.minwcount = minwcount
self.nlp = nlp
self.proc_split = proc_split
self.samplingans = samplingans
# preprocessing
self.has_valset = has_valset
self.has_testset = has_testset
self.has_testset_anno = has_testset_anno
self.has_testdevset = has_testdevset
self.has_answers_occurence = has_answers_occurence
self.do_tokenize_answers = do_tokenize_answers
# sanity checks
if self.split in ['test', 'val'] and self.samplingans:
raise ValueError()
self.dir_raw = os.path.join(self.dir_data, 'raw')
if not os.path.exists(self.dir_raw):
self.download()
self.dir_processed = os.path.join(self.dir_data, 'processed')
self.subdir_processed = self.get_subdir_processed()
self.path_wid_to_word = osp.join(self.subdir_processed,
'wid_to_word.pth')
self.path_word_to_wid = osp.join(self.subdir_processed,
'word_to_wid.pth')
self.path_aid_to_ans = osp.join(self.subdir_processed,
'aid_to_ans.pth')
self.path_ans_to_aid = osp.join(self.subdir_processed,
'ans_to_aid.pth')
self.path_trainset = osp.join(self.subdir_processed, 'trainset.pth')
self.path_valset = osp.join(self.subdir_processed, 'valset.pth')
self.path_is_qid_testdev = osp.join(self.subdir_processed,
'is_qid_testdev.pth')
self.path_testset = osp.join(self.subdir_processed, 'testset.pth')
if not os.path.exists(self.subdir_processed):
self.process()
self.wid_to_word = torch.load(self.path_wid_to_word)
self.word_to_wid = torch.load(self.path_word_to_wid)
self.aid_to_ans = torch.load(self.path_aid_to_ans)
self.ans_to_aid = torch.load(self.path_ans_to_aid)
if 'train' in self.split:
self.dataset = torch.load(self.path_trainset)
elif self.split == 'val':
if self.proc_split == 'train':
self.dataset = torch.load(self.path_valset)
elif self.proc_split == 'trainval':
self.dataset = torch.load(self.path_trainset)
elif self.split == 'test':
self.dataset = torch.load(self.path_testset)
if self.has_testdevset:
self.is_qid_testdev = torch.load(self.path_is_qid_testdev)
self.collate_fn = bootstrap_tf.Compose([
bootstrap_tf.ListDictsToDictLists(),
bootstrap_tf.PadTensors(use_keys=[
'question', 'pooled_feat', 'cls_scores', 'rois', 'cls',
'cls_oh', 'norm_rois'
]),
#bootstrap_tf.SortByKey(key='lengths'), # no need for the current implementation
bootstrap_tf.StackTensors()
])
if self.proc_split == 'trainval' and self.split in ['train', 'val']:
self.bootstrapping()
self.qid_to_idx = {
item['question_id']: idx
for idx, item in enumerate(self.dataset['questions'])
}
def __init__(self,
dir_data,
split,
win_size,
im_size,
layer, # "goal" or "cause"
frame_position,
traintest_mode,
fps=10,
horizon=2, # in seconds
extract_mode=False,
batch_size=2,
debug=False,
shuffle=False,
pin_memory=False,
nb_threads=0):
self.win_size = win_size
self.frame_position = frame_position
super(HDDClassif, self).__init__(dir_data,
split,
im_size,
fps,
horizon, # in seconds
batch_size,
debug,
shuffle,
pin_memory,
nb_threads)
self.layer = layer
if self.layer == "cause":
self.layer_id = '1'
self.classid_to_ix = [-1, 16, 17, 18, 19, 20, 22]
elif self.layer == "goal":
self.layer_id = '0'
self.classid_to_ix = [-1, 0, 1, 2, 3, 4, 5, 7, 8, 10, 11, 12]
else:
raise ValueError(self.layer)
# The classid 0 is the background class
self.ix_to_classid = dict((ix, classid) for classid, ix in enumerate(self.classid_to_ix))
self.class_freq = self.get_class_freq()
self.collate_fn = bootstrap_tf.Compose([
bootstrap_tf.ListDictsToDictLists(),
bootstrap_tf.StackTensors()
])
self.dir_navig_features = self.dir_processed_annot
self.im_transform = transforms.Compose([transforms.Resize((self.im_h, self.im_w)),
transforms.ToTensor(),
transforms.Normalize(mean = [0.43216, 0.394666, 0.37645],
std = [0.22803, 0.22145, 0.216989])])
self.traintest_mode = traintest_mode
if self.traintest_mode:
self.make_batch_loader = self._make_batch_loader_traintest
else:
self.make_batch_loader = self._make_batch_loader
def default_items_tf():
return transforms.Compose([
transforms.ListDictsToDictLists(),
transforms.PadTensors(value=0),
transforms.StackTensors()
])
def __init__(
self,
dir_data,
dir_coco,
dir_vg,
split,
val_size=0.05,
image_features="default",
background_coco=None,
background_vg=None,
background=False,
background_merge=2,
proportion_opposite=0.0, # not used
train_selection=None, # not used
no_features=False,
path_questions=None,
sampling=None,
shuffle=None,
batch_size=None,
):
super().__init__()
self.dir_data = dir_data
self.dir_coco = dir_coco
self.dir_vg = dir_vg
self.split = split
self.image_features = image_features
self.dir_coco_lvis = "data/vqa/coco/extract_rcnn/lvis"
self.dir_vg_lvis = "data/vqa/vgenome/extract_rcnn/lvis"
self.background_coco = background_coco
self.background_vg = background_vg
self.background = background
self.background_merge = background_merge
self.no_features = no_features
self.val_size = val_size
self.path_questions = path_questions # to override path to questions (default dir_data/split.json)
self.sampling = sampling
self.shuffle = shuffle
self.batch_size = batch_size
if self.dir_coco.endswith(".zip"):
self.zip_coco = None # lazy loading zipfile.ZipFile(self.dir_coco)
if self.dir_vg.endswith(".zip"):
self.zip_vg = None # lazy loading zipfile.ZipFile(self.dir_vg)
if self.background_coco is not None and self.background_coco.endswith(".zip"):
self.zip_bg_coco = None # zipfile.ZipFile(self.background_coco)
if self.background_vg is not None and self.background_vg.endswith(".zip"):
self.zip_bg_vg = None # lazy loading zipfile.ZipFile(self.background_vg)
if self.dir_coco.endswith(".lmdb"):
self.lmdb_coco = None
if self.split not in ["train", "test"]:
self.process_split()
# path = os.path.join(self.dir_data, "processed", "questions.json")
q_path = self.get_path_questions() # train or test
Logger()("Loading questions")
with open(q_path) as f:
self.questions = json.load(f)
self.path_wid_to_word = os.path.join(
self.dir_data, "processed", "wid_to_word.pth"
)
if os.path.exists(self.path_wid_to_word):
self.wid_to_word = torch.load(self.path_wid_to_word)
else:
os.makedirs(os.path.join(self.dir_data, "processed"), exist_ok=True)
word_list = self.get_token_list()
self.wid_to_word = {wid + 1: word for wid, word in enumerate(word_list)}
torch.save(self.wid_to_word, self.path_wid_to_word)
self.word_to_wid = {word: wid for wid, word in self.wid_to_word.items()}
self.aid_to_ans = [str(a) for a in list(range(16))]
self.ans_to_aid = {ans: i for i, ans in enumerate(self.aid_to_ans)}
self.collate_fn = bootstrap_tf.Compose(
[
bootstrap_tf.ListDictsToDictLists(),
bootstrap_tf.PadTensors(
use_keys=[
"question",
"pooled_feat",
"cls_scores",
"rois",
"cls",
"cls_oh",
"norm_rois",
]
),
# bootstrap_tf.SortByKey(key='lengths'), # no need for the current implementation
bootstrap_tf.StackTensors(),
]
)
def main():
global args
args = parser.parse_args()
#dir_root = '/home/carvalho/experiments/im2recipe.pytorch/logs/lmdb/2017_10_10_23_54_31_517_anm_IRR1.0_RII1.0_SIRR0.1_SRII0.1_80epochs'
dir_root = '/home/carvalho/experiments/im2recipe.pytorch/logs/lmdb/2017_10_06_08_10_47_631_anm_IRR_RII_80epochs'
dir_embs = os.path.join(dir_root, 'embeddings_train')
dir_img_items = os.path.join(dir_embs, 'img')
dir_rcp_items = os.path.join(dir_embs, 'rcp')
#dir_img_items = os.path.join(dir_img_embs, '2017_10_10_23_54_31_517_anm_IRR1.0_RII1.0_SIRR0.1_SRII0.1_80epochs')
#dir_rcp_items = os.path.join(dir_rcp_embs, '2017_10_10_23_54_31_517_anm_IRR1.0_RII1.0_SIRR0.1_SRII0.1_80epochs')
path_load_img_out = os.path.join(dir_embs, 'load_img_out.pth')
path_load_rcp_out = os.path.join(dir_embs, 'load_rcp_out.pth')
dir_visu = os.path.join(dir_root, 'visualizations')
dir_fig = os.path.join(dir_visu, 'fig_train_6')
os.system('mkdir -p '+dir_fig)
Logger(dir_fig)
Logger()('Begin')
rcp_items = []
img_items = []
if not os.path.isfile(path_load_img_out):
Logger()('Loading embeddings...')
for filename in os.listdir(dir_img_items):
idx = int(filename.split('.')[0].split('_')[-1])
#
path_item = os.path.join(dir_img_items, filename)
img_item = torch.load(path_item)
img_item['idx'] = idx
img_items.append(img_item)
#
path_item = path_item.replace('img', 'rcp')
rcp_item = torch.load(path_item)
rcp_item['idx'] = idx
rcp_items.append(rcp_item)
Logger()('Stacking image items...')
X_img = stack()(img_items)
torch.save(X_img, path_load_img_out)
Logger()('Stacking rcpipe items...')
X_rcp = stack()(rcp_items)
torch.save(X_rcp, path_load_rcp_out)
else:
X_img = torch.load(path_load_img_out)
X_rcp = torch.load(path_load_rcp_out)
# all_classes = list(set(X_img['class_name']))
# nb_items_by_class = {class_name:0 for class_name in all_classes}
# for i in range(len(X_img['index'])):
# class_name = X_img['class_name'][i]
# nb_items_by_class[class_name] += 1
# for key, value in sorted(nb_items_by_class.items(), key=lambda item: item[1]):
# print("{}: {}".format(key, value))
# import ipdb;ipdb.set_trace()
#print(set(X_img['class_name']))
#return
X_new = {
#'path':[],
'class_name':[],
'index':[],
'data':[],
'class_id':[],
'type':[],
'idx': []
}
# tiramisu
# bread salad
# pork chop
#classes = ['pork chop', 'strawberry pie', 'cheddar cheese', 'greek salad', 'curry chicken']
#classes = ['bell pepper', 'chocolate banana', 'celery root', 'fruit salad', 'pasta sauce']
#classes = ['chocolate banana', 'lemon pepper', 'fruit salad', 'pasta sauce']
classes = ['pizza', 'pork chops', 'cupcake', 'hamburger', 'green beans']
#classes = ['sweet potato', 'pizza', 'chocolate chip', 'crock pot', 'peanut butter']
#classes = ['crock pot', 'peanut butter']
colors = ['crimson', 'darkgreen', 'navy', 'darkorange', 'deeppink']
class_color = {class_name: colors[i] for i, class_name in enumerate(classes)}
Logger()(classes)
nb_points = 80
total_nb_points = len(classes)*nb_points*2
def filter(X, X_new, classes, nb_points):
for c in classes:
idx = 0
for i in range(len(X['index'])):
if c == X['class_name'][i]:
if 'path' in X:
X_new['type'].append('img')
else:
X_new['type'].append('rcp')
X_new['class_name'].append(X['class_name'][i])
X_new['index'].append(X['index'][i])
X_new['data'].append(X['data'][i])
X_new['class_id'].append(X['class_id'][i])
X_new['idx'].append(X['idx'][i])
idx += 1
if idx >= nb_points:
break
if idx < nb_points:
Logger()('Warning: classe {} has {} items'.format(c, idx))
return X_new
X_new = filter(X_img, X_new, classes, nb_points)
X_new = filter(X_rcp, X_new, classes, nb_points)
def shuffle(X):
length = int(len(X['index'])/2)
indexes = list(torch.randperm(length))
X_new = {}
for key in ['class_name', 'index', 'data', 'class_id', 'type', 'idx']:
X_new[key] = []
# shuffle img
for idx in indexes:
X_new[key].append(X[key][idx])
# shuffle rcp
for idx in indexes:
X_new[key].append(X[key][idx+length])
return X_new
X = shuffle(X_new)
X['data'] = transforms.StackTensors()(X['data'])
X['data'] = X['data'].numpy()
print(X['data'].shape)
for perplexity in range(0,100,2):#args.perplexity:
for exaggeration in range(1,10,2):#args.exaggeration:
path_tsne_out = os.path.join(dir_fig, 'ckpt_perplexity,{}_exaggeration,{}.pth'.format(perplexity, exaggeration))
if True or not os.path.isfile(path_tsne_out):
Logger()('Calculating TSNE...')
X_embedded = TSNE(n_components=2,
perplexity=perplexity,
early_exaggeration=exaggeration,
learning_rate=100.0,
n_iter=5000,
n_iter_without_progress=300,
min_grad_norm=1e-07,
metric='euclidean',
init='random',#'random',
verbose=0,
random_state=None,
method='exact',#'barnes_hut',
angle=0.5).fit_transform(X['data'])
torch.save(torch.from_numpy(X_embedded), path_tsne_out)
else:
X_embedded = torch.load(path_tsne_out).numpy()
Logger()('Painting...')
# set min point to 0 and scale
X_embedded = X_embedded - np.min(X_embedded)
X_embedded = X_embedded / np.max(X_embedded)
X['tsne'] = []
for i in range(total_nb_points):
X['tsne'].append(X_embedded[i])
# X_img_per_class = {}
# X_rcp_per_class = {}
# for c in classes:
# X_img_per_class[c] = np.zeros((nb_points,2))
# X_rcp_per_class[c] = np.zeros((nb_points,2))
# idx_img = 0
# idx_rcp = 0
# for i in range(len(X['idx'])):
# if c == X['class_name'][i]:
# if X['type'][i] == 'img':
# X_img_per_class[c][idx_img] = X_embedded[i]
# idx_img += 1
# else:
# X_rcp_per_class[c][idx_rcp] = X_embedded[i]
# idx_rcp += 1
# import ipdb; ipdb.set_trace()
fig = plt.figure(figsize=(20,20))
#ax = plt.subplot(111)
for i in range(total_nb_points):
if X['type'][i] == 'img':
marker = '+'
else:
marker = '.'
class_name = X['class_name'][i]
color = class_color[class_name]
x = X['tsne'][i][0]
y = X['tsne'][i][1]
plt.scatter(x, y, color=color, marker=marker, label=class_name, s=1000)
nb_img_points = int(total_nb_points/2)
for i in range(nb_img_points):
class_name = X['class_name'][i]
color = class_color[class_name]
img_x = X['tsne'][i][0]
img_y = X['tsne'][i][1]
rcp_x = X['tsne'][nb_img_points+i][0]
rcp_y = X['tsne'][nb_img_points+i][1]
plt.plot([img_x, rcp_x], [img_y, rcp_y], '-', color=color, lw=2)
#plt.grid(True)
#plt.xticks([x/10 for x in range(0,11)])
#plt.yticks([x/10 for x in range(0,11)])
#plt.legend()
plt.yticks([])
plt.xticks([])
path_fig = os.path.join(dir_fig, 'fig_perplexity,{}_exaggeration,{}.png'.format(perplexity, exaggeration))
fig.savefig(path_fig)
Logger()('Saved fig to '+path_fig)
plt.show()
Logger()('End')
