def get_image(self, **kwargs):
img = Image.open(self.img_path).convert('RGB')
# plt.imshow(img)
# plt.title("Image en entier")
# plt.show()
crop = scaled_crop_and_pad(raw_img=img,
bbox=self.bbox,
scale=self.scale)
crop = resize_image(crop, self.width, self.height)
# print("/*/*/*/*/******* Image ********/*/*/*/*/*/*/*/ {}".format(self.img_path))
# plt.imshow(crop)
# plt.title("Crop")
# plt.show()
# print("image_loader ..")
# exit()
crop = np.array(crop, dtype=np.float32)
if self.channel is not None:
crop -= self.channel[None, None, :]
return crop
def _get_image(self, **kwargs):
img = Image.open(self.img_path).convert('RGB')
img = resize_image(img, self.width, self.height)
img = np.array(img, dtype=np.float32)
if self.channel is not None:
img -= self.channel[None, None, :]
return img
def _get_image(self, **kwargs):
img = Image.open(self.img_path).convert('RGB')
crop = scaled_crop_and_pad(raw_img=img, bbox=self.bbox, scale=self.scale)
crop = resize_image(crop, self.width, self.height)
crop = np.array(crop, dtype=np.float32)
# should it be before/after the padding?
if self.channel is not None:
img -= self.channel[None, None, :]
return crop
def apply(self, games):
sources = self.sources
tokenizer = self.tokenizer
batch = collections.defaultdict(list)
for i, game in enumerate(games):
batch['raw'].append(game)
image = game.image
if 'question' in sources:
assert len(game.questions) == 1
batch['question'].append(tokenizer.apply(game.questions[0]))
if 'answer' in sources:
assert len(game.answers) == 1
batch['answer'].append(answer_dict[game.answers[0]])
if 'category' in sources:
batch['category'].append(game.object.category_id)
if 'spatial' in sources:
spat_feat = get_spatial_feat(game.object.bbox, image.width,
image.height)
batch['spatial'].append(spat_feat)
if 'crop' in sources:
batch['crop'].append(game.object.get_crop())
if 'image' in sources:
batch['image'].append(image.get_image())
if 'mask' in sources:
assert "image" in batch[
'image'], "mask input require the image source"
mask = game.object.get_mask()
ft_width, ft_height = batch['image'][-1].shape[1],\
batch['image'][-1].shape[2] # Use the image feature size (not the original img size)
mask = resize_image(Image.fromarray(mask),
height=ft_height,
width=ft_width)
batch['mask'].append(mask)
# pad the questions
if 'question' in sources:
batch['question'], batch['seq_length'] = padder(
batch['question'],
padding_symbol=tokenizer.word2i['<padding>'])
return batch
def get_image(self, **kwargs):
img = Image.open(self.img_path).convert('RGB')
img = resize_image(img, self.width, self.height)
# print("/*/*/*/*/******* Image ********/*/*/*/*/*/*/*/ {}".format(self.img_path))
# print("img_shape = {} {} ".format(self.width,self.height))
# imgplot = plt.imshow(img)
# plt.show()
# exit()
img = np.array(img, dtype=np.float32)
if self.channel is not None:
img -= self.channel[None, None, :]
return img
def apply(self, games):
batch = collections.defaultdict(list)
batch_size = len(games)
for i, game in enumerate(games):
batch["raw"].append(game)
# Get referit sentence
sentence = self.tokenizer.encode_question(game.sentence)
batch['question'].append(sentence)
# Get gloves
if self.glove is not None:
words = self.tokenizer.tokenize_question(game.sentence)
glove_vectors = self.glove.get_embeddings(words)
batch['glove'].append(glove_vectors)
if 'answer' in self.sources:
answer = [0, 0]
answer[int(game.correct_object)] = 1
batch['answer'].append(answer)
if "image" in self.sources:
img = game.image.get_image()
if "image" not in batch: # initialize an empty array for better memory consumption
batch["image"] = np.zeros((batch_size, ) + img.shape)
batch["image"][i] = img
if "crop" in self.sources:
crop = game.object.get_crop()
if "crop" not in batch: # initialize an empty array for better memory consumption
batch["crop"] = np.zeros((batch_size, ) + crop.shape)
batch["crop"][i] = crop
if 'image_mask' in self.sources:
assert "image" in batch, "mask input require the image source"
mask = game.object.get_mask()
ft_width, ft_height = batch['image'][-1].shape[1], \
batch['image'][-1].shape[0] # Use the image feature size (not the original img size)
mask = resize_image(PImage.fromarray(mask),
height=ft_height,
width=ft_width)
batch['image_mask'].append(np.array(mask))
if 'crop_mask' in self.sources:
assert "crop" in batch, "mask input require the crop source"
cmask = game.object.get_mask()
ft_width, ft_height = batch['crop'][-1].shape[1], \
batch['crop'][-1].shape[0] # Use the crop feature size (not the original img size)
cmask = scaled_crop_and_pad(raw_img=PImage.fromarray(cmask),
bbox=game.object.bbox,
scale=game.object.crop_scale)
cmask = resize_image(cmask, height=ft_height, width=ft_width)
batch['crop_mask'].append(np.array(cmask))
if 'category' in self.sources:
batch['category'].append(game.object.category_id)
if 'spatial' in self.sources:
spat_feat = get_spatial_feat(game.object.bbox,
game.image.width,
game.image.height)
batch['spatial'].append(spat_feat)
# Pad referit sentence
batch['question'], batch['seq_length'] = padder(
batch['question'], padding_symbol=self.tokenizer.padding_token)
if self.glove is not None:
batch['glove'], _ = padder_3d(batch['glove'])
return batch
def apply(self, games):
sources = self.sources
batch = collections.defaultdict(list)
batch_size = len(games)
assert batch_size > 0
for i, game in enumerate(games):
batch['raw'].append(game)
image = game.image
if 'question' in sources:
question = self.tokenizer_question.apply(game.questions[0],
use_dict_ques=False)
# print("+++++ words_question = {} ".format(question))
sp_zeros = np.zeros((14))
sp_zeros[0:len(question)] = question
batch["question"].append(sp_zeros)
batch["seq_length_question"].append(len(question))
if 'embedding_vector_ques' in sources:
assert len(game.questions) == 1
# Add glove vectors (NB even <unk> may have a specific glove)
# print("oracle_batchifier | question = {}".format(game.questions[0]))
words = self.tokenizer_question.apply(game.questions[0],
tokent_int=False)
if "question_pos" in sources:
# print("/////////// question_pos")
embedding_vectors, embedding_pos = get_embeddings(
words,
pos=self.config["model"]["question"]["pos"],
lemme=self.config["model"]["question"]["lemme"],
model_wordd=self.model_wordd,
model_worddl=self.model_worddl,
model_word=self.model_word,
model_wordl=self.model_wordl,
model_posd=self.model_posd,
model_pos=self.model_pos
) # slow (copy gloves in process)
# print("..... question_pos............. embedding_vectors",len(embedding_vectors[0]))
batch['embedding_vector_ques'].append(embedding_vectors)
batch['embedding_vector_ques_pos'].append(embedding_pos)
batch['question_pos'].append(question)
else:
embedding_vectors = self.embedding.get_embedding(words)
# print("embedding = {}".format(np.asarray(embedding_vectors).shape ))
# exit()
# if "embedding_vector_ques" not in batch:
# batch['embedding_vector_ques'] = np.zeros((batch_size,7,100))
batch['embedding_vector_ques'].append(embedding_vectors)
if 'description' in sources:
description = self.tokenizer_question.apply(
game.image.description, use_dict_ques=False)
# print("+++++ words_question = {} ".format(question))
batch["description"].append(description)
if 'ques_hist_H0' in sources:
assert len(game.questions) == 1
# description = self.tokenizer_description.apply(game.image.description)
# batch['description'].append(description)
for j in range(6):
question_answer = game.all_last_question[0]
words = []
if len(question_answer) > 1:
word = self.tokenizer_question.apply(
game.all_last_question[0][1][0])
words = word
else:
word = self.tokenizer_question.apply(
game.all_last_question[0][0])
words = word
sp_zeros = np.zeros((14))
# print("words = {} ".format(words))
sp_zeros[0:len(words)] = words
# print("sp_zeros = {} ".format(sp_zeros))
batch['ques_hist_H{}'.format(j)].append(sp_zeros)
batch['seq_length_question_history_H{}'.format(j)].append(
len(words))
# print('embedding_vector_des'in sources)
if 'embedding_vector_des' in sources:
description = self.tokenizer_description.apply(
game.image.description, tokent_int=False)
#print("*************** Description =",description)
# batch['description'].append(description)
if "des_pos" in sources:
embedding_vectors, embedding_pos = get_embeddings(
description,
pos=self.config["model"]["question"]["pos"],
lemme=self.config["model"]["question"]["lemme"],
model_wordd=self.model_wordd,
model_worddl=self.model_worddl,
model_word=self.model_word,
model_wordl=self.model_wordl,
model_posd=self.model_posd,
model_pos=self.model_pos
) # slow (copy gloves in process)
batch['embedding_vector_des'].append(embedding_vectors)
batch['embedding_vector_des_pos'].append(embedding_pos)
# batch['des_pos'].append(question)
else:
if self.config["model"]["fasttext"]:
#print("++++++----- ++++++++ Dans fasttext ")
embedding_vectors, _ = get_embeddings(
description,
pos=self.config["model"]["question"]["pos"],
lemme=self.config["model"]["question"]["lemme"],
model_wordd=self.model_wordd,
model_worddl=self.model_worddl,
model_word=self.model_word,
model_wordl=self.model_wordl,
model_posd=self.model_posd,
model_pos=self.model_pos
) # slow (copy gloves in process)
elif self.config["model"]["glove"]:
#print("++++++----- ++++++++ Dans glove ")
embedding_vectors = self.glove.get_embeddings(
description)
# print("------ ELSE".format(embedding_vectors))
# exit()
batch['embedding_vector_des'].append(embedding_vectors)
if 'answer' in sources:
if "answer" not in batch:
batch["answer"] = np.zeros((batch_size, 3))
# print("game.amswer = {}".format(game.answers))
# exit()
assert len(game.answers) == 1
batch['answer'][i] = answer_dict[game.answers[0]]
#print(" Correct Answer = ",game.answers[0])
if 'category' in sources:
use_embedding_cat = self.config["model"]["category"][
"use_embedding"]
if "category" not in batch:
if use_embedding_cat:
batch['category'] = np.zeros((batch_size, 100))
else:
batch['category'] = np.zeros((batch_size))
if use_embedding_cat:
embc = np.asarray(
self.embedding.get_embedding([game.object.category]))
# embc = self.tokenizer_question.apply(game.object.category,use_dict_ques=False)
category_input = embc.reshape((100))
else:
category_input = game.object.category_id
# print("category = {} ".format(category_input))
batch['category'][i] = category_input
if 'allcategory' in sources:
allcategory = []
allcategory_hot = np.zeros(shape=(90), dtype=int)
# print("Oracle_batchifier | Allcategory -------------------------------")
for obj in game.objects:
allcategory.append(obj.category_id - 1)
allcategory_hot[allcategory] = 1
batch['allcategory'].append(allcategory_hot)
if 'spatial' in sources:
if 'spatial' not in batch:
batch['spatial'] = np.zeros((batch_size, 8), dtype=float)
spat_feat = get_spatial_feat(game.object.bbox, image.width,
image.height)
batch['spatial'][i] = spat_feat
if 'crop' in sources:
batch['crop'].append(game.object.get_crop())
batch['image_id'].append(image.get_idimage())
# batch['crop_id'].append(game.object_id)
# print("crop_id=",game.object.get_crop().shape)
# exit()
if 'image' in sources:
features_image = image.get_image()
batch['image'].append(features_image)
batch['image_id'].append(image.get_idimage())
if 'mask' in sources:
assert "image" in batch[
'image'], "mask input require the image source"
mask = game.object.get_mask()
ft_width, ft_height = batch['image'][-1].shape[1],\
batch['image'][-1].shape[2] # Use the image feature size (not the original img size)
mask = resize_image(Image.fromarray(mask),
height=ft_height,
width=ft_width)
batch['mask'].append(mask)
# padding = self.embedding.get_embeddings(["<padding>"])[0]
# print("padding | = {}".format(padding))
# pad the questions
# if "question" in sources:
# batch['question'] , batch['seq_length_question'] = padder(batch['question'],max_seq_length=14)
if "question_pos" in sources:
batch['question_pos'], batch['seq_length_ques_pos'] = padder(
batch['question_pos'],
padding_symbol=self.tokenizer_question.padding_token)
if "description" in sources:
batch['description'], batch['seq_length_description'] = padder(
batch['description'])
# batch['embedding_vector_pos'], _ = padder_3d(batch['embedding_vector_pos'])
if 'embedding_vector_ques' in sources:
batch['embedding_vector_ques'], s = padder_3d(
batch['embedding_vector_ques'], max_seq_length=12)
if 'embedding_vector_ques_hist' in sources:
# print("Shape=",np.asarray(batch['embedding_vector_ques_hist'] ).shape)
batch_hist, size_sentences, max_seq = padder_4d(
batch['embedding_vector_ques_hist'], max_seq_length=14)
batch_hist = np.asarray(batch_hist)
size_sentences = np.asarray(size_sentences)
batch['embedding_vector_ques_hist'] = batch_hist
for i in range(6):
batch['ques_hist_H{}'.format(i)] = batch_hist[:, i, :]
batch['seq_length_question_history_H{}'.format(
i)] = size_sentences[:, i]
#print("Len=",len(batch['seq_length_question']))
if 'embedding_vector_ques_pos' in sources:
batch['embedding_vector_ques_pos'], _ = padder_3d(
batch['embedding_vector_ques_pos'])
if 'embedding_vector_des' in sources:
batch['embedding_vector_des'], batch[
'seq_length_description'] = padder_3d(
batch['embedding_vector_des'])
if 'embedding_vector_des_pos' in sources:
batch['embedding_vector_des_pos'], _ = padder_3d(
batch['embedding_vector_des_pos'])
# if 'description' in sources:
# # complete par padding en prenons la taille maximal
# batch['description'], batch['seq_length_description'] = padder_3d(batch['description'])
# print(" Bath = {} ".format(batch.keys()))
# exit()
# print("finish oracle_bachifier .... time=",total)
# print("TotalBatch=",total)
#print("TotalBatch=",total)
return batch
def apply(self, games, skip_targets=False):
batch = collections.defaultdict(list)
batch["raw"] = games
batch_size = len(games)
for i, game in enumerate(games):
if 'question' in self.sources:
assert len(game.questions) == 1
batch['question'].append(
self.tokenizer.encode(game.questions[0]))
# questions = []
# for q, a in zip(game.questions[:-1], game.answers[:-1]):
# questions.append(self.tokenizer.encode(q, add_stop_token=True))
# questions.append(self.tokenizer.encode(a, is_answer=True))
# questions.append(self.tokenizer.encode(game.questions[-1], add_stop_token=True))
# batch['question'].append(list(chain.from_iterable(questions)))
if 'glove' in self.sources:
words = self.tokenizer.decode(batch['question'][i])
glove_vectors = self.glove.get_embeddings(words)
batch['glove'].append(glove_vectors)
if 'answer' in self.sources and not skip_targets:
batch['answer'].append(
self.tokenizer.encode_oracle_answer(game.answers[-1],
sparse=False))
if 'category' in self.sources:
batch['category'].append(game.object.category_id)
if 'spatial' in self.sources:
spat_feat = get_spatial_feat(game.object.bbox,
game.image.width,
game.image.height)
batch['spatial'].append(spat_feat)
if 'crop' in self.sources:
crop = game.object.get_crop()
if "crop" not in batch: # initialize an empty array for better memory consumption
batch["crop"] = np.zeros((batch_size, ) + crop.shape)
batch["crop"][i] = crop
if 'image' in self.sources:
img = game.image.get_image()
if "image" not in batch: # initialize an empty array for better memory consumption
batch["image"] = np.zeros((batch_size, ) + img.shape)
batch["image"][i] = img
if 'image_mask' in self.sources:
assert "image" in batch, "mask input require the image source"
mask = game.object.get_mask()
ft_width, ft_height = img.shape[1], img.shape[0]
# ft_width, ft_height = batch['image'][-1].shape[1], \
# batch['image'][-1].shape[0] # Use the image feature size (not the original img size)
mask = resize_image(Image.fromarray(mask),
height=ft_height,
width=ft_width)
batch['image_mask'].append(np.array(mask))
if 'crop_mask' in self.sources:
assert "crop" in batch, "mask input require the crop source"
cmask = game.object.get_mask()
ft_width, ft_height = batch['crop'][-1].shape[1], \
batch['crop'][-1].shape[0] # Use the crop feature size (not the original img size)
cmask = scaled_crop_and_pad(raw_img=Image.fromarray(cmask),
bbox=game.object.bbox,
scale=game.object.crop_scale)
cmask = resize_image(cmask, height=ft_height, width=ft_width)
batch['crop_mask'].append(np.array(cmask))
# Pad the questions
if 'question' in self.sources:
batch['question'], batch['seq_length'], _ = padder(
batch['question'], padding_symbol=self.tokenizer.padding_token)
if 'glove' in self.sources:
# (?, 16, 300) (batch, max num word, glove emb size)
batch['glove'], _ = padder_3d(batch['glove'])
return batch
