def get_image(self, **kwargs): img = Image.open(self.img_path).convert('RGB') # plt.imshow(img) # plt.title("Image en entier") # plt.show() crop = scaled_crop_and_pad(raw_img=img, bbox=self.bbox, scale=self.scale) crop = resize_image(crop, self.width, self.height) # print("/*/*/*/*/******* Image ********/*/*/*/*/*/*/*/ {}".format(self.img_path)) # plt.imshow(crop) # plt.title("Crop") # plt.show() # print("image_loader ..") # exit() crop = np.array(crop, dtype=np.float32) if self.channel is not None: crop -= self.channel[None, None, :] return crop
def _get_image(self, **kwargs): img = Image.open(self.img_path).convert('RGB') img = resize_image(img, self.width, self.height) img = np.array(img, dtype=np.float32) if self.channel is not None: img -= self.channel[None, None, :] return img
def _get_image(self, **kwargs): img = Image.open(self.img_path).convert('RGB') crop = scaled_crop_and_pad(raw_img=img, bbox=self.bbox, scale=self.scale) crop = resize_image(crop, self.width, self.height) crop = np.array(crop, dtype=np.float32) # should it be before/after the padding? if self.channel is not None: img -= self.channel[None, None, :] return crop
def apply(self, games): sources = self.sources tokenizer = self.tokenizer batch = collections.defaultdict(list) for i, game in enumerate(games): batch['raw'].append(game) image = game.image if 'question' in sources: assert len(game.questions) == 1 batch['question'].append(tokenizer.apply(game.questions[0])) if 'answer' in sources: assert len(game.answers) == 1 batch['answer'].append(answer_dict[game.answers[0]]) if 'category' in sources: batch['category'].append(game.object.category_id) if 'spatial' in sources: spat_feat = get_spatial_feat(game.object.bbox, image.width, image.height) batch['spatial'].append(spat_feat) if 'crop' in sources: batch['crop'].append(game.object.get_crop()) if 'image' in sources: batch['image'].append(image.get_image()) if 'mask' in sources: assert "image" in batch[ 'image'], "mask input require the image source" mask = game.object.get_mask() ft_width, ft_height = batch['image'][-1].shape[1],\ batch['image'][-1].shape[2] # Use the image feature size (not the original img size) mask = resize_image(Image.fromarray(mask), height=ft_height, width=ft_width) batch['mask'].append(mask) # pad the questions if 'question' in sources: batch['question'], batch['seq_length'] = padder( batch['question'], padding_symbol=tokenizer.word2i['<padding>']) return batch
def get_image(self, **kwargs): img = Image.open(self.img_path).convert('RGB') img = resize_image(img, self.width, self.height) # print("/*/*/*/*/******* Image ********/*/*/*/*/*/*/*/ {}".format(self.img_path)) # print("img_shape = {} {} ".format(self.width,self.height)) # imgplot = plt.imshow(img) # plt.show() # exit() img = np.array(img, dtype=np.float32) if self.channel is not None: img -= self.channel[None, None, :] return img
def apply(self, games): batch = collections.defaultdict(list) batch_size = len(games) for i, game in enumerate(games): batch["raw"].append(game) # Get referit sentence sentence = self.tokenizer.encode_question(game.sentence) batch['question'].append(sentence) # Get gloves if self.glove is not None: words = self.tokenizer.tokenize_question(game.sentence) glove_vectors = self.glove.get_embeddings(words) batch['glove'].append(glove_vectors) if 'answer' in self.sources: answer = [0, 0] answer[int(game.correct_object)] = 1 batch['answer'].append(answer) if "image" in self.sources: img = game.image.get_image() if "image" not in batch: # initialize an empty array for better memory consumption batch["image"] = np.zeros((batch_size, ) + img.shape) batch["image"][i] = img if "crop" in self.sources: crop = game.object.get_crop() if "crop" not in batch: # initialize an empty array for better memory consumption batch["crop"] = np.zeros((batch_size, ) + crop.shape) batch["crop"][i] = crop if 'image_mask' in self.sources: assert "image" in batch, "mask input require the image source" mask = game.object.get_mask() ft_width, ft_height = batch['image'][-1].shape[1], \ batch['image'][-1].shape[0] # Use the image feature size (not the original img size) mask = resize_image(PImage.fromarray(mask), height=ft_height, width=ft_width) batch['image_mask'].append(np.array(mask)) if 'crop_mask' in self.sources: assert "crop" in batch, "mask input require the crop source" cmask = game.object.get_mask() ft_width, ft_height = batch['crop'][-1].shape[1], \ batch['crop'][-1].shape[0] # Use the crop feature size (not the original img size) cmask = scaled_crop_and_pad(raw_img=PImage.fromarray(cmask), bbox=game.object.bbox, scale=game.object.crop_scale) cmask = resize_image(cmask, height=ft_height, width=ft_width) batch['crop_mask'].append(np.array(cmask)) if 'category' in self.sources: batch['category'].append(game.object.category_id) if 'spatial' in self.sources: spat_feat = get_spatial_feat(game.object.bbox, game.image.width, game.image.height) batch['spatial'].append(spat_feat) # Pad referit sentence batch['question'], batch['seq_length'] = padder( batch['question'], padding_symbol=self.tokenizer.padding_token) if self.glove is not None: batch['glove'], _ = padder_3d(batch['glove']) return batch
def apply(self, games): sources = self.sources batch = collections.defaultdict(list) batch_size = len(games) assert batch_size > 0 for i, game in enumerate(games): batch['raw'].append(game) image = game.image if 'question' in sources: question = self.tokenizer_question.apply(game.questions[0], use_dict_ques=False) # print("+++++ words_question = {} ".format(question)) sp_zeros = np.zeros((14)) sp_zeros[0:len(question)] = question batch["question"].append(sp_zeros) batch["seq_length_question"].append(len(question)) if 'embedding_vector_ques' in sources: assert len(game.questions) == 1 # Add glove vectors (NB even <unk> may have a specific glove) # print("oracle_batchifier | question = {}".format(game.questions[0])) words = self.tokenizer_question.apply(game.questions[0], tokent_int=False) if "question_pos" in sources: # print("/////////// question_pos") embedding_vectors, embedding_pos = get_embeddings( words, pos=self.config["model"]["question"]["pos"], lemme=self.config["model"]["question"]["lemme"], model_wordd=self.model_wordd, model_worddl=self.model_worddl, model_word=self.model_word, model_wordl=self.model_wordl, model_posd=self.model_posd, model_pos=self.model_pos ) # slow (copy gloves in process) # print("..... question_pos............. embedding_vectors",len(embedding_vectors[0])) batch['embedding_vector_ques'].append(embedding_vectors) batch['embedding_vector_ques_pos'].append(embedding_pos) batch['question_pos'].append(question) else: embedding_vectors = self.embedding.get_embedding(words) # print("embedding = {}".format(np.asarray(embedding_vectors).shape )) # exit() # if "embedding_vector_ques" not in batch: # batch['embedding_vector_ques'] = np.zeros((batch_size,7,100)) batch['embedding_vector_ques'].append(embedding_vectors) if 'description' in sources: description = self.tokenizer_question.apply( game.image.description, use_dict_ques=False) # print("+++++ words_question = {} ".format(question)) batch["description"].append(description) if 'ques_hist_H0' in sources: assert len(game.questions) == 1 # description = self.tokenizer_description.apply(game.image.description) # batch['description'].append(description) for j in range(6): question_answer = game.all_last_question[0] words = [] if len(question_answer) > 1: word = self.tokenizer_question.apply( game.all_last_question[0][1][0]) words = word else: word = self.tokenizer_question.apply( game.all_last_question[0][0]) words = word sp_zeros = np.zeros((14)) # print("words = {} ".format(words)) sp_zeros[0:len(words)] = words # print("sp_zeros = {} ".format(sp_zeros)) batch['ques_hist_H{}'.format(j)].append(sp_zeros) batch['seq_length_question_history_H{}'.format(j)].append( len(words)) # print('embedding_vector_des'in sources) if 'embedding_vector_des' in sources: description = self.tokenizer_description.apply( game.image.description, tokent_int=False) #print("*************** Description =",description) # batch['description'].append(description) if "des_pos" in sources: embedding_vectors, embedding_pos = get_embeddings( description, pos=self.config["model"]["question"]["pos"], lemme=self.config["model"]["question"]["lemme"], model_wordd=self.model_wordd, model_worddl=self.model_worddl, model_word=self.model_word, model_wordl=self.model_wordl, model_posd=self.model_posd, model_pos=self.model_pos ) # slow (copy gloves in process) batch['embedding_vector_des'].append(embedding_vectors) batch['embedding_vector_des_pos'].append(embedding_pos) # batch['des_pos'].append(question) else: if self.config["model"]["fasttext"]: #print("++++++----- ++++++++ Dans fasttext ") embedding_vectors, _ = get_embeddings( description, pos=self.config["model"]["question"]["pos"], lemme=self.config["model"]["question"]["lemme"], model_wordd=self.model_wordd, model_worddl=self.model_worddl, model_word=self.model_word, model_wordl=self.model_wordl, model_posd=self.model_posd, model_pos=self.model_pos ) # slow (copy gloves in process) elif self.config["model"]["glove"]: #print("++++++----- ++++++++ Dans glove ") embedding_vectors = self.glove.get_embeddings( description) # print("------ ELSE".format(embedding_vectors)) # exit() batch['embedding_vector_des'].append(embedding_vectors) if 'answer' in sources: if "answer" not in batch: batch["answer"] = np.zeros((batch_size, 3)) # print("game.amswer = {}".format(game.answers)) # exit() assert len(game.answers) == 1 batch['answer'][i] = answer_dict[game.answers[0]] #print(" Correct Answer = ",game.answers[0]) if 'category' in sources: use_embedding_cat = self.config["model"]["category"][ "use_embedding"] if "category" not in batch: if use_embedding_cat: batch['category'] = np.zeros((batch_size, 100)) else: batch['category'] = np.zeros((batch_size)) if use_embedding_cat: embc = np.asarray( self.embedding.get_embedding([game.object.category])) # embc = self.tokenizer_question.apply(game.object.category,use_dict_ques=False) category_input = embc.reshape((100)) else: category_input = game.object.category_id # print("category = {} ".format(category_input)) batch['category'][i] = category_input if 'allcategory' in sources: allcategory = [] allcategory_hot = np.zeros(shape=(90), dtype=int) # print("Oracle_batchifier | Allcategory -------------------------------") for obj in game.objects: allcategory.append(obj.category_id - 1) allcategory_hot[allcategory] = 1 batch['allcategory'].append(allcategory_hot) if 'spatial' in sources: if 'spatial' not in batch: batch['spatial'] = np.zeros((batch_size, 8), dtype=float) spat_feat = get_spatial_feat(game.object.bbox, image.width, image.height) batch['spatial'][i] = spat_feat if 'crop' in sources: batch['crop'].append(game.object.get_crop()) batch['image_id'].append(image.get_idimage()) # batch['crop_id'].append(game.object_id) # print("crop_id=",game.object.get_crop().shape) # exit() if 'image' in sources: features_image = image.get_image() batch['image'].append(features_image) batch['image_id'].append(image.get_idimage()) if 'mask' in sources: assert "image" in batch[ 'image'], "mask input require the image source" mask = game.object.get_mask() ft_width, ft_height = batch['image'][-1].shape[1],\ batch['image'][-1].shape[2] # Use the image feature size (not the original img size) mask = resize_image(Image.fromarray(mask), height=ft_height, width=ft_width) batch['mask'].append(mask) # padding = self.embedding.get_embeddings(["<padding>"])[0] # print("padding | = {}".format(padding)) # pad the questions # if "question" in sources: # batch['question'] , batch['seq_length_question'] = padder(batch['question'],max_seq_length=14) if "question_pos" in sources: batch['question_pos'], batch['seq_length_ques_pos'] = padder( batch['question_pos'], padding_symbol=self.tokenizer_question.padding_token) if "description" in sources: batch['description'], batch['seq_length_description'] = padder( batch['description']) # batch['embedding_vector_pos'], _ = padder_3d(batch['embedding_vector_pos']) if 'embedding_vector_ques' in sources: batch['embedding_vector_ques'], s = padder_3d( batch['embedding_vector_ques'], max_seq_length=12) if 'embedding_vector_ques_hist' in sources: # print("Shape=",np.asarray(batch['embedding_vector_ques_hist'] ).shape) batch_hist, size_sentences, max_seq = padder_4d( batch['embedding_vector_ques_hist'], max_seq_length=14) batch_hist = np.asarray(batch_hist) size_sentences = np.asarray(size_sentences) batch['embedding_vector_ques_hist'] = batch_hist for i in range(6): batch['ques_hist_H{}'.format(i)] = batch_hist[:, i, :] batch['seq_length_question_history_H{}'.format( i)] = size_sentences[:, i] #print("Len=",len(batch['seq_length_question'])) if 'embedding_vector_ques_pos' in sources: batch['embedding_vector_ques_pos'], _ = padder_3d( batch['embedding_vector_ques_pos']) if 'embedding_vector_des' in sources: batch['embedding_vector_des'], batch[ 'seq_length_description'] = padder_3d( batch['embedding_vector_des']) if 'embedding_vector_des_pos' in sources: batch['embedding_vector_des_pos'], _ = padder_3d( batch['embedding_vector_des_pos']) # if 'description' in sources: # # complete par padding en prenons la taille maximal # batch['description'], batch['seq_length_description'] = padder_3d(batch['description']) # print(" Bath = {} ".format(batch.keys())) # exit() # print("finish oracle_bachifier .... time=",total) # print("TotalBatch=",total) #print("TotalBatch=",total) return batch
def apply(self, games, skip_targets=False): batch = collections.defaultdict(list) batch["raw"] = games batch_size = len(games) for i, game in enumerate(games): if 'question' in self.sources: assert len(game.questions) == 1 batch['question'].append( self.tokenizer.encode(game.questions[0])) # questions = [] # for q, a in zip(game.questions[:-1], game.answers[:-1]): # questions.append(self.tokenizer.encode(q, add_stop_token=True)) # questions.append(self.tokenizer.encode(a, is_answer=True)) # questions.append(self.tokenizer.encode(game.questions[-1], add_stop_token=True)) # batch['question'].append(list(chain.from_iterable(questions))) if 'glove' in self.sources: words = self.tokenizer.decode(batch['question'][i]) glove_vectors = self.glove.get_embeddings(words) batch['glove'].append(glove_vectors) if 'answer' in self.sources and not skip_targets: batch['answer'].append( self.tokenizer.encode_oracle_answer(game.answers[-1], sparse=False)) if 'category' in self.sources: batch['category'].append(game.object.category_id) if 'spatial' in self.sources: spat_feat = get_spatial_feat(game.object.bbox, game.image.width, game.image.height) batch['spatial'].append(spat_feat) if 'crop' in self.sources: crop = game.object.get_crop() if "crop" not in batch: # initialize an empty array for better memory consumption batch["crop"] = np.zeros((batch_size, ) + crop.shape) batch["crop"][i] = crop if 'image' in self.sources: img = game.image.get_image() if "image" not in batch: # initialize an empty array for better memory consumption batch["image"] = np.zeros((batch_size, ) + img.shape) batch["image"][i] = img if 'image_mask' in self.sources: assert "image" in batch, "mask input require the image source" mask = game.object.get_mask() ft_width, ft_height = img.shape[1], img.shape[0] # ft_width, ft_height = batch['image'][-1].shape[1], \ # batch['image'][-1].shape[0] # Use the image feature size (not the original img size) mask = resize_image(Image.fromarray(mask), height=ft_height, width=ft_width) batch['image_mask'].append(np.array(mask)) if 'crop_mask' in self.sources: assert "crop" in batch, "mask input require the crop source" cmask = game.object.get_mask() ft_width, ft_height = batch['crop'][-1].shape[1], \ batch['crop'][-1].shape[0] # Use the crop feature size (not the original img size) cmask = scaled_crop_and_pad(raw_img=Image.fromarray(cmask), bbox=game.object.bbox, scale=game.object.crop_scale) cmask = resize_image(cmask, height=ft_height, width=ft_width) batch['crop_mask'].append(np.array(cmask)) # Pad the questions if 'question' in self.sources: batch['question'], batch['seq_length'], _ = padder( batch['question'], padding_symbol=self.tokenizer.padding_token) if 'glove' in self.sources: # (?, 16, 300) (batch, max num word, glove emb size) batch['glove'], _ = padder_3d(batch['glove']) return batch