def load_birds(): birds = [] feats = np.zeros(4096) feats_sq = np.zeros(4096) word_counter = defaultdict(lambda: 0) with open("data/birds/cub_0917_5cap.tsv") as caption_f: for line in caption_f: parts = line.strip().split("\t") caption = parts[-1] caption = (caption.lower() .replace(".", "") .replace(",", " , ")) words = ["<s>"] + caption.split() + ["</s>"] for word in words: word_counter[word] += 1 for word, count in word_counter.items(): if count >= MIN_WORD_COUNT: WORD_INDEX.index(word) with open("data/birds/cub_0917_5cap.tsv") as caption_f: caption_f.readline() for line in caption_f: parts = line.strip().split("\t") caption = parts[-1] image_path = parts[-2] image_id = image_path.split("/")[-1] caption = (caption.lower() .replace(".", "") .replace(",", " , ")) words = ["<s>"] + caption.split() + ["</s>"] word_ids = [WORD_INDEX[w] for w in words] with np.load("data/birds/embeddings/%s.npz" % image_id) as feature_f: features = feature_f[feature_f.keys()[0]] birds.append(Bird(image_id, word_ids, features)) feats += features feats_sq += features ** 2 mean_feats = feats / len(birds) mean_feats_sq = feats_sq / len(birds) var_feats = mean_feats_sq - (mean_feats ** 2) std_feats = np.sqrt(var_feats) std_feats += 0.0001 for bird in birds: bird.features[...] -= mean_feats bird.features[...] /= std_feats train_birds = birds[:-1100] val_birds = birds[-1100:-100] test_birds = birds[-100:] return train_birds, val_birds, test_birds
def load_birds(): birds = [] feats = np.zeros(4096) feats_sq = np.zeros(4096) word_counter = defaultdict(lambda: 0) with open("data/birds/cub_0917_5cap.tsv") as caption_f: for line in caption_f: parts = line.strip().split("\t") caption = parts[-1] caption = (caption.lower().replace(".", "").replace(",", " , ")) words = ["<s>"] + caption.split() + ["</s>"] for word in words: word_counter[word] += 1 for word, count in word_counter.items(): if count >= MIN_WORD_COUNT: WORD_INDEX.index(word) with open("data/birds/cub_0917_5cap.tsv") as caption_f: caption_f.readline() for line in caption_f: parts = line.strip().split("\t") caption = parts[-1] image_path = parts[-2] image_id = image_path.split("/")[-1] caption = (caption.lower().replace(".", "").replace(",", " , ")) words = ["<s>"] + caption.split() + ["</s>"] word_ids = [WORD_INDEX[w] for w in words] with np.load("data/birds/embeddings/%s.npz" % image_id) as feature_f: features = feature_f[feature_f.keys()[0]] birds.append(Bird(image_id, word_ids, features)) feats += features feats_sq += features**2 mean_feats = feats / len(birds) mean_feats_sq = feats_sq / len(birds) var_feats = mean_feats_sq - (mean_feats**2) std_feats = np.sqrt(var_feats) std_feats += 0.0001 for bird in birds: bird.features[...] -= mean_feats bird.features[...] /= std_feats train_birds = birds[:-1100] val_birds = birds[-1100:-100] test_birds = birds[-100:] return train_birds, val_birds, test_birds
def load_scenes(scene_props): scenes = [] word_counter = defaultdict(lambda: 0) for sent_file_id in range(1, 3): with open("data/abstract/SimpleSentences/SimpleSentences%d_10020.txt" % sent_file_id) as sent_f: for sent_line in sent_f: sent_parts = sent_line.strip().split("\t") sent = sent_parts[2] sent = sent.replace('"', ' " ') sent = sent.replace("'", " ' ") sent = re.sub(r"[.?!]", "", sent) words = sent.lower().split() words = ["<s>"] + words + ["</s>"] for word in words: word_counter[word] += 1 for word, count in word_counter.items(): if count >= MIN_WORD_COUNT: WORD_INDEX.index(word) for sent_file_id in range(1, 3): with open("data/abstract/SimpleSentences/SimpleSentences%d_10020.txt" % sent_file_id) as sent_f: for sent_line in sent_f: sent_parts = sent_line.strip().split("\t") scene_id = int(sent_parts[0]) props = scene_props[scene_id] sent_id = int(sent_parts[1]) image_id = scene_id / 10 image_subid = scene_id % 10 image_strid = "%d_%d" % (image_id, image_subid) sent = sent_parts[2] sent = sent.replace('"', "") sent = re.sub(r"[.?!']", "", sent) words = sent.lower().split() words = ["<s>"] + words + ["</s>"] word_ids = [WORD_INDEX[w] or 0 for w in words] with np.load("data/abstract/EmbeddedScenes/Scene%s.png.npz" % image_strid) as feature_f: features = feature_f[feature_f.keys()[0]] scenes.append(Scene(image_strid, props, word_ids, features)) return scenes
def load_scenes(scene_props): scenes = [] word_counter = defaultdict(lambda: 0) for sent_file_id in range(1, 3): with open("data/abstract/SimpleSentences/SimpleSentences%d_10020.txt" % sent_file_id) as sent_f: for sent_line in sent_f: sent_parts = sent_line.strip().split("\t") sent = sent_parts[2] sent = sent.replace('"', ' " ') sent = sent.replace("'", " ' ") sent = re.sub(r"[.?!]", "", sent) words = sent.lower().split() words = ["<s>"] + words + ["</s>"] for word in words: word_counter[word] += 1 for word, count in word_counter.items(): if count >= MIN_WORD_COUNT: WORD_INDEX.index(word) for sent_file_id in range(1, 3): with open("data/abstract/SimpleSentences/SimpleSentences%d_10020.txt" % sent_file_id) as sent_f: for sent_line in sent_f: sent_parts = sent_line.strip().split("\t") scene_id = int(sent_parts[0]) props = scene_props[scene_id] sent_id = int(sent_parts[1]) image_id = scene_id / 10 image_subid = scene_id % 10 image_strid = "%d_%d" % (image_id, image_subid) sent = sent_parts[2] sent = sent.replace('"', "") sent = re.sub(r"[.?!']", "", sent) words = sent.lower().split() words = ["<s>"] + words + ["</s>"] word_ids = [WORD_INDEX[w] or 0 for w in words] with np.load("data/abstract/EmbeddedScenes/Scene%s.png.npz" % image_strid) as feature_f: features = feature_f[feature_f.keys()[0]] scenes.append(Scene(image_strid, props, word_ids, features)) return scenes