def load_birds():
birds = []
feats = np.zeros(4096)
feats_sq = np.zeros(4096)
word_counter = defaultdict(lambda: 0)
with open("data/birds/cub_0917_5cap.tsv") as caption_f:
for line in caption_f:
parts = line.strip().split("\t")
caption = parts[-1]
caption = (caption.lower()
.replace(".", "")
.replace(",", " , "))
words = ["<s>"] + caption.split() + ["</s>"]
for word in words:
word_counter[word] += 1
for word, count in word_counter.items():
if count >= MIN_WORD_COUNT:
WORD_INDEX.index(word)
with open("data/birds/cub_0917_5cap.tsv") as caption_f:
caption_f.readline()
for line in caption_f:
parts = line.strip().split("\t")
caption = parts[-1]
image_path = parts[-2]
image_id = image_path.split("/")[-1]
caption = (caption.lower()
.replace(".", "")
.replace(",", " , "))
words = ["<s>"] + caption.split() + ["</s>"]
word_ids = [WORD_INDEX[w] for w in words]
with np.load("data/birds/embeddings/%s.npz" % image_id) as feature_f:
features = feature_f[feature_f.keys()[0]]
birds.append(Bird(image_id, word_ids, features))
feats += features
feats_sq += features ** 2
mean_feats = feats / len(birds)
mean_feats_sq = feats_sq / len(birds)
var_feats = mean_feats_sq - (mean_feats ** 2)
std_feats = np.sqrt(var_feats)
std_feats += 0.0001
for bird in birds:
bird.features[...] -= mean_feats
bird.features[...] /= std_feats
train_birds = birds[:-1100]
val_birds = birds[-1100:-100]
test_birds = birds[-100:]
return train_birds, val_birds, test_birds
def load_birds():
birds = []
feats = np.zeros(4096)
feats_sq = np.zeros(4096)
word_counter = defaultdict(lambda: 0)
with open("data/birds/cub_0917_5cap.tsv") as caption_f:
for line in caption_f:
parts = line.strip().split("\t")
caption = parts[-1]
caption = (caption.lower().replace(".", "").replace(",", " , "))
words = ["<s>"] + caption.split() + ["</s>"]
for word in words:
word_counter[word] += 1
for word, count in word_counter.items():
if count >= MIN_WORD_COUNT:
WORD_INDEX.index(word)
with open("data/birds/cub_0917_5cap.tsv") as caption_f:
caption_f.readline()
for line in caption_f:
parts = line.strip().split("\t")
caption = parts[-1]
image_path = parts[-2]
image_id = image_path.split("/")[-1]
caption = (caption.lower().replace(".", "").replace(",", " , "))
words = ["<s>"] + caption.split() + ["</s>"]
word_ids = [WORD_INDEX[w] for w in words]
with np.load("data/birds/embeddings/%s.npz" %
image_id) as feature_f:
features = feature_f[feature_f.keys()[0]]
birds.append(Bird(image_id, word_ids, features))
feats += features
feats_sq += features**2
mean_feats = feats / len(birds)
mean_feats_sq = feats_sq / len(birds)
var_feats = mean_feats_sq - (mean_feats**2)
std_feats = np.sqrt(var_feats)
std_feats += 0.0001
for bird in birds:
bird.features[...] -= mean_feats
bird.features[...] /= std_feats
train_birds = birds[:-1100]
val_birds = birds[-1100:-100]
test_birds = birds[-100:]
return train_birds, val_birds, test_birds
def load_scenes(scene_props):
scenes = []
word_counter = defaultdict(lambda: 0)
for sent_file_id in range(1, 3):
with open("data/abstract/SimpleSentences/SimpleSentences%d_10020.txt" %
sent_file_id) as sent_f:
for sent_line in sent_f:
sent_parts = sent_line.strip().split("\t")
sent = sent_parts[2]
sent = sent.replace('"', ' " ')
sent = sent.replace("'", " ' ")
sent = re.sub(r"[.?!]", "", sent)
words = sent.lower().split()
words = ["<s>"] + words + ["</s>"]
for word in words:
word_counter[word] += 1
for word, count in word_counter.items():
if count >= MIN_WORD_COUNT:
WORD_INDEX.index(word)
for sent_file_id in range(1, 3):
with open("data/abstract/SimpleSentences/SimpleSentences%d_10020.txt" %
sent_file_id) as sent_f:
for sent_line in sent_f:
sent_parts = sent_line.strip().split("\t")
scene_id = int(sent_parts[0])
props = scene_props[scene_id]
sent_id = int(sent_parts[1])
image_id = scene_id / 10
image_subid = scene_id % 10
image_strid = "%d_%d" % (image_id, image_subid)
sent = sent_parts[2]
sent = sent.replace('"', "")
sent = re.sub(r"[.?!']", "", sent)
words = sent.lower().split()
words = ["<s>"] + words + ["</s>"]
word_ids = [WORD_INDEX[w] or 0 for w in words]
with np.load("data/abstract/EmbeddedScenes/Scene%s.png.npz" %
image_strid) as feature_f:
features = feature_f[feature_f.keys()[0]]
scenes.append(Scene(image_strid, props, word_ids, features))
return scenes
def load_scenes(scene_props):
scenes = []
word_counter = defaultdict(lambda: 0)
for sent_file_id in range(1, 3):
with open("data/abstract/SimpleSentences/SimpleSentences%d_10020.txt" %
sent_file_id) as sent_f:
for sent_line in sent_f:
sent_parts = sent_line.strip().split("\t")
sent = sent_parts[2]
sent = sent.replace('"', ' " ')
sent = sent.replace("'", " ' ")
sent = re.sub(r"[.?!]", "", sent)
words = sent.lower().split()
words = ["<s>"] + words + ["</s>"]
for word in words:
word_counter[word] += 1
for word, count in word_counter.items():
if count >= MIN_WORD_COUNT:
WORD_INDEX.index(word)
for sent_file_id in range(1, 3):
with open("data/abstract/SimpleSentences/SimpleSentences%d_10020.txt" %
sent_file_id) as sent_f:
for sent_line in sent_f:
sent_parts = sent_line.strip().split("\t")
scene_id = int(sent_parts[0])
props = scene_props[scene_id]
sent_id = int(sent_parts[1])
image_id = scene_id / 10
image_subid = scene_id % 10
image_strid = "%d_%d" % (image_id, image_subid)
sent = sent_parts[2]
sent = sent.replace('"', "")
sent = re.sub(r"[.?!']", "", sent)
words = sent.lower().split()
words = ["<s>"] + words + ["</s>"]
word_ids = [WORD_INDEX[w] or 0 for w in words]
with np.load("data/abstract/EmbeddedScenes/Scene%s.png.npz" %
image_strid) as feature_f:
features = feature_f[feature_f.keys()[0]]
scenes.append(Scene(image_strid, props, word_ids, features))
return scenes
