def __init__(self, config):
super().__init__()
self.config = config
self.output_attentions = self.config.output_attentions
self.output_hidden_states = self.config.output_hidden_states
# If bert_model_name is not specified, you will need to specify
# all of the required parameters for BERTConfig and a pretrained
# model won't be loaded
self.bert_model_name = getattr(self.config, "bert_model_name", None)
self.bert_config = BertConfig.from_dict(
OmegaConf.to_container(self.config, resolve=True)
)
if self.bert_model_name is None:
self.bert = VisualBERTBase(
self.bert_config,
visual_embedding_dim=self.config.visual_embedding_dim,
embedding_strategy=self.config.embedding_strategy,
bypass_transformer=self.config.bypass_transformer,
output_attentions=self.config.output_attentions,
output_hidden_states=self.config.output_hidden_states,
)
else:
self.bert = VisualBERTBase.from_pretrained(
self.config.bert_model_name,
config=self.bert_config,
cache_dir=os.path.join(
get_mmf_cache_dir(), "distributed_{}".format(-1)
),
visual_embedding_dim=self.config.visual_embedding_dim,
embedding_strategy=self.config.embedding_strategy,
bypass_transformer=self.config.bypass_transformer,
output_attentions=self.config.output_attentions,
output_hidden_states=self.config.output_hidden_states,
)
self.vocab_size = self.bert.config.vocab_size
# TODO: Once omegaconf fixes int keys issue, bring this back
# See https://github.com/omry/omegaconf/issues/149
# with omegaconf.open_dict(self.config):
# # Add bert config such as hidden_state to our main config
# self.config.update(self.bert.config.to_dict())
if self.bert_model_name is None:
bert_masked_lm = BertForPreTraining(self.bert.config)
else:
bert_masked_lm = BertForPreTraining.from_pretrained(
self.config.bert_model_name,
config=self.bert.config,
cache_dir=os.path.join(
get_mmf_cache_dir(), "distributed_{}".format(-1)
),
)
self.cls = deepcopy(bert_masked_lm.cls)
self.loss_fct = nn.CrossEntropyLoss(ignore_index=-1)
self.init_weights()
def __init__(self, embedding_name, *args, **kwargs):
"""Use this if you want to use pretrained embedding. See description
of IntersectedVocab to get a list of the embedding available from
torchtext
Parameters
----------
embedding_name : str
Name of the pretrained alias for the embedding to used
"""
self.type = "pretrained"
if embedding_name not in vocab.pretrained_aliases:
from mmf.common.registry import registry
writer = registry.get("writer")
error = "Unknown embedding type: %s" % embedding_name, "error"
if writer is not None:
writer.write(error, "error")
raise RuntimeError(error)
vector_cache = get_mmf_cache_dir()
# First test loading the vectors in master so that everybody doesn't
# download it in case it doesn't exist
if is_master():
vocab.pretrained_aliases[embedding_name](cache=vector_cache)
synchronize()
embedding = vocab.pretrained_aliases[embedding_name](
cache=vector_cache)
self.UNK_INDEX = 3
self.stoi = defaultdict(lambda: self.UNK_INDEX)
self.itos = {}
self.itos[self.PAD_INDEX] = self.PAD_TOKEN
self.itos[self.SOS_INDEX] = self.SOS_TOKEN
self.itos[self.EOS_INDEX] = self.EOS_TOKEN
self.itos[self.UNK_INDEX] = self.UNK_TOKEN
self.stoi[self.SOS_TOKEN] = self.SOS_INDEX
self.stoi[self.EOS_TOKEN] = self.EOS_INDEX
self.stoi[self.PAD_TOKEN] = self.PAD_INDEX
self.stoi[self.UNK_TOKEN] = self.UNK_INDEX
self.vectors = torch.FloatTensor(
len(self.itos.keys()) + len(embedding.itos),
len(embedding.vectors[0]))
for i in range(4):
self.vectors[i] = torch.ones_like(self.vectors[i]) * 0.1 * i
index = 4
for word in embedding.stoi:
self.itos[index] = word
self.stoi[word] = index
actual_index = embedding.stoi[word]
self.vectors[index] = embedding.vectors[actual_index]
index += 1
def _try_download(self):
_is_master = is_master()
if self._already_downloaded:
return
needs_download = False
if not hasattr(self.config, "model_file"):
if _is_master:
warnings.warn("'model_file' key is required but missing "
"from FastTextProcessor's config.")
needs_download = True
model_file = self.config.model_file
# If model_file is already an existing path don't join to cache dir
if not PathManager.exists(model_file):
model_file = os.path.join(get_mmf_cache_dir(), model_file)
if not PathManager.exists(model_file):
if _is_master:
warnings.warn(f"No model file present at {model_file}.")
needs_download = True
if needs_download:
logger.info("Downloading FastText bin")
model_file = self._download_model()
self.model_file = model_file
self._already_downloaded = True
synchronize()
def __init__(self, config, extra_config):
super().__init__()
self.config = config
self.output_attentions = self.config.output_attentions
self.output_hidden_states = self.config.output_hidden_states
self.pooler_strategy = self.config.get("pooler_strategy", "default")
# Graph input params
self.feed_graph_to_vb = extra_config["feed_graph_to_vb"]
self.graph_node_hid_dim = extra_config["node_hid_dim"]
self.graph_feed_mode = extra_config["feed_mode"]
self.graph_topk = extra_config["topk_ans_feed"]
# If doing graph, make a graph embedding layer
if self.feed_graph_to_vb:
self.graph_embedding = nn.Sequential(
nn.Linear(self.graph_node_hid_dim, config.hidden_size),
nn.LayerNorm(config.hidden_size, eps=1e-12),
nn.Dropout(config.hidden_dropout_prob), # hidden_dropout_prb
)
# If bert_model_name is not specified, you will need to specify
# all of the required parameters for BERTConfig and a pretrained
# model won't be loaded
self.bert_model_name = self.config.get("bert_model_name", None)
self.bert_config = BertConfig.from_dict(
OmegaConf.to_container(self.config, resolve=True)
)
if self.bert_model_name is None or self.bert_model_name == "nopretrain":
self.bert = VisualBERTBase(
self.bert_config,
visual_embedding_dim=self.config.visual_embedding_dim,
embedding_strategy=self.config.embedding_strategy,
bypass_transformer=self.config.bypass_transformer,
output_attentions=self.config.output_attentions,
output_hidden_states=self.config.output_hidden_states,
)
else:
self.bert = VisualBERTBase.from_pretrained(
self.config.bert_model_name,
config=self.bert_config,
cache_dir=os.path.join(
get_mmf_cache_dir(), "distributed_{}".format(-1)
),
visual_embedding_dim=self.config.visual_embedding_dim,
embedding_strategy=self.config.embedding_strategy,
bypass_transformer=self.config.bypass_transformer,
output_attentions=self.config.output_attentions,
output_hidden_states=self.config.output_hidden_states,
)
self.training_head_type = self.config.training_head_type
self.dropout = nn.Dropout(self.bert.config.hidden_dropout_prob)
if self.config.training_head_type == "nlvr2":
self.bert.config.hidden_size *= 2
self.classifier = nn.Sequential(BertPredictionHeadTransform(self.bert.config))
self.init_weights()
def _load_fasttext_model(self, model_file):
from fastText import load_model
model_file = os.path.join(get_mmf_cache_dir(), model_file)
logger.info(f"Loading fasttext model now from {model_file}")
self.model = load_model(model_file)
self.stoi = WordToVectorDict(self.model)
def __init__(self, config, *args, **kwargs):
super().__init__()
self.config = config
self.module = AutoModel.from_pretrained(
self.config.bert_model_name,
config=self._build_encoder_config(config),
cache_dir=os.path.join(get_mmf_cache_dir(), "distributed_{}".format(-1)),
)
self.embeddings = self.module.embeddings
self.config = self.module.config
def _load_fasttext_model(self, model_file):
from fastText import load_model
from mmf.common.registry import registry
model_file = os.path.join(get_mmf_cache_dir(), model_file)
registry.get("writer").write("Loading fasttext model now from %s" % model_file)
self.model = load_model(model_file)
self.stoi = WordToVectorDict(self.model)
def __init__(self, config):
super().__init__()
self.config = config
self.output_attentions = self.config.output_attentions
self.output_hidden_states = self.config.output_hidden_states
self.pooler_strategy = self.config.get("pooler_strategy", "default")
# If bert_model_name is not specified, you will need to specify
# all of the required parameters for BERTConfig and a pretrained
# model won't be loaded
self.bert_model_name = getattr(self.config, "bert_model_name", None)
self.bert_config = BertConfig.from_dict(
OmegaConf.to_container(self.config, resolve=True)
)
if self.bert_model_name is None:
self.bert = VisualBERTBase(
self.bert_config,
visual_embedding_dim=self.config.visual_embedding_dim,
embedding_strategy=self.config.embedding_strategy,
bypass_transformer=self.config.bypass_transformer,
output_attentions=self.config.output_attentions,
output_hidden_states=self.config.output_hidden_states,
)
else:
self.bert = VisualBERTBase.from_pretrained(
self.config.bert_model_name,
config=self.bert_config,
cache_dir=os.path.join(
get_mmf_cache_dir(), "distributed_{}".format(-1)
),
visual_embedding_dim=self.config.visual_embedding_dim,
embedding_strategy=self.config.embedding_strategy,
bypass_transformer=self.config.bypass_transformer,
output_attentions=self.config.output_attentions,
output_hidden_states=self.config.output_hidden_states,
)
self.training_head_type = self.config.training_head_type
self.num_labels = self.config.num_labels
self.dropout = nn.Dropout(self.bert.config.hidden_dropout_prob)
if self.config.training_head_type == "nlvr2":
self.bert.config.hidden_size *= 2
self.classifier = nn.Sequential(
BertPredictionHeadTransform(self.bert.config),
nn.Linear(self.bert.config.hidden_size, self.config.num_labels),
)
self.init_weights()
def __init__(self, config, *args, **kwargs):
super().__init__()
self.config = config
self.bert = MMBTBase(config, *args, **kwargs)
self.encoder_config = self.bert.encoder_config
# TODO : Switch to AutoModelForPreTraining after transformers
# package upgrade to 2.5
pretraining_module = BertForPreTraining.from_pretrained(
self.config.bert_model_name,
config=self.encoder_config,
cache_dir=os.path.join(get_mmf_cache_dir(), "distributed_{}".format(-1)),
)
self.cls = deepcopy(pretraining_module.cls)
self.loss_fct = nn.CrossEntropyLoss(ignore_index=-1)
self.tie_weights()
def __init__(self, config, mode="lxr"):
super().__init__()
self.config = config
self.num_labels = config.num_labels
self.gqa_labels = config.gqa_labels
self.mode = config.mode
self.bert = LXMERTBase.from_pretrained(
self.config.bert_model_name,
config=BertConfig.from_dict(
OmegaConf.to_container(self.config, resolve=True)),
cache_dir=os.path.join(get_mmf_cache_dir(),
"distributed_{}".format(-1)),
)
self.classifier = BertVisualAnswerHead(
config, [self.num_labels, self.gqa_labels])
self.init_weights()
def _download_model(self):
_is_master = is_master()
model_file_path = os.path.join(get_mmf_cache_dir(), "wiki.en.bin")
if not _is_master:
return model_file_path
if PathManager.exists(model_file_path):
self.writer.write(
"Vectors already present at {}.".format(model_file_path),
"info")
return model_file_path
import requests
from mmf.common.constants import FASTTEXT_WIKI_URL
from tqdm import tqdm
PathManager.mkdirs(os.path.dirname(model_file_path))
response = requests.get(FASTTEXT_WIKI_URL, stream=True)
with PathManager.open(model_file_path, "wb") as f:
pbar = tqdm(
total=int(response.headers["Content-Length"]) / 4096,
miniters=50,
disable=not _is_master,
)
idx = 0
for data in response.iter_content(chunk_size=4096):
if data:
if idx % 50 == 0:
pbar.update(len(data))
f.write(data)
idx += 1
pbar.close()
self.writer.write(
"fastText bin downloaded at {}.".format(model_file_path), "info")
return model_file_path
def build_transformer(self):
"""Build the transformer encoder. This uses transformers AutoModel to load a
pretrained model given the name of the transformer based model. All the layers
in the transformer model will be available (encoder, embeddings etc.) for use.
Different HUggingface transformer models have different naming conventions for
the layers. Adjust your derived class based on the transformer base you want to
use.
Example ::
self.transformer = AutoModel.from_pretrained(
"bert-base-uncased",
config=self.transformer_config,
)
"""
self.transformer = AutoModel.from_pretrained(
self.config.transformer_base,
config=self.transformer_config,
cache_dir=os.path.join(get_mmf_cache_dir(),
"distributed_{}".format(-1)),
)
def __init__(self, config):
super().__init__()
# Configuration
self.config = config
# LXMERT backbone
self.bert = LXMERTBase.from_pretrained(
self.config.bert_model_name,
config=BertConfig.from_dict(
OmegaConf.to_container(self.config, resolve=True)),
cache_dir=os.path.join(get_mmf_cache_dir(),
"distributed_{}".format(-1)),
)
self.num_labels = config.num_labels
self.gqa_labels = config.gqa_labels
self.task_mask_lm = config.task_mask_lm
self.task_obj_predict = config.task_obj_predict
self.task_matched = config.task_matched
self.task_qa = config.task_qa
self.visual_losses = config.visual_losses
self.visual_loss_config = config.visual_loss_config
# Pre-training heads
self.cls = BertPreTrainingHeads(
config, self.bert.embeddings.word_embeddings.weight)
if self.task_obj_predict:
self.obj_predict_head = BertVisualObjHead(config)
if self.task_qa:
self.answer_head = BertVisualAnswerHead(
config, [self.num_labels, self.gqa_labels])
# loss functions
self.loss_fcts = {
"l2": SmoothL1Loss(reduction="none"),
"ce": CrossEntropyLoss(ignore_index=-1, reduction="none"),
"ce_lang": CrossEntropyLoss(ignore_index=-1),
}
def __init__(self, vocab_file, embedding_name, *args, **kwargs):
"""Use this vocab class when you have a custom vocabulary class but you
want to use pretrained embedding vectos for it. This will only load
the vectors which intersect with your vocabulary. Use the
embedding_name specified in torchtext's pretrained aliases:
['charngram.100d', 'fasttext.en.300d', 'fasttext.simple.300d',
'glove.42B.300d', 'glove.840B.300d', 'glove.twitter.27B.25d',
'glove.twitter.27B.50d', 'glove.twitter.27B.100d',
'glove.twitter.27B.200d', 'glove.6B.50d', 'glove.6B.100d',
'glove.6B.200d', 'glove.6B.300d']
Parameters
----------
vocab_file : str
Vocabulary file containing list of words with one word per line
which will be used to collect vectors
embedding_name : str
Embedding name picked up from the list of the pretrained aliases
mentioned above
"""
super().__init__(vocab_file, *args, **kwargs)
self.type = "intersected"
name = embedding_name.split(".")[0]
dim = embedding_name.split(".")[2][:-1]
middle = embedding_name.split(".")[1]
class_name = EMBEDDING_NAME_CLASS_MAPPING[name]
if not hasattr(vocab, class_name):
raise RuntimeError(f"Unknown embedding type: {name}")
params = [middle]
if name == "glove":
params.append(int(dim))
vector_cache = get_mmf_cache_dir()
# First test loading the vectors in master so that everybody doesn't
# download it in case it doesn't exist
if is_main():
vocab.pretrained_aliases[embedding_name](cache=vector_cache)
synchronize()
embedding = getattr(vocab, class_name)(*params, cache=vector_cache)
self.vectors = torch.empty(
(self.get_size(), len(embedding.vectors[0])), dtype=torch.float)
self.embedding_dim = len(embedding.vectors[0])
for i in range(0, 4):
self.vectors[i] = torch.ones_like(self.vectors[i]) * 0.1 * i
for i in range(4, self.get_size()):
word = self.itos[i]
embedding_index = embedding.stoi.get(word, None)
if embedding_index is None:
self.vectors[i] = self.vectors[self.UNK_INDEX]
else:
self.vectors[i] = embedding.vectors[embedding_index]
import os
import pickle as pkl
import shutil
import tarfile
from collections import OrderedDict
from pathlib import Path
from urllib.parse import urlparse
from zipfile import ZipFile, is_zipfile
import numpy as np
import requests
from filelock import FileLock
from mmf.utils.configuration import get_mmf_cache_dir
from omegaconf import OmegaConf
mmf_cache_home = get_mmf_cache_dir()
try:
import torch
_torch_available = True
except ImportError:
_torch_available = False
default_cache_path = os.path.join(mmf_cache_home, "transformers")
PATH = "/".join(str(Path(__file__).resolve()).split("/")[:-1])
CONFIG = os.path.join(PATH, "config.yaml")
ATTRIBUTES = os.path.join(PATH, "attributes.txt")
OBJECTS = os.path.join(PATH, "objects.txt")
PYTORCH_PRETRAINED_BERT_CACHE = os.getenv("PYTORCH_PRETRAINED_BERT_CACHE",
"image_id": quest["image_id"],
"feature_path": f"{image_name}.npy",
"question_id": q_id,
"question_str": quest["question"],
"question_tokens": tokenize(quest["question"]),
"answers": answers,
"all_answers": all_answers,
}
imdb.append(entry)
print("Unknown questions:", count)
return np.array(imdb)
if __name__ == "__main__":
split = "val2014"
root_dir = os.path.join(get_mmf_cache_dir(), "data", "datasets", "okvqa",
"defaults")
ann_path = os.path.join(root_dir, "annotations",
f"mscoco_{split}_annotations.json")
quest_path = os.path.join(root_dir, "annotations",
f"OpenEnded_mscoco_{split}_questions.json")
answer_vocab_path = os.path.join(root_dir, "extras", "vocabs",
"answers_okvqa.txt")
res_path = os.path.join(root_dir, "annotations", f"imdb_{split}.npy")
imdb = get_imdb(ann_path, quest_path, split, answer_vocab_path)
np.save(res_path, imdb)
import os
import json
from mmf.utils.configuration import get_mmf_cache_dir
from mmf.utils.file_io import PathManager
if __name__ == "__main__":
src_dataset = 'vqa2'
dst_dataset = 'okvqa'
src_fname = "answers_vqa.txt"
dst_fname = "answers_okvqa.txt"
gt2raw_fname = "gt2raw_answers.json"
use_raw = True
use_raw_str = "_raw" if use_raw else ""
out_fname = f"{src_dataset}2{dst_dataset}{use_raw_str}.json"
src_dir = os.path.join(get_mmf_cache_dir(), "data", "datasets",
src_dataset, "defaults", "extras", "vocabs")
dst_dir = os.path.join(get_mmf_cache_dir(), "data", "datasets",
dst_dataset, "defaults", "extras", "vocabs")
with PathManager.open(os.path.join(src_dir, src_fname), "r") as f:
src_vocab = f.read().splitlines()
with PathManager.open(os.path.join(dst_dir, dst_fname), "r") as f:
dst_vocab = f.read().splitlines()
if use_raw:
with PathManager.open(os.path.join(dst_dir, gt2raw_fname), "r") as f:
gt2raw = json.load(f)
src_dict = {w: i for i, w in enumerate(src_vocab)}
