def __init__(self, config): super().__init__() self.config = config self.output_attentions = self.config.output_attentions self.output_hidden_states = self.config.output_hidden_states # If bert_model_name is not specified, you will need to specify # all of the required parameters for BERTConfig and a pretrained # model won't be loaded self.bert_model_name = getattr(self.config, "bert_model_name", None) self.bert_config = BertConfig.from_dict( OmegaConf.to_container(self.config, resolve=True) ) if self.bert_model_name is None: self.bert = VisualBERTBase( self.bert_config, visual_embedding_dim=self.config.visual_embedding_dim, embedding_strategy=self.config.embedding_strategy, bypass_transformer=self.config.bypass_transformer, output_attentions=self.config.output_attentions, output_hidden_states=self.config.output_hidden_states, ) else: self.bert = VisualBERTBase.from_pretrained( self.config.bert_model_name, config=self.bert_config, cache_dir=os.path.join( get_mmf_cache_dir(), "distributed_{}".format(-1) ), visual_embedding_dim=self.config.visual_embedding_dim, embedding_strategy=self.config.embedding_strategy, bypass_transformer=self.config.bypass_transformer, output_attentions=self.config.output_attentions, output_hidden_states=self.config.output_hidden_states, ) self.vocab_size = self.bert.config.vocab_size # TODO: Once omegaconf fixes int keys issue, bring this back # See https://github.com/omry/omegaconf/issues/149 # with omegaconf.open_dict(self.config): # # Add bert config such as hidden_state to our main config # self.config.update(self.bert.config.to_dict()) if self.bert_model_name is None: bert_masked_lm = BertForPreTraining(self.bert.config) else: bert_masked_lm = BertForPreTraining.from_pretrained( self.config.bert_model_name, config=self.bert.config, cache_dir=os.path.join( get_mmf_cache_dir(), "distributed_{}".format(-1) ), ) self.cls = deepcopy(bert_masked_lm.cls) self.loss_fct = nn.CrossEntropyLoss(ignore_index=-1) self.init_weights()
def __init__(self, embedding_name, *args, **kwargs): """Use this if you want to use pretrained embedding. See description of IntersectedVocab to get a list of the embedding available from torchtext Parameters ---------- embedding_name : str Name of the pretrained alias for the embedding to used """ self.type = "pretrained" if embedding_name not in vocab.pretrained_aliases: from mmf.common.registry import registry writer = registry.get("writer") error = "Unknown embedding type: %s" % embedding_name, "error" if writer is not None: writer.write(error, "error") raise RuntimeError(error) vector_cache = get_mmf_cache_dir() # First test loading the vectors in master so that everybody doesn't # download it in case it doesn't exist if is_master(): vocab.pretrained_aliases[embedding_name](cache=vector_cache) synchronize() embedding = vocab.pretrained_aliases[embedding_name]( cache=vector_cache) self.UNK_INDEX = 3 self.stoi = defaultdict(lambda: self.UNK_INDEX) self.itos = {} self.itos[self.PAD_INDEX] = self.PAD_TOKEN self.itos[self.SOS_INDEX] = self.SOS_TOKEN self.itos[self.EOS_INDEX] = self.EOS_TOKEN self.itos[self.UNK_INDEX] = self.UNK_TOKEN self.stoi[self.SOS_TOKEN] = self.SOS_INDEX self.stoi[self.EOS_TOKEN] = self.EOS_INDEX self.stoi[self.PAD_TOKEN] = self.PAD_INDEX self.stoi[self.UNK_TOKEN] = self.UNK_INDEX self.vectors = torch.FloatTensor( len(self.itos.keys()) + len(embedding.itos), len(embedding.vectors[0])) for i in range(4): self.vectors[i] = torch.ones_like(self.vectors[i]) * 0.1 * i index = 4 for word in embedding.stoi: self.itos[index] = word self.stoi[word] = index actual_index = embedding.stoi[word] self.vectors[index] = embedding.vectors[actual_index] index += 1
def _try_download(self): _is_master = is_master() if self._already_downloaded: return needs_download = False if not hasattr(self.config, "model_file"): if _is_master: warnings.warn("'model_file' key is required but missing " "from FastTextProcessor's config.") needs_download = True model_file = self.config.model_file # If model_file is already an existing path don't join to cache dir if not PathManager.exists(model_file): model_file = os.path.join(get_mmf_cache_dir(), model_file) if not PathManager.exists(model_file): if _is_master: warnings.warn(f"No model file present at {model_file}.") needs_download = True if needs_download: logger.info("Downloading FastText bin") model_file = self._download_model() self.model_file = model_file self._already_downloaded = True synchronize()
def __init__(self, config, extra_config): super().__init__() self.config = config self.output_attentions = self.config.output_attentions self.output_hidden_states = self.config.output_hidden_states self.pooler_strategy = self.config.get("pooler_strategy", "default") # Graph input params self.feed_graph_to_vb = extra_config["feed_graph_to_vb"] self.graph_node_hid_dim = extra_config["node_hid_dim"] self.graph_feed_mode = extra_config["feed_mode"] self.graph_topk = extra_config["topk_ans_feed"] # If doing graph, make a graph embedding layer if self.feed_graph_to_vb: self.graph_embedding = nn.Sequential( nn.Linear(self.graph_node_hid_dim, config.hidden_size), nn.LayerNorm(config.hidden_size, eps=1e-12), nn.Dropout(config.hidden_dropout_prob), # hidden_dropout_prb ) # If bert_model_name is not specified, you will need to specify # all of the required parameters for BERTConfig and a pretrained # model won't be loaded self.bert_model_name = self.config.get("bert_model_name", None) self.bert_config = BertConfig.from_dict( OmegaConf.to_container(self.config, resolve=True) ) if self.bert_model_name is None or self.bert_model_name == "nopretrain": self.bert = VisualBERTBase( self.bert_config, visual_embedding_dim=self.config.visual_embedding_dim, embedding_strategy=self.config.embedding_strategy, bypass_transformer=self.config.bypass_transformer, output_attentions=self.config.output_attentions, output_hidden_states=self.config.output_hidden_states, ) else: self.bert = VisualBERTBase.from_pretrained( self.config.bert_model_name, config=self.bert_config, cache_dir=os.path.join( get_mmf_cache_dir(), "distributed_{}".format(-1) ), visual_embedding_dim=self.config.visual_embedding_dim, embedding_strategy=self.config.embedding_strategy, bypass_transformer=self.config.bypass_transformer, output_attentions=self.config.output_attentions, output_hidden_states=self.config.output_hidden_states, ) self.training_head_type = self.config.training_head_type self.dropout = nn.Dropout(self.bert.config.hidden_dropout_prob) if self.config.training_head_type == "nlvr2": self.bert.config.hidden_size *= 2 self.classifier = nn.Sequential(BertPredictionHeadTransform(self.bert.config)) self.init_weights()
def _load_fasttext_model(self, model_file): from fastText import load_model model_file = os.path.join(get_mmf_cache_dir(), model_file) logger.info(f"Loading fasttext model now from {model_file}") self.model = load_model(model_file) self.stoi = WordToVectorDict(self.model)
def __init__(self, config, *args, **kwargs): super().__init__() self.config = config self.module = AutoModel.from_pretrained( self.config.bert_model_name, config=self._build_encoder_config(config), cache_dir=os.path.join(get_mmf_cache_dir(), "distributed_{}".format(-1)), ) self.embeddings = self.module.embeddings self.config = self.module.config
def _load_fasttext_model(self, model_file): from fastText import load_model from mmf.common.registry import registry model_file = os.path.join(get_mmf_cache_dir(), model_file) registry.get("writer").write("Loading fasttext model now from %s" % model_file) self.model = load_model(model_file) self.stoi = WordToVectorDict(self.model)
def __init__(self, config): super().__init__() self.config = config self.output_attentions = self.config.output_attentions self.output_hidden_states = self.config.output_hidden_states self.pooler_strategy = self.config.get("pooler_strategy", "default") # If bert_model_name is not specified, you will need to specify # all of the required parameters for BERTConfig and a pretrained # model won't be loaded self.bert_model_name = getattr(self.config, "bert_model_name", None) self.bert_config = BertConfig.from_dict( OmegaConf.to_container(self.config, resolve=True) ) if self.bert_model_name is None: self.bert = VisualBERTBase( self.bert_config, visual_embedding_dim=self.config.visual_embedding_dim, embedding_strategy=self.config.embedding_strategy, bypass_transformer=self.config.bypass_transformer, output_attentions=self.config.output_attentions, output_hidden_states=self.config.output_hidden_states, ) else: self.bert = VisualBERTBase.from_pretrained( self.config.bert_model_name, config=self.bert_config, cache_dir=os.path.join( get_mmf_cache_dir(), "distributed_{}".format(-1) ), visual_embedding_dim=self.config.visual_embedding_dim, embedding_strategy=self.config.embedding_strategy, bypass_transformer=self.config.bypass_transformer, output_attentions=self.config.output_attentions, output_hidden_states=self.config.output_hidden_states, ) self.training_head_type = self.config.training_head_type self.num_labels = self.config.num_labels self.dropout = nn.Dropout(self.bert.config.hidden_dropout_prob) if self.config.training_head_type == "nlvr2": self.bert.config.hidden_size *= 2 self.classifier = nn.Sequential( BertPredictionHeadTransform(self.bert.config), nn.Linear(self.bert.config.hidden_size, self.config.num_labels), ) self.init_weights()
def __init__(self, config, *args, **kwargs): super().__init__() self.config = config self.bert = MMBTBase(config, *args, **kwargs) self.encoder_config = self.bert.encoder_config # TODO : Switch to AutoModelForPreTraining after transformers # package upgrade to 2.5 pretraining_module = BertForPreTraining.from_pretrained( self.config.bert_model_name, config=self.encoder_config, cache_dir=os.path.join(get_mmf_cache_dir(), "distributed_{}".format(-1)), ) self.cls = deepcopy(pretraining_module.cls) self.loss_fct = nn.CrossEntropyLoss(ignore_index=-1) self.tie_weights()
def __init__(self, config, mode="lxr"): super().__init__() self.config = config self.num_labels = config.num_labels self.gqa_labels = config.gqa_labels self.mode = config.mode self.bert = LXMERTBase.from_pretrained( self.config.bert_model_name, config=BertConfig.from_dict( OmegaConf.to_container(self.config, resolve=True)), cache_dir=os.path.join(get_mmf_cache_dir(), "distributed_{}".format(-1)), ) self.classifier = BertVisualAnswerHead( config, [self.num_labels, self.gqa_labels]) self.init_weights()
def _download_model(self): _is_master = is_master() model_file_path = os.path.join(get_mmf_cache_dir(), "wiki.en.bin") if not _is_master: return model_file_path if PathManager.exists(model_file_path): self.writer.write( "Vectors already present at {}.".format(model_file_path), "info") return model_file_path import requests from mmf.common.constants import FASTTEXT_WIKI_URL from tqdm import tqdm PathManager.mkdirs(os.path.dirname(model_file_path)) response = requests.get(FASTTEXT_WIKI_URL, stream=True) with PathManager.open(model_file_path, "wb") as f: pbar = tqdm( total=int(response.headers["Content-Length"]) / 4096, miniters=50, disable=not _is_master, ) idx = 0 for data in response.iter_content(chunk_size=4096): if data: if idx % 50 == 0: pbar.update(len(data)) f.write(data) idx += 1 pbar.close() self.writer.write( "fastText bin downloaded at {}.".format(model_file_path), "info") return model_file_path
def build_transformer(self): """Build the transformer encoder. This uses transformers AutoModel to load a pretrained model given the name of the transformer based model. All the layers in the transformer model will be available (encoder, embeddings etc.) for use. Different HUggingface transformer models have different naming conventions for the layers. Adjust your derived class based on the transformer base you want to use. Example :: self.transformer = AutoModel.from_pretrained( "bert-base-uncased", config=self.transformer_config, ) """ self.transformer = AutoModel.from_pretrained( self.config.transformer_base, config=self.transformer_config, cache_dir=os.path.join(get_mmf_cache_dir(), "distributed_{}".format(-1)), )
def __init__(self, config): super().__init__() # Configuration self.config = config # LXMERT backbone self.bert = LXMERTBase.from_pretrained( self.config.bert_model_name, config=BertConfig.from_dict( OmegaConf.to_container(self.config, resolve=True)), cache_dir=os.path.join(get_mmf_cache_dir(), "distributed_{}".format(-1)), ) self.num_labels = config.num_labels self.gqa_labels = config.gqa_labels self.task_mask_lm = config.task_mask_lm self.task_obj_predict = config.task_obj_predict self.task_matched = config.task_matched self.task_qa = config.task_qa self.visual_losses = config.visual_losses self.visual_loss_config = config.visual_loss_config # Pre-training heads self.cls = BertPreTrainingHeads( config, self.bert.embeddings.word_embeddings.weight) if self.task_obj_predict: self.obj_predict_head = BertVisualObjHead(config) if self.task_qa: self.answer_head = BertVisualAnswerHead( config, [self.num_labels, self.gqa_labels]) # loss functions self.loss_fcts = { "l2": SmoothL1Loss(reduction="none"), "ce": CrossEntropyLoss(ignore_index=-1, reduction="none"), "ce_lang": CrossEntropyLoss(ignore_index=-1), }
def __init__(self, vocab_file, embedding_name, *args, **kwargs): """Use this vocab class when you have a custom vocabulary class but you want to use pretrained embedding vectos for it. This will only load the vectors which intersect with your vocabulary. Use the embedding_name specified in torchtext's pretrained aliases: ['charngram.100d', 'fasttext.en.300d', 'fasttext.simple.300d', 'glove.42B.300d', 'glove.840B.300d', 'glove.twitter.27B.25d', 'glove.twitter.27B.50d', 'glove.twitter.27B.100d', 'glove.twitter.27B.200d', 'glove.6B.50d', 'glove.6B.100d', 'glove.6B.200d', 'glove.6B.300d'] Parameters ---------- vocab_file : str Vocabulary file containing list of words with one word per line which will be used to collect vectors embedding_name : str Embedding name picked up from the list of the pretrained aliases mentioned above """ super().__init__(vocab_file, *args, **kwargs) self.type = "intersected" name = embedding_name.split(".")[0] dim = embedding_name.split(".")[2][:-1] middle = embedding_name.split(".")[1] class_name = EMBEDDING_NAME_CLASS_MAPPING[name] if not hasattr(vocab, class_name): raise RuntimeError(f"Unknown embedding type: {name}") params = [middle] if name == "glove": params.append(int(dim)) vector_cache = get_mmf_cache_dir() # First test loading the vectors in master so that everybody doesn't # download it in case it doesn't exist if is_main(): vocab.pretrained_aliases[embedding_name](cache=vector_cache) synchronize() embedding = getattr(vocab, class_name)(*params, cache=vector_cache) self.vectors = torch.empty( (self.get_size(), len(embedding.vectors[0])), dtype=torch.float) self.embedding_dim = len(embedding.vectors[0]) for i in range(0, 4): self.vectors[i] = torch.ones_like(self.vectors[i]) * 0.1 * i for i in range(4, self.get_size()): word = self.itos[i] embedding_index = embedding.stoi.get(word, None) if embedding_index is None: self.vectors[i] = self.vectors[self.UNK_INDEX] else: self.vectors[i] = embedding.vectors[embedding_index]
import os import pickle as pkl import shutil import tarfile from collections import OrderedDict from pathlib import Path from urllib.parse import urlparse from zipfile import ZipFile, is_zipfile import numpy as np import requests from filelock import FileLock from mmf.utils.configuration import get_mmf_cache_dir from omegaconf import OmegaConf mmf_cache_home = get_mmf_cache_dir() try: import torch _torch_available = True except ImportError: _torch_available = False default_cache_path = os.path.join(mmf_cache_home, "transformers") PATH = "/".join(str(Path(__file__).resolve()).split("/")[:-1]) CONFIG = os.path.join(PATH, "config.yaml") ATTRIBUTES = os.path.join(PATH, "attributes.txt") OBJECTS = os.path.join(PATH, "objects.txt") PYTORCH_PRETRAINED_BERT_CACHE = os.getenv("PYTORCH_PRETRAINED_BERT_CACHE",
"image_id": quest["image_id"], "feature_path": f"{image_name}.npy", "question_id": q_id, "question_str": quest["question"], "question_tokens": tokenize(quest["question"]), "answers": answers, "all_answers": all_answers, } imdb.append(entry) print("Unknown questions:", count) return np.array(imdb) if __name__ == "__main__": split = "val2014" root_dir = os.path.join(get_mmf_cache_dir(), "data", "datasets", "okvqa", "defaults") ann_path = os.path.join(root_dir, "annotations", f"mscoco_{split}_annotations.json") quest_path = os.path.join(root_dir, "annotations", f"OpenEnded_mscoco_{split}_questions.json") answer_vocab_path = os.path.join(root_dir, "extras", "vocabs", "answers_okvqa.txt") res_path = os.path.join(root_dir, "annotations", f"imdb_{split}.npy") imdb = get_imdb(ann_path, quest_path, split, answer_vocab_path) np.save(res_path, imdb)
import os import json from mmf.utils.configuration import get_mmf_cache_dir from mmf.utils.file_io import PathManager if __name__ == "__main__": src_dataset = 'vqa2' dst_dataset = 'okvqa' src_fname = "answers_vqa.txt" dst_fname = "answers_okvqa.txt" gt2raw_fname = "gt2raw_answers.json" use_raw = True use_raw_str = "_raw" if use_raw else "" out_fname = f"{src_dataset}2{dst_dataset}{use_raw_str}.json" src_dir = os.path.join(get_mmf_cache_dir(), "data", "datasets", src_dataset, "defaults", "extras", "vocabs") dst_dir = os.path.join(get_mmf_cache_dir(), "data", "datasets", dst_dataset, "defaults", "extras", "vocabs") with PathManager.open(os.path.join(src_dir, src_fname), "r") as f: src_vocab = f.read().splitlines() with PathManager.open(os.path.join(dst_dir, dst_fname), "r") as f: dst_vocab = f.read().splitlines() if use_raw: with PathManager.open(os.path.join(dst_dir, gt2raw_fname), "r") as f: gt2raw = json.load(f) src_dict = {w: i for i, w in enumerate(src_vocab)}