def __init__(self, config):
super(ResNetVLBERTForPretrainingMultitaskNoVision,
self).__init__(config)
# Constructs/initialises model elements
self.image_feature_extractor = FastRCNN(
config,
average_pool=True,
final_dim=config.NETWORK.IMAGE_FINAL_DIM,
enable_cnn_reg_loss=False)
self.object_linguistic_embeddings = nn.Embedding(
1, config.NETWORK.VLBERT.hidden_size)
if config.NETWORK.IMAGE_FEAT_PRECOMPUTED or (
not config.NETWORK.MASK_RAW_PIXELS):
self.object_mask_visual_embedding = nn.Embedding(1, 2048)
if config.NETWORK.WITH_MVRC_LOSS:
self.object_mask_word_embedding = nn.Embedding(
1, config.NETWORK.VLBERT.hidden_size)
self.aux_text_visual_embedding = nn.Embedding(
1, config.NETWORK.VLBERT.hidden_size)
self.image_feature_bn_eval = config.NETWORK.IMAGE_FROZEN_BN
self.tokenizer = BertTokenizer.from_pretrained(
config.NETWORK.BERT_MODEL_NAME)
# Can specify pre-trained model or use the downloaded pretrained model specific in .yaml file
language_pretrained_model_path = None
if config.NETWORK.BERT_PRETRAINED != '':
# language_pretrained_model_path = '{}-{:04d}.model'.format(config.NETWORK.BERT_PRETRAINED,
# config.NETWORK.BERT_PRETRAINED_EPOCH)
#FM edit: just use path of pretrained model
language_pretrained_model_path = config.NETWORK.BERT_PRETRAINED
elif os.path.isdir(config.NETWORK.BERT_MODEL_NAME):
weight_path = os.path.join(config.NETWORK.BERT_MODEL_NAME,
BERT_WEIGHTS_NAME)
if os.path.isfile(weight_path):
language_pretrained_model_path = weight_path
if language_pretrained_model_path is None:
print(
"Warning: no pretrained language model found, training from scratch!!!"
)
self.vlbert = VisualLinguisticBertForPretraining(
config.NETWORK.VLBERT,
language_pretrained_model_path=None
if config.NETWORK.VLBERT.from_scratch else
language_pretrained_model_path,
with_rel_head=config.NETWORK.WITH_REL_LOSS,
with_mlm_head=config.NETWORK.WITH_MLM_LOSS,
with_mvrc_head=config.NETWORK.WITH_MVRC_LOSS,
with_MLT_head=config.NETWORK.WITH_MLT_LOSS)
# init weights
self.init_weight()
self.fix_params()
def __init__(self, config):
super(ResNetVLBERTForPretrainingGenerateNoVision,
self).__init__(config)
self.image_feature_extractor = FastRCNN(
config,
average_pool=True,
final_dim=config.NETWORK.IMAGE_FINAL_DIM,
enable_cnn_reg_loss=False)
self.object_linguistic_embeddings = nn.Embedding(
1, config.NETWORK.VLBERT.hidden_size)
if config.NETWORK.IMAGE_FEAT_PRECOMPUTED:
self.object_mask_visual_embedding = nn.Embedding(1, 2048)
if config.NETWORK.WITH_MVRC_LOSS:
self.object_mask_word_embedding = nn.Embedding(
1, config.NETWORK.VLBERT.hidden_size)
self.image_feature_bn_eval = config.NETWORK.IMAGE_FROZEN_BN
self.tokenizer = BertTokenizer.from_pretrained(
config.NETWORK.BERT_MODEL_NAME)
language_pretrained_model_path = None
if config.NETWORK.BERT_PRETRAINED != '':
language_pretrained_model_path = '{}-{:04d}.model'.format(
config.NETWORK.BERT_PRETRAINED,
config.NETWORK.BERT_PRETRAINED_EPOCH)
elif os.path.isdir(config.NETWORK.BERT_MODEL_NAME):
weight_path = os.path.join(config.NETWORK.BERT_MODEL_NAME,
BERT_WEIGHTS_NAME)
if os.path.isfile(weight_path):
language_pretrained_model_path = weight_path
if language_pretrained_model_path is None:
print(
"Warning: no pretrained language model found, training from scratch!!!"
)
self.vlbert = VisualLinguisticBertForPretraining(
config.NETWORK.VLBERT,
language_pretrained_model_path=None
if config.NETWORK.VLBERT.from_scratch else
language_pretrained_model_path,
with_rel_head=config.NETWORK.WITH_REL_LOSS,
with_mlm_head=config.NETWORK.WITH_MLM_LOSS,
with_mvrc_head=config.NETWORK.WITH_MVRC_LOSS,
)
# init weights
self.init_weight()
self.fix_params()