def __init__(self, args, train_loader=None, val_loader=None, test_loader=None, train=True):
self.args = args
self.train_loader = train_loader
self.val_loader = val_loader
self.test_loader = test_loader
self.verbose = True
if self.args.distributed:
if self.args.gpu != 0:
self.verbose = False
if self.args.tokenizer is None:
self.args.tokenizer = self.args.backbone
if not self.verbose:
set_global_logging_level(logging.ERROR, ["transformers"])
from tqdm import tqdm
import torch
import torch.nn as nn
# from torch.utils.data.dataloader import DataLoader
import logging
import shutil
from pprint import pprint
from param import parse_args
from mmt_data import get_loader
from utils import load_state_dict, LossMeter, set_global_logging_level
import wandb
from pprint import pformat
set_global_logging_level(logging.ERROR, ["transformers"])
proj_dir = Path(__file__).resolve().parent.parent
_use_native_amp = False
_use_apex = False
# Check if Pytorch version >= 1.6 to switch between Native AMP and Apex
if version.parse(torch.__version__) < version.parse("1.6"):
from transormers.file_utils import is_apex_available
if is_apex_available():
from apex import amp
_use_apex = True
else:
_use_native_amp = True
def __init__(self, args, train_loader=None, val_loader=None, test_loader=None, train=True):
super().__init__(
args,
train_loader=train_loader,
val_loader=val_loader,
test_loader=test_loader,
train=train)
if not self.verbose:
set_global_logging_level(logging.ERROR, ["transformers"])
from vqa_model import VLT5VQA, VLBartVQA
model_kwargs = {}
if 't5' in args.backbone:
model_class = VLT5VQA
elif 'bart' in args.backbone:
model_class = VLBartVQA
config = self.create_config()
self.tokenizer = self.create_tokenizer()
if 'bart' in self.args.tokenizer:
num_added_toks = 0
if config.use_vis_order_embedding:
additional_special_tokens = [f'<extra_id_{i}>' for i in range(100-1, -1, -1)] + \
[f'<vis_extra_id_{i}>' for i in range(100-1, -1, -1)]
special_tokens_dict = {'additional_special_tokens': additional_special_tokens}
num_added_toks = self.tokenizer.add_special_tokens(special_tokens_dict)
config.default_obj_order_ids = self.tokenizer.convert_tokens_to_ids([f'<vis_extra_id_{i}>' for i in range(100)])
self.model = self.create_model(model_class, config, **model_kwargs)
if 't5' in self.args.tokenizer:
self.model.resize_token_embeddings(self.tokenizer.vocab_size)
elif 'bart' in self.args.tokenizer:
self.model.resize_token_embeddings(self.model.model.shared.num_embeddings + num_added_toks)
self.model.tokenizer = self.tokenizer
# Load Checkpoint
self.start_epoch = None
if args.load is not None:
ckpt_path = args.load + '.pth'
self.load_checkpoint(ckpt_path)
if self.args.from_scratch:
self.init_weights()
# GPU Options
print(f'Model Launching at GPU {self.args.gpu}')
if self.verbose:
from time import time
start = time()
self.model = self.model.to(args.gpu)
# Optimizer
if train:
self.optim, self.lr_scheduler = self.create_optimizer_and_scheduler()
if self.args.fp16 and _use_native_amp:
self.scaler = torch.cuda.amp.GradScaler()
elif _use_apex:
self.model, self.optim = amp.initialize(
self.model, self.optim, opt_level='O1', verbosity=self.verbose)
if args.multiGPU:
if args.distributed:
self.model = DDP(self.model, device_ids=[args.gpu],
find_unused_parameters=True
)
if self.verbose:
print(f'It took {time() - start:.1f}s')
def __init__(self,
args,
train_loader=None,
val_loader=None,
test_loader=None,
train=True):
super().__init__(args,
train_loader=train_loader,
val_loader=val_loader,
test_loader=test_loader,
train=train)
if not self.verbose:
set_global_logging_level(logging.ERROR, ["transformers"])
from multitask_model import VLT5MultiTask
model_kwargs = {}
if 't5' in args.backbone:
model_class = VLT5MultiTask
config = self.create_config()
self.tokenizer = self.create_tokenizer()
self.model = self.create_model(model_class, config, **model_kwargs)
if 't5' in self.args.tokenizer:
self.model.resize_token_embeddings(self.tokenizer.vocab_size)
self.model.tokenizer = self.tokenizer
if 't5' in self.args.tokenizer or 'bart' in self.args.tokenizer:
self.model.true_id = self.tokenizer(
'true', add_special_tokens=False).input_ids[0]
self.model.false_id = self.tokenizer(
'false', add_special_tokens=False).input_ids[0]
# Load Checkpoint
self.start_epoch = None
if args.load is not None:
ckpt_path = args.load + '.pth'
self.load_checkpoint(ckpt_path)
if self.args.from_scratch:
self.init_weights()
# GPU Options
print(f'Model Launching at GPU {self.args.gpu}')
if self.verbose:
from time import time
start = time()
self.model = self.model.to(args.gpu)
# Optimizer
if train:
self.optim, self.lr_scheduler = self.create_optimizer_and_scheduler(
)
if self.args.fp16 and _use_native_amp:
self.scaler = torch.cuda.amp.GradScaler()
elif _use_apex:
self.model, self.optim = amp.initialize(self.model,
self.optim,
opt_level='O1',
verbosity=self.verbose)
if args.multiGPU:
if args.distributed:
self.model = DDP(self.model,
device_ids=[args.gpu],
find_unused_parameters=True)
if self.verbose:
print(f'It took {time() - start:.1f}s')