def main():
parser = argparse.ArgumentParser("Test a trained model on SQuAD")
parent = argparse.ArgumentParser(add_help=False)
util.add_data_args(parent)
util.add_test_args(parent)
subparsers = parser.add_subparsers()
add_subparser("bidaf", "bidaf", subparsers, parent, bidaf_trainer)
add_subparser("glove_transformer", "bidaf", subparsers, parent,
glove_transformer_trainer)
add_subparser("roberta_finetune", "bpe", subparsers, parent,
roberta_finetune)
args = parser.parse_args()
# Require load_path for test.py
if not args.load_path:
raise argparse.ArgumentError("Missing required argument --load_path")
args.save_dir = util.get_save_dir(args.save_dir, args.name, training=False)
args.data_dir = util.get_data_dir(args.data_dir, args.data_sub_dir)
util.build_data_dir_path(args)
test = args.test
del args.test
test(args)
def main():
# define parser and arguments
args = get_train_test_args()
util.set_seed(args.seed)
device = torch.device('cuda') if torch.cuda.is_available() else torch.device('cpu')
DistilBert = DistilBertModel.from_pretrained('distilbert-base-uncased')
Experts = [DistilBertQA(DistilBertModel.from_pretrained('distilbert-base-uncased')).to(device) for _ in range(args.num_experts)]
tokenizer = DistilBertTokenizerFast.from_pretrained('distilbert-base-uncased')
gate_model = GateNetwork(384, 3,3, DistilBert.config).to(device)
print(f'Args: {json.dumps(vars(args), indent=4, sort_keys=True)}')
if args.do_train:
if not os.path.exists(args.save_dir):
os.makedirs(args.save_dir)
args.save_dir = util.get_save_dir(args.save_dir, args.run_name)
log = util.get_logger(args.save_dir, 'log_train')
log.info(f'Args: {json.dumps(vars(args), indent=4, sort_keys=True)}')
log.info("Preparing Training Data...")
args.device = device
trainer = train.Trainer(args, log)
train_dataset, _ = get_dataset(args, args.train_datasets, args.train_dir, tokenizer, 'train')
log.info("Preparing Validation Data...")
val_dataset, val_dict = get_dataset(args, args.train_datasets, args.val_dir, tokenizer, 'val')
train_loader = DataLoader(train_dataset,
batch_size=args.batch_size,
sampler=RandomSampler(train_dataset))
val_loader = DataLoader(val_dataset,
batch_size=1,
sampler=SequentialSampler(val_dataset))
best_scores = trainer.train(Experts, gate_model, train_loader, val_loader, val_dict, args.num_experts)
if args.do_eval:
split_name = 'test' if 'test' in args.eval_dir else 'validation'
log = util.get_logger(args.save_dir, f'log_{split_name}')
trainer = train.Trainer(args, log)
# load model
restore_model("",args.num_experts, Experts, gate_model)
eval_dataset, eval_dict = get_dataset(args, args.eval_datasets, args.eval_dir, tokenizer, split_name)
eval_loader = DataLoader(eval_dataset,
batch_size=1,
sampler=SequentialSampler(eval_dataset))
args.device = device
eval_preds, eval_scores = trainer.evaluate(Experts, gate_model, eval_loader,
eval_dict, return_preds=True,
split=split_name)
results_str = ', '.join(f'{k}: {v:05.2f}' for k, v in eval_scores.items())
log.info(f'Eval {results_str}')
# Write submission file
sub_path = os.path.join(args.save_dir, split_name + '_' + args.sub_file)
log.info(f'Writing submission file to {sub_path}...')
with open(sub_path, 'w', newline='', encoding='utf-8') as csv_fh:
csv_writer = csv.writer(csv_fh, delimiter=',')
csv_writer.writerow(['Id', 'Predicted'])
for uuid in sorted(eval_preds):
csv_writer.writerow([uuid, eval_preds[uuid]])
def save_video(n_frame, type_, exp=None):
save_dir = get_save_dir(exp)
save_path = os.path.join(save_dir, '{}.avi'.format(type_))
video_writer = cv2.VideoWriter(save_path,
cv2.VideoWriter_fourcc('M', 'J', 'P', 'G'),
25, (352, 288))
for i in range(n_frame):
img_name = './result/{}_{}.png'.format(type_, i)
f = cv2.imread(img_name)
os.remove(img_name)
video_writer.write(f)
video_writer.release()
def test(args):
args.save_dir = util.get_save_dir(args.save_dir, args.name, training=False)
log = util.get_logger(args.save_dir, args.name)
log.info('saving to directory {}'.format(args.save_dir))
tbx = SummaryWriter(args.save_dir)
if args.gpu_ids == 'cpu':
device, args.gpu_ids = torch.device('cpu'), []
else:
device, args.gpu_ids = util.get_available_devices()
log.info('testing on device {} with gpu_id {}'.format(str(device), str(args.gpu_ids)))
log.info('Building model...')
if args.task == 'tag':
model = SummarizerLinear()
# model = SummarizerLinearAttended(128, 256)
# model = SummarizerRNN(128, 256)
else:
model = SummarizerAbstractive(128, 256, device)
if len(args.gpu_ids) > 0:
model = nn.DataParallel(model, args.gpu_ids)
if args.load_path:
log.info('Loading checkpoint from {}...'.format(args.load_path))
model, step = util.load_model(model, args.load_path, args.gpu_ids)
else:
raise Exception('no specified checkpoint, abort')
model = model.to(device)
model.eval()
log.info('Building dataset...')
data_path = PROCESSED_DATA_SUPER_TINY if 'super_tiny' in args.split else PROCESSED_DATA
with open(data_path, 'rb') as f:
all_data = pickle.load(f)
if 'tiny' in args.split:
test_split = all_data['tiny']
elif 'dev' in args.split:
test_split = all_data['dev']
else:
test_split = all_data['test']
test_dataset = SummarizationDataset(
test_split['X'], test_split['y'], test_split['gold'])
collate_fn = tag_collate_fn if args.task == 'tag' else decode_collate_fn
test_loader = data.DataLoader(test_dataset,
batch_size=args.batch_size,
num_workers=args.num_workers,
shuffle=False,
collate_fn=collate_fn)
# Evaluate
log.info('Evaluating at step {}...'.format(step))
results, pred = evaluate(args, model, test_loader, device)
if results is None:
log.info('Selected predicted no select for all in batch')
raise Exception('no results found')
return results, pred
def main():
torch.autograd.set_detect_anomaly(True)
# define parser and arguments
args = get_train_test_args()
util.set_seed(args.seed)
tokenizer = DistilBertTokenizerFast.from_pretrained('distilbert-base-uncased')
if not os.path.exists(args.save_dir):
os.makedirs(args.save_dir)
if args.model_dir == 'save/xxx':
args.save_dir = util.get_save_dir(args.save_dir, args.run_name)
else:
args.save_dir = args.model_dir
if args.train_adv:
trainer = AdvTrainer(args)
else:
trainer = Trainer(args)
print("Preparing Training Data...")
train_datasets = []
for dataset_idx, train_dataset_name in enumerate(args.train_datasets.split(",")):
train_dataset, _ = get_dataset(args, train_dataset_name, args.train_dir, tokenizer, 'train', dataset_idx)
train_datasets.append(train_dataset)
train_loader = DataLoader(ConcatDataset(train_datasets),
batch_size=args.batch_size,
shuffle=True)
print("Preparing ind Validation Data...")
ind_val_dataset, ind_val_dict = get_dataset(args, args.train_datasets.split(","), args.ind_val_dir, tokenizer, 'ind_val')
ind_val_loader = DataLoader(ind_val_dataset,
batch_size=args.batch_size,
shuffle=False)
print("Preparing ood Validation Data...")
ood_val_loaders = []
ood_val_dicts = []
ood_val_names = []
for ood_val_dataset_name in args.ood_val_datasets.split(","):
ood_val_dataset, ood_val_dict = get_dataset(args, ood_val_dataset_name, args.ood_val_dir, tokenizer, 'ood_val')
ood_val_loader = DataLoader(ood_val_dataset,
batch_size=args.batch_size,
shuffle=False)
ood_val_loaders.append(ood_val_loader)
ood_val_dicts.append(ood_val_dict)
ood_val_names.append(ood_val_dataset_name)
trainer.train(train_loader, ind_val_loader, ind_val_dict, ood_val_loaders, ood_val_dicts, ood_val_names, args.resume_iters)
def main(args):
import pandas as pd
experiment_save_dir = util.get_save_dir(args.save_dir,
args.name,
args.dataset,
mode="hyper")
training_function = create_training_function(args, experiment_save_dir)
grid_search = GridSearch.load(args.experiments_file)
results = [
training_function(experiment, config) for experiment, config in
grid_search.random_experiments(args.max_experiments)
]
df = pd.DataFrame(results)
df.to_csv("hyperparams.csv")
print("Best config: ", df.sort_values('F1', ascending=False).iloc[0])
def predict(args, cw_idxs, qn_idxs):
args.save_dir = util.get_save_dir(args.save_dir, args.name, training=False)
log = util.get_logger(args.save_dir, args.name)
log.info('Args: {}'.format(dumps(vars(args), indent=4, sort_keys=True)))
device, gpu_ids = util.get_available_devices()
args.batch_size *= max(1, len(gpu_ids))
# Get embeddings
log.info('Loading embeddings...')
word_vectors = util.torch_from_json(args.word_emb_file)
# Get model
log.info('Building model...')
model = BiDAF(word_vectors=word_vectors, hidden_size=args.hidden_size)
model = nn.DataParallel(model, gpu_ids)
log.info('Loading checkpoint from {}...'.format(args.load_path))
model = util.load_model(model, args.load_path, gpu_ids, return_step=False)
model = model.to(device)
model.eval()
y_pred = model(cw_idxs, qn_idxs)
return y_pred
def main():
parser = argparse.ArgumentParser("Train a model on SQuAD")
parent = argparse.ArgumentParser(add_help=False)
util.add_data_args(parent)
util.add_train_args(parent)
subparsers = parser.add_subparsers()
add_subparser("bidaf", "bidaf", subparsers, parent, bidaf_trainer)
add_subparser("glove_transformer", "bidaf", subparsers, parent,
glove_transformer_trainer)
add_subparser("roberta_pretrain", "bpe", subparsers, parent,
roberta_pretrainer)
add_subparser("electra_pretrain", "bpe", subparsers, parent,
electra_pretrainer)
add_subparser("didae_pretrain", "bpe", subparsers, parent,
didae_pretrainer)
add_subparser("roberta_finetune", "bpe", subparsers, parent,
roberta_finetune)
add_subparser("roberta_augment", "bpe", subparsers, parent,
roberta_augment)
args = parser.parse_args()
if args.metric_name.startswith("NLL"):
# Best checkpoint is the one that minimizes negative log-likelihood
args.maximize_metric = False
elif args.metric_name in ("EM",
"F1") or args.metric_name.startswith("acc"):
# Best checkpoint is the one that maximizes EM or F1
args.maximize_metric = True
else:
raise ValueError(f'Unrecognized metric name: "{args.metric_name}"')
args.save_dir = util.get_save_dir(args.save_dir, args.name, training=True)
args.data_dir = util.get_data_dir(args.data_dir, args.data_sub_dir)
util.build_data_dir_path(args)
train = args.train
del args.train
train(args)
def main(args):
# Set up logging and devices
args.save_dir = util.get_save_dir(args.save_dir, args.name, training=True)
log = util.get_logger(args.save_dir, args.name)
tbx = SummaryWriter(args.save_dir)
device, args.gpu_ids = util.get_available_devices()
log.info(f'Args: {dumps(vars(args), indent=4, sort_keys=True)}')
args.batch_size *= max(1, len(args.gpu_ids))
# Set random seed
log.info(f'Using random seed {args.seed}...')
random.seed(args.seed)
np.random.seed(args.seed)
torch.manual_seed(args.seed)
torch.cuda.manual_seed_all(args.seed)
# Load embeddings
log.info('Loading embeddings...')
word_vectors = util.torch_from_json(args.word_emb_file)
# Build QA model
log.info('Building model...')
model = QA_Model(word_vectors=word_vectors,
hidden_size=args.hidden_size,
drop_prob=args.drop_prob,
attention_type=args.attention_type,
train_embeddings=args.train_embeddings)
model = nn.DataParallel(model, args.gpu_ids)
if args.load_path:
# Load QA model from file
log.info(f'Loading checkpoint from {args.load_path}...')
model, step = util.load_model(model, args.load_path, args.gpu_ids)
else:
step = 0
model = model.to(device)
model.train()
# Get saver
saver = util.CheckpointSaver(args.save_dir,
max_checkpoints=args.max_checkpoints,
metric_name=args.metric_name,
maximize_metric=args.maximize_metric,
log=log)
# Get optimizer
optimizer = optim.Adadelta(model.parameters(),
args.lr,
weight_decay=args.l2_wd)
#optimizer = optim.Adam(model.parameters(), lr=args.lr)
# Get data loader
log.info('Building dataset...')
train_dataset = SQuAD(args.train_record_file, args.use_squad_v2)
train_loader = data.DataLoader(train_dataset,
batch_size=args.batch_size,
shuffle=True,
num_workers=args.num_workers,
collate_fn=collate_fn)
dev_dataset = SQuAD(args.dev_record_file, args.use_squad_v2)
dev_loader = data.DataLoader(dev_dataset,
batch_size=args.batch_size,
shuffle=False,
num_workers=args.num_workers,
collate_fn=collate_fn)
# Train
log.info('Training...')
steps_till_eval = args.eval_steps
epoch = step // len(train_dataset)
while epoch != args.num_epochs:
epoch += 1
log.info(f'Starting epoch {epoch}...')
with torch.enable_grad(), \
tqdm(total=len(train_loader.dataset)) as progress_bar:
for cw_idxs, cc_idxs, qw_idxs, qc_idxs, y1, y2, ids in train_loader:
# Setup for forward
cw_idxs = cw_idxs.to(device)
qw_idxs = qw_idxs.to(device)
batch_size = cw_idxs.size(0)
optimizer.zero_grad()
# Forward
log_p1, log_p2 = model(cw_idxs, qw_idxs)
y1, y2 = y1.to(device), y2.to(device)
loss = F.nll_loss(log_p1, y1) + F.nll_loss(log_p2, y2)
loss_val = loss.item()
# Backward
loss.backward()
nn.utils.clip_grad_norm_(model.parameters(),
args.max_grad_norm)
optimizer.step()
# Log info
step += batch_size
progress_bar.update(batch_size)
progress_bar.set_postfix(epoch=epoch, NLL=loss_val)
steps_till_eval -= batch_size
if steps_till_eval <= 0:
steps_till_eval = args.eval_steps
# Evaluate and save checkpoint
log.info(f'Evaluating at step {step}...')
results, pred_dict = evaluate(model, dev_loader, device,
args.dev_eval_file,
args.max_ans_len,
args.use_squad_v2)
saver.save(step, model, results[args.metric_name], device)
# Log to console
results_str = ', '.join(f'{k}: {v:05.2f}'
for k, v in results.items())
log.info(f'Dev {results_str}')
def main(args):
# Set up logging and devices
args.save_dir = util.get_save_dir(args.save_dir, args.name, training=True)
log = util.get_logger(args.save_dir, args.name)
tbx = SummaryWriter(args.save_dir)
device, args.gpu_ids = util.get_available_devices()
log.info('Args: {}'.format(dumps(vars(args), indent=4, sort_keys=True)))
args.batch_size *= max(1, len(args.gpu_ids))
# Set random seed
log.info('Using random seed {}...'.format(args.seed))
random.seed(args.seed)
np.random.seed(args.seed)
torch.manual_seed(args.seed)
torch.cuda.manual_seed_all(args.seed)
# Get embeddings
log.info('Loading embeddings...')
word_vectors = util.torch_from_json(args.word_emb_file)
char_vectors = util.torch_from_json(args.char_emb_file)
# ######################################
# tokenizer = BertTokenizer.from_pretrained('bert-large-uncased', do_lower_case=True)
# train_examples = None
# train_examples = read_squad_examples(
# input_file=args.train_file, is_training=True, version_2_with_negative=args.version_2_with_negative)
# train_features = convert_examples_to_features(
# examples=train_examples,
# tokenizer=tokenizer,
# max_seq_length=args.max_seq_length,
# doc_stride=args.doc_stride,
# max_query_length=args.max_query_length,
# is_training=True)
# if args.local_rank == -1 or torch.distributed.get_rank() == 0:
# logger.info(" Saving train features into cached file %s", cached_train_features_file)
# with open(cached_train_features_file, "wb") as writer:
# pickle.dump(train_features, writer)
# all_input_ids = torch.tensor([f.input_ids for f in train_features], dtype=torch.long)
# x = all_input_ids
###########################################
# Get model
log.info('Building model...')
model = BiDAF(word_vectors=word_vectors,
char_vectors=char_vectors,
hidden_size=args.hidden_size,
drop_prob=args.drop_prob)
model = nn.DataParallel(model, args.gpu_ids)
if args.load_path:
log.info('Loading checkpoint from {}...'.format(args.load_path))
model, step = util.load_model(model, args.load_path, args.gpu_ids)
else:
step = 0
model = model.to(device)
model.train()
ema = util.EMA(model, args.ema_decay)
# Get saver
saver = util.CheckpointSaver(args.save_dir,
max_checkpoints=args.max_checkpoints,
metric_name=args.metric_name,
maximize_metric=args.maximize_metric,
log=log)
# Get optimizer and scheduler
optimizer = optim.Adadelta(model.parameters(),
args.lr,
weight_decay=args.l2_wd)
scheduler = sched.LambdaLR(optimizer, lambda s: 1.) # Constant LR
# Get data loader
log.info('Building dataset...')
train_dataset = SQuAD(args.train_record_file, args.use_squad_v2)
train_loader = data.DataLoader(train_dataset,
batch_size=args.batch_size,
shuffle=True,
num_workers=args.num_workers,
collate_fn=collate_fn)
dev_dataset = SQuAD(args.dev_record_file, args.use_squad_v2)
dev_loader = data.DataLoader(dev_dataset,
batch_size=args.batch_size,
shuffle=False,
num_workers=args.num_workers,
collate_fn=collate_fn)
# Train
log.info('Training...')
steps_till_eval = args.eval_steps
epoch = step // len(train_dataset)
while epoch != args.num_epochs:
epoch += 1
log.info('Starting epoch {}...'.format(epoch))
with torch.enable_grad(), \
tqdm(total=len(train_loader.dataset)) as progress_bar:
for cw_idxs, cc_idxs, qw_idxs, qc_idxs, y1, y2, ids in train_loader:
# Setup for forward
cw_idxs = cw_idxs.to(device)
qw_idxs = qw_idxs.to(device)
batch_size = cw_idxs.size(0)
# added_flag
cc_idxs = cc_idxs.to(device)
qc_idxs = qc_idxs.to(device)
optimizer.zero_grad()
# Forward
# log_p1, log_p2 = model(cw_idxs, qw_idxs)
log_p1, log_p2 = model(cw_idxs, qw_idxs, cc_idxs, qc_idxs)
y1, y2 = y1.to(device), y2.to(device)
loss = F.nll_loss(log_p1, y1) + F.nll_loss(log_p2, y2)
loss_val = loss.item()
# Backward
loss.backward()
nn.utils.clip_grad_norm_(model.parameters(),
args.max_grad_norm)
optimizer.step()
scheduler.step(step // batch_size)
ema(model, step // batch_size)
# Log info
step += batch_size
progress_bar.update(batch_size)
progress_bar.set_postfix(epoch=epoch, NLL=loss_val)
tbx.add_scalar('train/NLL', loss_val, step)
tbx.add_scalar('train/LR', optimizer.param_groups[0]['lr'],
step)
steps_till_eval -= batch_size
if steps_till_eval <= 0:
steps_till_eval = args.eval_steps
# Evaluate and save checkpoint
log.info('Evaluating at step {}...'.format(step))
ema.assign(model)
results, pred_dict = evaluate(model, dev_loader, device,
args.dev_eval_file,
args.max_ans_len,
args.use_squad_v2)
saver.save(step, model, results[args.metric_name], device)
ema.resume(model)
# Log to console
results_str = ', '.join('{}: {:05.2f}'.format(k, v)
for k, v in results.items())
log.info('Dev {}'.format(results_str))
# Log to TensorBoard
log.info('Visualizing in TensorBoard...')
for k, v in results.items():
tbx.add_scalar('dev/{}'.format(k), v, step)
util.visualize(tbx,
pred_dict=pred_dict,
eval_path=args.dev_eval_file,
step=step,
split='dev',
num_visuals=args.num_visuals)
def main():
torch.backends.cudnn.enabled = False
# Make save dir for logfiles and state_dicts
run_name = "optflow_nvidia"
save_path = osp.join("save", run_name)
save_dir = util.get_save_dir(save_path, run_name,
training=True) # unique save dir
log = util.get_logger(save_dir, run_name) # logger
saver = util.CheckpointSaver(
save_dir, # save model
max_checkpoints=10,
maximize_metric=False,
metric_name="MSE",
log=log)
# Data, batches & epochs
max_epoch = 25
batch_size = 32
window = 7
train_loader, val_loader = create_datasplit(batch_size=batch_size,
window=window)
# Model Creation
log.info("Building model")
# model = resnet18(sample_size=(240, 640), sample_duration=2*window+1, shortcut_type="A", num_classes=1)
# model = C3D()
model = nvidia()
device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")
model = nn.DataParallel(
model
) # If loading state dict throws KeyError ie. unexpected keys "module.convX.X"
model.to(device)
model.train()
# Possibly load model state_dict here
step = 0
# load_path = 'c3d.pickle'
# log.info('Loading checkpoint from {}...'.format(load_path))
# model.load_state_dict(torch.load(load_path))
# model = util.load_model(model, load_path, 0) # uses the saved step num
# Loss & Optimizer
criterion = nn.MSELoss()
lr = 1e-4
weight_decay = 1e-5
# optimizer = optim.SGD(model.parameters(), lr=lr, momentum=.9, weight_decay=weight_decay)
optimizer = optim.Adam(model.parameters(),
lr=lr,
weight_decay=weight_decay)
# Log args
# log.info('Args: {}'.format(dumps(vars({"lr": lr, "max_epoch": max_epoch, "batch_size": batch_size,
# "window_size": window}), indent=4, sort_keys=True)))
# Initialize epoch and step_to_eval counters
steps_till_eval = int(.4 * len(train_loader.dataset))
epoch = step // len(train_loader.dataset)
while epoch != max_epoch:
epoch += 1
log.info("=============Epoch %i=============" % epoch)
with torch.enable_grad(), tqdm(
total=len(train_loader.dataset)) as progress_bar:
for sample_batch in train_loader:
model.zero_grad() # Zero gradients after each batch
x, y = sample_batch
x = x.to(device)
y = y.to(device)
f = model(x)
loss = criterion(f, y)
loss.backward()
optimizer.step()
step += batch_size
progress_bar.update(batch_size)
progress_bar.set_postfix(epoch=epoch, MSE=loss.item())
steps_till_eval -= batch_size
if steps_till_eval <= 0:
steps_till_eval = int(.4 * len(train_loader.dataset))
# Eval on validation set
val_mse = evalu(model, val_loader, device)
# Save checkpoint
saver.save(step, model, val_mse, device)
# Print to console
results_str = "MSE: " + str(val_mse)
log.info('Dev {}'.format(results_str))
def main(args):
# Set up logging
args.save_dir = util.get_save_dir(args.save_dir, args.name, training=False)
log = util.get_logger(args.save_dir, args.name)
log.info(f'Args: {dumps(vars(args), indent=4, sort_keys=True)}')
device, gpu_ids = util.get_available_devices()
args.batch_size *= max(1, len(gpu_ids))
seed = 42
torch.manual_seed(seed)
torch.backends.cudnn.benchmark = False
torch.backends.cudnn.deterministic = True
# Get model
#log.info(f'Loading checkpoint from {args.load_path}...')
model = resnet.resnet50()
model = nn.DataParallel(model, gpu_ids)
#log.info(f'Loading checkpoint from {args.load_path}...')
#model = util.load_model(model, args.load_path, gpu_ids, return_step=False)
model = model.to(device)
model.eval()
# Get data loader
log.info('loading dataset...')
input_data_file = '/home/mahbub/research/flat-resnet/data/dev_images.pt'
#vars(args)[f'{args.input_data_file}']
dataset = ImageDataset(input_data_file)
data_loader = data.DataLoader(dataset,
batch_size=args.batch_size,
shuffle=False,
num_workers=args.num_workers,
collate_fn=None)
#class_label_file = '/home/mahbub/research/flat-resnet/imagenet_classes.txt'
# Read the categories
#with open(class_label_file, "r") as f:
# categories = [s.strip() for s in f.readlines()]
# Evaluate
log.info(f'Running inference ...')
output = torch.zeros(
len(dataset), 1000
) # TODO: 1000 is number of class or resnet output size, remove hard coding.
out_idx = 0
with torch.no_grad(), \
tqdm(total=len(dataset)) as progress_bar:
for images in data_loader:
# Setup for forward
images = images.to(device)
batch_size = images.shape[0]
#print ("batch size is {}".format(batch_size))
#print("Input is : {}".format(images[0,0,0,:10]))
# Forward
output[out_idx:out_idx + batch_size] = model(images)
out_idx += batch_size
#print("output shape is {}".format(output.shape))
#print("Output is: {}".format(output))
#probabilities = torch.nn.functional.softmax(output, dim=1)
#print("probabilities shape is {}".format(probabilities.shape))
#print ("probabilities sum = {}".format(probabilities.sum(axis=1)))
# Show top categories per image
#K = 5
#top_prob, top_catid = torch.topk(probabilities, K)
#print("top catid shape is {}".format(top_catid.shape))
#for i in range(top_prob.shape[0]):
# for k in range(K):
# print(categories[top_catid[i,k]], top_prob[i,k].item())
# Log info
progress_bar.update(batch_size)
# Write output to a file
torch.save(output, "resnet50_output")
def main():
#save_dir = util.get_save_dir('save','vgglinear', training=False)
#log = util.get_logger(save_dir, 'vgglinear')
save_dir = util.get_save_dir('save', 'TimeCNN', training=False)
log = util.get_logger(save_dir, 'TimeCNN')
device, gpu_ids = util.get_available_devices()
tbx = SummaryWriter(save_dir)
'save/train/TimeCNN-wd0.01-epoch100-01/best.pth.tar'
#path = 'save/train/Resnet-82/best.pth.tar'
#path = 'save/train/TimeCNN-epoch30-1024-01/best.pth.tar'
#path = 'save/train/vgglinear-02/best.pth.tar'
#build model here
log.info("Building model")
#model = Baseline(8 * 96 * 64)
model = TimeCNN()
#model = Resnet()
#model = VGGLinear()
model = nn.DataParallel(model, gpu_ids)
model = util.load_model(model, path, gpu_ids, return_step=False)
model = model.to(device)
model = model.double()
model.eval()
log.info("Building Dataset")
test_dataset = Shots("videos/test.h5py", "labels/test.npy")
test_loader = data.DataLoader(test_dataset,
batch_size=BATCH_SIZE,
shuffle=False,
num_workers=4,
collate_fn=collate_fn)
num_correct = 0
num_samples = 0
missed_1, missed_0 = 0, 0
num_1_predicted = 0
num_0_predicted = 0
with torch.no_grad():
for frames, y in test_loader:
frames = frames.to(device)
y = y.to(device)
scores = model(frames)
loss = F.cross_entropy(scores, y)
_, preds = scores.max(1)
num_correct += (preds == y).sum()
# This accumulates how many 1's and 0's were misclassified
for i in range(y.shape[0]):
if y[i] == 1 and preds[i] == 0:
missed_1 += 1
elif y[i] == 0 and preds[i] == 1:
missed_0 += 1
num_samples += preds.shape[0]
num_1_predicted += (preds == 1).sum()
num_0_predicted += (preds == 0).sum()
acc = float(num_correct) / num_samples
log.info("Path: {}".format(path))
log.info("Accuracy on test set is {}".format(acc))
log.info("Missed 1's: {}, Missed 0's: {}".format(missed_1, missed_0))
log.info("Number 1's predicted: {}".format(num_1_predicted))
log.info("Number 0's predicted: {}".format(num_0_predicted))
log.info('-----------------')
log.info("Best Accuracy on test set is {} and path was {}".format(
best_accuracy, best_path))
def main(args):
print("in main")
print("args: ", args)
if True:
args.save_dir = util.get_save_dir(args.save_dir, args.name, training=True)
log = util.get_logger(args.save_dir, args.name)
tbx = SummaryWriter(args.save_dir)
device, args.gpu_ids = util.get_available_devices()
log.info('Args: {}'.format(dumps(vars(args), indent=4, sort_keys=True)))
args.batch_size *= max(1, len(args.gpu_ids))
# Set random seed
log.info('Using random seed {}...'.format(args.seed))
random.seed(args.seed)
np.random.seed(args.seed)
torch.manual_seed(args.seed)
torch.cuda.manual_seed_all(args.seed)
# Get embeddings
log.info('Loading embeddings...')
# CHECK IF WE NEED TO USE ALL OF THESE????
word_vectors = util.torch_from_json(args.word_emb_file)
# Get model
log.info('Building model...')
model = BiDAF(word_vectors=word_vectors,
hidden_size=args.hidden_size,
drop_prob=args.drop_prob)
model = nn.DataParallel(model, args.gpu_ids)
if args.load_path:
log.info('Loading checkpoint from {}...'.format(args.load_path))
model, step = util.load_model(model, args.load_path, args.gpu_ids)
else:
step = 0
model = model.to(device)
model.train()
ema = util.EMA(model, args.ema_decay)
# Get saver
saver = util.CheckpointSaver(args.save_dir,
max_checkpoints=args.max_checkpoints,
metric_name=args.metric_name,
maximize_metric=args.maximize_metric,
log=log)
# Get optimizer and scheduler
optimizer = optim.Adadelta(model.parameters(), args.lr,
weight_decay=args.l2_wd)
scheduler = sched.LambdaLR(optimizer, lambda s: 1.) # Constant LR
# Get data loader
log.info('Building dataset...')
train_dataset = SQuAD(args.train_record_file, args.use_squad_v2)
print("train dataset!: ", train_dataset)
train_loader = data.DataLoader(train_dataset,
batch_size=args.batch_size,
shuffle=True,
num_workers=args.num_workers,
collate_fn=collate_fn)
dev_dataset = SQuAD(args.dev_record_file, args.use_squad_v2)
dev_loader = data.DataLoader(dev_dataset,
batch_size=args.batch_size,
shuffle=False,
num_workers=args.num_workers,
collate_fn=collate_fn)
# Train
log.info('Training...')
steps_till_eval = args.eval_steps
epoch = step // len(train_dataset)
while epoch != args.num_epochs:
epoch += 1
log.info('Starting epoch {}...'.format(epoch))
with torch.enable_grad(), \
tqdm(total=len(train_loader.dataset)) as progress_bar:
for cw_idxs, cc_idxs, qw_idxs, qc_idxs, y1, y2, ids in train_loader:
# Setup for forward
cw_idxs = cw_idxs.to(device)
qw_idxs = qw_idxs.to(device)
batch_size = cw_idxs.size(0)
optimizer.zero_grad()
# Forward
log_p1, log_p2 = model(cw_idxs, qw_idxs)
y1, y2 = y1.to(device), y2.to(device)
loss = F.nll_loss(log_p1, y1) + F.nll_loss(log_p2, y2)
loss_val = loss.item()
# Backward
loss.backward()
nn.utils.clip_grad_norm_(model.parameters(), args.max_grad_norm)
optimizer.step()
scheduler.step(step // batch_size)
ema(model, step // batch_size)
# Log info
step += batch_size
progress_bar.update(batch_size)
progress_bar.set_postfix(epoch=epoch,
NLL=loss_val)
tbx.add_scalar('train/NLL', loss_val, step)
tbx.add_scalar('train/LR',
optimizer.param_groups[0]['lr'],
step)
steps_till_eval -= batch_size
if steps_till_eval <= 0:
steps_till_eval = args.eval_steps
# Evaluate and save checkpoint
log.info('Evaluating at step {}...'.format(step))
ema.assign(model)
results, pred_dict = evaluate(model, dev_loader, device,
args.dev_eval_file,
args.max_ans_len,
args.use_squad_v2)
saver.save(step, model, results[args.metric_name], device)
ema.resume(model)
# Log to console
results_str = ', '.join('{}: {:05.2f}'.format(k, v)
for k, v in results.items())
log.info('Dev {}'.format(results_str))
# Log to TensorBoard
log.info('Visualizing in TensorBoard...')
for k, v in results.items():
tbx.add_scalar('dev/{}'.format(k), v, step)
util.visualize(tbx,
pred_dict=pred_dict,
eval_path=args.dev_eval_file,
step=step,
split='dev',
num_visuals=args.num_visuals)
def main(args):
# Set up logging and devices
args.save_dir = util.get_save_dir(args.save_dir, args.name, training=True)
log = util.get_logger(args.save_dir, args.name)
tbx = SummaryWriter(args.save_dir)
device, args.gpu_ids = util.get_available_devices()
log.info(f'Args: {dumps(vars(args), indent=4, sort_keys=True)}')
args.batch_size *= max(1, len(args.gpu_ids))
# Set random seed
log.info(f'Using random seed {args.seed}...')
random.seed(args.seed)
np.random.seed(args.seed)
torch.manual_seed(args.seed)
torch.cuda.manual_seed_all(args.seed)
# Get embeddings
log.info('Loading embeddings...')
word_vectors = util.torch_from_json(args.word_emb_file)
# Get model
log.info('Building model...')
model = BiDAF(word_vectors=word_vectors,
hidden_size=args.hidden_size,
drop_prob=args.drop_prob)
model = nn.DataParallel(model, args.gpu_ids)
if args.load_path:
log.info(f'Loading checkpoint from {args.load_path}...')
model, step = util.load_model(model, args.load_path, args.gpu_ids)
else:
step = 0
model = model.to(device)
model.train()
ema = util.EMA(model, args.ema_decay)
# Get saver
saver = util.CheckpointSaver(args.save_dir,
max_checkpoints=args.max_checkpoints,
metric_name=args.metric_name,
maximize_metric=args.maximize_metric,
log=log)
# Get optimizer and scheduler
optimizer = optim.Adadelta(model.parameters(),
args.lr,
weight_decay=args.l2_wd)
scheduler = sched.LambdaLR(optimizer, lambda s: 1.) # Constant LR
# Get data loader
log.info('Building dataset...')
train_dataset = SQuAD(args.train_record_file, args.use_squad_v2)
train_loader = data.DataLoader(train_dataset,
batch_size=args.batch_size,
shuffle=True,
num_workers=args.num_workers,
collate_fn=collate_fn)
dev_dataset = SQuAD(args.dev_record_file, args.use_squad_v2)
dev_loader = data.DataLoader(dev_dataset,
batch_size=args.batch_size,
shuffle=False,
num_workers=args.num_workers,
collate_fn=collate_fn)
# Train
log.info('Training...')
steps_till_eval = args.eval_steps
epoch = step // len(train_dataset)
while epoch != args.num_epochs:
epoch += 1
log.info(f'Starting epoch {epoch}...')
for cw_idxs, cc_idxs, qw_idxs, qc_idxs, y1, y2, ids in train_loader:
# Setup for forward
cw_idxs = cw_idxs.to(device)
qw_idxs = qw_idxs.to(device)
batch_size = cw_idxs.size(0)
optimizer.zero_grad()
# Forward
log_p1, log_p2 = model(cw_idxs, qw_idxs)
y1, y2 = y1.to(device), y2.to(device)
loss = F.nll_loss(log_p1, y1) + F.nll_loss(log_p2, y2)
loss_val = loss.item()
# Backward
loss.backward()
nn.utils.clip_grad_norm_(model.parameters(), args.max_grad_norm)
optimizer.step()
scheduler.step(step // batch_size)
ema(model, step // batch_size)
# Log info
step += batch_size
if step % 1000 == 0 and step > 0:
log.info(f'Step {step}: training loss {loss_val}...')
steps_till_eval -= batch_size
if steps_till_eval <= 0:
steps_till_eval = args.eval_steps
# Evaluate and save checkpoint
log.info(f'Evaluating at step {step}...')
ema.assign(model)
results, pred_dict = evaluate(model, dev_loader, device,
args.dev_eval_file,
args.max_ans_len,
args.use_squad_v2)
saver.save(step, model, results[args.metric_name], device)
ema.resume(model)
# Log to console
results_str = ', '.join(f'{k}: {v:05.2f}'
for k, v in results.items())
log.info(f'Dev {results_str}')
# Log to TensorBoard
log.info('Visualizing in TensorBoard...')
for k, v in results.items():
tbx.add_scalar(f'dev/{k}', v, step)
util.visualize(tbx,
pred_dict=pred_dict,
eval_path=args.dev_eval_file,
step=step,
split='dev',
num_visuals=args.num_visuals)
def main(args):
# Set up logging
args.save_dir = util.get_save_dir(args.save_dir, args.name, training=False)
log = util.get_logger(args.save_dir, args.name)
log.info(f'Args: {dumps(vars(args), indent=4, sort_keys=True)}')
device, gpu_ids = util.get_available_devices()
args.batch_size *= max(1, len(gpu_ids))
# Get embeddings
log.info('Loading embeddings...')
word_vectors = util.torch_from_json(args.word_emb_file)
ch_vectors = util.torch_from_json(args.char_emb_file)
# Get model
log.info('Building model...')
model = BiDAF(word_vectors=word_vectors,
ch_vectors=ch_vectors,
hidden_size=args.hidden_size)
model = nn.DataParallel(model, gpu_ids)
log.info(f'Loading checkpoint from {args.load_path}...')
model = util.load_model(model, args.load_path, gpu_ids, return_step=False)
model = model.to(device)
model.eval()
# Get data loader
log.info('Building dataset...')
record_file = vars(args)[f'{args.split}_record_file']
dataset = SQuAD(record_file, args.use_squad_v2)
data_loader = data.DataLoader(dataset,
batch_size=args.batch_size,
shuffle=False,
num_workers=args.num_workers,
collate_fn=collate_fn)
# Evaluate
log.info(f'Evaluating on {args.split} split...')
nll_meter = util.AverageMeter()
pred_dict = {} # Predictions for TensorBoard
sub_dict = {} # Predictions for submission
eval_file = vars(args)[f'{args.split}_eval_file']
with open(eval_file, 'r') as fh:
gold_dict = json_load(fh)
with torch.no_grad(), \
tqdm(total=len(dataset)) as progress_bar:
for cw_idxs, cc_idxs, qw_idxs, qc_idxs, y1, y2, ids in data_loader:
# Setup for forward
cw_idxs = cw_idxs.to(device)
qw_idxs = qw_idxs.to(device)
cc_idxs = cc_idxs.to(device)
qc_idxs = qc_idxs.to(device)
batch_size = cw_idxs.size(0)
# Forward
log_p1, log_p2 = model(cw_idxs, qw_idxs, cc_idxs, qc_idxs)
y1, y2 = y1.to(device), y2.to(device)
loss = F.nll_loss(log_p1, y1) + F.nll_loss(log_p2, y2)
nll_meter.update(loss.item(), batch_size)
# Get F1 and EM scores
p1, p2 = log_p1.exp(), log_p2.exp()
starts, ends = util.discretize(p1, p2, args.max_ans_len,
args.use_squad_v2)
# Log info
progress_bar.update(batch_size)
if args.split != 'test':
# No labels for the test set, so NLL would be invalid
progress_bar.set_postfix(NLL=nll_meter.avg)
idx2pred, uuid2pred = util.convert_tokens(gold_dict, ids.tolist(),
starts.tolist(),
ends.tolist(),
args.use_squad_v2)
pred_dict.update(idx2pred)
sub_dict.update(uuid2pred)
# Log results (except for test set, since it does not come with labels)
if args.split != 'test':
results = util.eval_dicts(gold_dict, pred_dict, args.use_squad_v2)
results_list = [('NLL', nll_meter.avg), ('F1', results['F1']),
('EM', results['EM'])]
if args.use_squad_v2:
results_list.append(('AvNA', results['AvNA']))
results = OrderedDict(results_list)
# Log to console
results_str = ', '.join(f'{k}: {v:05.2f}' for k, v in results.items())
log.info(f'{args.split.title()} {results_str}')
# Log to TensorBoard
tbx = SummaryWriter(args.save_dir)
util.visualize(tbx,
pred_dict=pred_dict,
eval_path=eval_file,
step=0,
split=args.split,
num_visuals=args.num_visuals)
# Write submission file
sub_path = join(args.save_dir, args.split + '_' + args.sub_file)
log.info(f'Writing submission file to {sub_path}...')
with open(sub_path, 'w', newline='', encoding='utf-8') as csv_fh:
csv_writer = csv.writer(csv_fh, delimiter=',')
csv_writer.writerow(['Id', 'Predicted'])
for uuid in sorted(sub_dict):
csv_writer.writerow([uuid, sub_dict[uuid]])
def main(args):
# Set up logging
args.save_dir = util.get_save_dir(args.save_dir, args.name, training=False)
log = util.get_logger(args.save_dir, args.name)
log.info(f'Args: {dumps(vars(args), indent=4, sort_keys=True)}')
device, gpu_ids = util.get_available_devices()
args.batch_size *= max(1, len(gpu_ids))
# Get embeddings
log.info('Loading embeddings...')
word_vectors = util.torch_from_json(args.word_emb_file)
# Get model
log.info('Building model...')
if args.model == 'bidaf':
model = BiDAF(word_vectors=word_vectors, hidden_size=args.hidden_size)
elif args.model == 'bidafextra':
model = BiDAFExtra(word_vectors=word_vectors, args=args)
elif args.model == 'fusionnet':
model = FusionNet(word_vectors=word_vectors, args=args)
model = nn.DataParallel(model, gpu_ids)
log.info(f'Loading checkpoint from {args.load_path}...')
model = util.load_model(model, args.load_path, gpu_ids, return_step=False)
model = model.to(device)
model.eval()
# Get data loader
log.info('Building dataset...')
record_file = vars(args)[f'{args.split}_record_file']
dataset = SQuAD(record_file, args)
data_loader = data.DataLoader(dataset,
batch_size=args.batch_size,
shuffle=False,
num_workers=args.num_workers,
collate_fn=collate_fn)
# print("*"*80)
# print(len(dataset.question_idxs))
# for question_idx in dataset.question_idxs:
# print(question_idx)
# print("*" * 80)
# print(self.question_idxs[question_idx])
# self.question_idxs[idx]
# print("data_loader: ",data_loader)
# Evaluate
log.info(f'Evaluating on {args.split} split...')
nll_meter = util.AverageMeter()
pred_dict = {} # Predictions for TensorBoard
sub_dict = {} # Predictions for submission
eval_file = vars(args)[f'{args.split}_eval_file']
with open(eval_file, 'r') as fh:
gold_dict = json_load(fh)
# create statistics
# print("*"*80)
# print(len(gold_dict))
# print(gold_dict['1']['question'])
count_questions_type = defaultdict(lambda: 0)
audit_trail_from_question_type = defaultdict(lambda: [])
list_of_interrogative_pronouns = [
"what", "whose", "why", "which", "where", "when", "how", "who", "whom"
]
for index in range(1, len(gold_dict)):
# transform the question in lower case to simplify the analysis, thus losing the benefit of the capital letters
# possibly indicating the position of the interrogative pronoun in the sentence.
question_lower_case = gold_dict[str(index)]['question'].lower()
list_question_lower_case_with_punctuation = question_lower_case.translate(
{ord(i): " "
for i in "'"}).split()
#
question_lower_case = []
for item in list_question_lower_case_with_punctuation:
question_lower_case.append(
item.translate({ord(i): ""
for i in ",.<>!@£$%^&*()_-+=?"}))
# defining a variable for the first word
first_word_question_lower_case = question_lower_case[0]
# defining variable for the second word
second_word_question_lower_case = question_lower_case[1]
# defining variable for the first and second word
combined_first_and_second_words = first_word_question_lower_case + " " + second_word_question_lower_case
#printing on the screen test for debugging purpose
# Analyzing the sentence
if first_word_question_lower_case in list_of_interrogative_pronouns:
count_questions_type[first_word_question_lower_case] += 1
audit_trail_from_question_type[
first_word_question_lower_case].append(str(index))
# composed question starting by in
elif first_word_question_lower_case == "in":
if second_word_question_lower_case in list_of_interrogative_pronouns and second_word_question_lower_case != "whose":
count_questions_type[combined_first_and_second_words] += 1
audit_trail_from_question_type[
combined_first_and_second_words].append(str(index))
else:
pronoun = find_first_interrogative_pronoun(
list_of_interrogative_pronouns, question_lower_case)
count_questions_type[pronoun] += 1
audit_trail_from_question_type[pronoun].append(str(index))
# composed question starting by by
elif first_word_question_lower_case == "by":
if second_word_question_lower_case in list_of_interrogative_pronouns \
and second_word_question_lower_case !="whom"\
and second_word_question_lower_case !="which"\
and second_word_question_lower_case !="when"\
and second_word_question_lower_case !="how":
count_questions_type[combined_first_and_second_words] += 1
audit_trail_from_question_type[
combined_first_and_second_words].append(str(index))
else:
pronoun = find_first_interrogative_pronoun(
list_of_interrogative_pronouns, question_lower_case)
count_questions_type[pronoun] += 1
audit_trail_from_question_type[pronoun].append(str(index))
else:
pronoun = find_first_interrogative_pronoun(
list_of_interrogative_pronouns, question_lower_case)
#if pronoun =="":
# print(">>", question_lower_case)
# print("@@@", gold_dict[str(index)]['question'])
count_questions_type[pronoun] += 1
audit_trail_from_question_type[pronoun].append(str(index))
# if pronoun =="":
# print(">>", question_lower_case.split())
# print()
#if first_word_question_lower_case == "if":
# print(">>", question_lower_case.split())
# print(count_questions_type)
# if gold_dict[str(index)]['question'].lower().split()[0] == "in":
# print(gold_dict[str(index)]['question'])
reverse_dict_by_value = OrderedDict(
sorted(count_questions_type.items(), key=lambda x: x[1]))
# print(count_questions_type)
total_questions = sum(count_questions_type.values())
# print(reverse_dict)
#for k, v in reverse_dict_by_value.items():
# print( "%s: %s and in percentage: %s" % (k, v, 100*v/total_questions))
#print(audit_trail_from_question_type)
# exit()
with torch.no_grad(), \
tqdm(total=len(dataset)) as progress_bar:
for cw_idxs, cc_idxs, qw_idxs, qc_idxs, cw_pos, cw_ner, cw_freq, cqw_extra, y1, y2, ids in data_loader:
# Setup for forward
cw_idxs = cw_idxs.to(device)
qw_idxs = qw_idxs.to(device)
batch_size = cw_idxs.size(0)
# Forward
if args.model == 'bidaf':
log_p1, log_p2 = model(cw_idxs, qw_idxs)
else:
log_p1, log_p2 = model(cw_idxs, qw_idxs, cw_pos, cw_ner,
cw_freq, cqw_extra)
y1, y2 = y1.to(device), y2.to(device)
loss = F.nll_loss(log_p1, y1) + F.nll_loss(log_p2, y2)
nll_meter.update(loss.item(), batch_size)
# Get F1 and EM scores
p1, p2 = log_p1.exp(), log_p2.exp()
starts, ends = util.discretize(p1, p2, args.max_ans_len,
args.use_squad_v2)
# Log info
progress_bar.update(batch_size)
if args.split != 'test':
# No labels for the test set, so NLL would be invalid
progress_bar.set_postfix(NLL=nll_meter.avg)
idx2pred, uuid2pred = util.convert_tokens(gold_dict, ids.tolist(),
starts.tolist(),
ends.tolist(),
args.use_squad_v2)
pred_dict.update(idx2pred)
sub_dict.update(uuid2pred)
# Log results (except for test set, since it does not come with labels)
if args.split != 'test':
results = util.eval_dicts(gold_dict, pred_dict, args.use_squad_v2)
# Printing information for questions without interrogative pronouns
""""
print("len(gold_dict): ", len(gold_dict))
print("len(pred_dict): ", len(pred_dict))
print("Is gold_dict.keys() identical to pred_dict.keys(): ", gold_dict.keys()==pred_dict.keys())
if gold_dict.keys()!=pred_dict.keys():
for key in gold_dict.keys():
if key not in pred_dict.keys():
print("key ", key, " missing in pred_dict.keys(")
"""
results_list = [('NLL', nll_meter.avg), ('F1', results['F1']),
('EM', results['EM'])]
if args.use_squad_v2:
results_list.append(('AvNA', results['AvNA']))
results = OrderedDict(results_list)
# Computing the F1 score for each type of question
#
# audit_trail_from_question_type[pronoun].append(str(index))
# create a list of the types of questions by extracting the keys from the dict audit_trail_from_question_type
types_of_questions = list(audit_trail_from_question_type.keys())
gold_dict_per_type_of_questions = defaultdict(lambda: [])
pred_dict_per_type_of_questions = {}
gold_dict_per_type_of_questions_start = {}
pred_dict_per_type_of_questions_start = {}
gold_dict_per_type_of_questions_middle = {}
pred_dict_per_type_of_questions_middle = {}
gold_dict_per_type_of_questions_end = {}
pred_dict_per_type_of_questions_end = {}
for type_of_questions in types_of_questions:
#gold_pred = {key: value for key, value in gold_dict.items() if key in audit_trail_from_question_type[type_of_questions]}
#lst_pred = {key: value for key, value in pred_dict.items() if key in audit_trail_from_question_type[type_of_questions]}
# Create two dictionnaries for each type of sentence for gold_dict_per_type_of_questions and pred_dict_per_type_of_questions
gold_dict_per_type_of_questions[type_of_questions] = {
key: value
for key, value in gold_dict.items()
if key in audit_trail_from_question_type[type_of_questions]
and key in pred_dict.keys()
}
pred_dict_per_type_of_questions[type_of_questions] = {
key: value
for key, value in pred_dict.items()
if key in audit_trail_from_question_type[type_of_questions]
and key in pred_dict.keys()
}
# print(type_of_questions," F1 score: ", util.eval_dicts(gold_dict_per_type_of_questions[type_of_questions], pred_dict_per_type_of_questions[type_of_questions], args.use_squad_v2)['F1'])
gold_dict_per_type_of_questions_start[type_of_questions] = {
key: value
for key, value in gold_dict.items()
if key in audit_trail_from_question_type[type_of_questions]
and key in pred_dict.keys()
}
pred_dict_per_type_of_questions_start[type_of_questions] = {
key: value
for key, value in pred_dict.items()
if key in audit_trail_from_question_type[type_of_questions]
and key in pred_dict.keys()
}
gold_dict_per_type_of_questions_middle[type_of_questions] = {
key: value
for key, value in gold_dict.items()
if key in audit_trail_from_question_type[type_of_questions]
and key in pred_dict.keys()
}
pred_dict_per_type_of_questions_middle[type_of_questions] = {
key: value
for key, value in pred_dict.items()
if key in audit_trail_from_question_type[type_of_questions]
and key in pred_dict.keys()
}
gold_dict_per_type_of_questions_end[type_of_questions] = {
key: value
for key, value in gold_dict.items()
if key in audit_trail_from_question_type[type_of_questions]
and key in pred_dict.keys()
}
pred_dict_per_type_of_questions_end[type_of_questions] = {
key: value
for key, value in pred_dict.items()
if key in audit_trail_from_question_type[type_of_questions]
and key in pred_dict.keys()
}
for key, value in gold_dict.items():
#if key in audit_trail_from_question_type[type_of_questions] and key in pred_dict.keys():
if key in audit_trail_from_question_type[
type_of_questions] and type_of_questions != "" and key in pred_dict_per_type_of_questions[
type_of_questions]:
"""
print("type_of_questions: ",type_of_questions)
print("key: ", key)
print("question: ", value["question"])
sub_index = value["question"].lower().find(type_of_questions)
print("sub_index: ",sub_index)
test_fc = value["question"].lower().find(type_of_questions)
print("present type of the var: ",type(test_fc))
#print("question: ", value["question"][str(key)])
print("length of the question: ", len(value["question"]))
print('Position of the interrogative pronoun in the question:', )
"""
# Create two dictionnaries for each type of sentence based at the start of the sentence
if value["question"].lower().find(
type_of_questions) == 1 or value["question"].lower(
).find(type_of_questions) == 0:
#print("BEGINNING")
if type_of_questions != "":
try:
del gold_dict_per_type_of_questions_middle[
type_of_questions][key]
except KeyError:
pass
try:
del pred_dict_per_type_of_questions_middle[
type_of_questions][key]
except KeyError:
pass
try:
del gold_dict_per_type_of_questions_end[
type_of_questions][key]
except KeyError:
pass
try:
del pred_dict_per_type_of_questions_end[
type_of_questions][key]
except KeyError:
pass
#pred_dict_per_type_of_questions_start[type_of_questions] = {key: pred_dict[key] for key in
# gold_dict_per_type_of_questions_start[
# type_of_questions].keys()}
elif value["question"].lower(
).find(type_of_questions) >= len(
value["question"]) - len(type_of_questions) - 5:
#print("END")
if type_of_questions != "":
try:
del gold_dict_per_type_of_questions_middle[
type_of_questions][key]
except KeyError:
pass
try:
del pred_dict_per_type_of_questions_middle[
type_of_questions][key]
except KeyError:
pass
try:
del gold_dict_per_type_of_questions_start[
type_of_questions][key]
except KeyError:
pass
try:
del pred_dict_per_type_of_questions_start[
type_of_questions][key]
except KeyError:
pass
#print("type_of_questions: ",type_of_questions)
#sub_index = value["question"].lower().find(type_of_questions)
#print("sub_index: ", sub_index)
#print("len(value['question']) - len(type_of_questions) - 2: ", len(value["question"])-len(type_of_questions)-2)
#start_string = len(value["question"])-len(type_of_questions)-6
#end_string = len(value["question"])-1
#print("extract at the end: ", value["question"][start_string:end_string])
else:
#print("MIDDLE")
if type_of_questions != "":
try:
del gold_dict_per_type_of_questions_start[
type_of_questions][key]
except KeyError:
pass
try:
del pred_dict_per_type_of_questions_start[
type_of_questions][key]
except KeyError:
pass
try:
del gold_dict_per_type_of_questions_end[
type_of_questions][key]
except KeyError:
pass
try:
del pred_dict_per_type_of_questions_end[
type_of_questions][key]
except KeyError:
pass
pass
"""
if type_of_questions != "":
gold_dict_per_type_of_questions_start[type_of_questions] = {key: value for key, value in gold_dict.items() if (key in audit_trail_from_question_type[type_of_questions] \
and (value["question"].lower().find(type_of_questions) <= 1) \
and key in pred_dict_per_type_of_questions[type_of_questions]) }
"""
"""
for key in gold_dict_per_type_of_questions_start[type_of_questions].keys():
print("key:: ", key )
print("type(key):: ", type(key) )
print("pred_dict[,key,] : ", pred_dict[key])
print("@@@@@@@@@@@@@@@@@@@@@@@@")
pred_dict_per_type_of_questions_start[type_of_questions] = {key: pred_dict[key] for key in gold_dict_per_type_of_questions_start[type_of_questions].keys()}
#pred_dict_per_type_of_questions_start[type_of_questions] = {key: value for key, value in pred_dict.items() if key in list(gold_dict_per_type_of_questions_start[type_of_questions].keys()) }
# Create two dictionnaries for each type of sentence based at the end of the sentence
gold_dict_per_type_of_questions_end[type_of_questions] = {key: value for key, value in gold_dict.items() if key in audit_trail_from_question_type[type_of_questions] \
and value["question"].lower().find(type_of_questions) >= len(value["question"])-len(type_of_questions)-2 \
and key in pred_dict_per_type_of_questions[type_of_questions]}
pred_dict_per_type_of_questions_end[type_of_questions] = {key: pred_dict[key] for key in list(gold_dict_per_type_of_questions_end[type_of_questions].keys())}
#print("*"*80)
# Create two dictionnaries for each type of sentence based at the middle of the sentencecount_questions_type
gold_dict_per_type_of_questions_middle[type_of_questions] = {key: value for key, value in gold_dict.items() if key not in list(gold_dict_per_type_of_questions_start[type_of_questions].keys()) \
and key not in list(gold_dict_per_type_of_questions_end[type_of_questions].keys())}
pred_dict_per_type_of_questions_middle[type_of_questions] = {key: pred_dict[key] for key in list(gold_dict_per_type_of_questions_end[type_of_questions].keys())}
else:
gold_dict_per_type_of_questions_start[""] = gold_dict_per_type_of_questions[""]
pred_dict_per_type_of_questions_start[""] = pred_dict_per_type_of_questions[""]
gold_dict_per_type_of_questions_end[""] = gold_dict_per_type_of_questions[""]
pred_dict_per_type_of_questions_end[""] = pred_dict_per_type_of_questions[""]
gold_dict_per_type_of_questions_middle[""] = gold_dict_per_type_of_questions[""]
pred_dict_per_type_of_questions_middle[""] = pred_dict_per_type_of_questions[""]
"""
positions_in_question = ["beginning", "middle", "end"]
# print(type_of_questions," F1 score: ", util.eval_dicts(gold_dict_per_type_of_questions[type_of_questions], pred_dict_per_type_of_questions[type_of_questions], args.use_squad_v2)['F1'])
list_beginning = [
util.eval_dicts(
gold_dict_per_type_of_questions_start[type_of_questions],
pred_dict_per_type_of_questions_start[type_of_questions],
args.use_squad_v2)['F1']
for type_of_questions in types_of_questions
]
list_middle = [
util.eval_dicts(
gold_dict_per_type_of_questions_middle[type_of_questions],
pred_dict_per_type_of_questions_middle[type_of_questions],
args.use_squad_v2)['F1']
for type_of_questions in types_of_questions
]
list_end = [
util.eval_dicts(
gold_dict_per_type_of_questions_end[type_of_questions],
pred_dict_per_type_of_questions_end[type_of_questions],
args.use_squad_v2)['F1']
for type_of_questions in types_of_questions
]
#for type_of_questions in types_of_questions:
# print("gold_dict_per_type_of_questions_start[type_of_questions]: ",gold_dict_per_type_of_questions_start[type_of_questions])
# print("pred_dict_per_type_of_questions[type_of_questions]: ",pred_dict_per_type_of_questions[type_of_questions])
F1 = np.array([list_beginning, list_middle, list_end])
m, n = F1.shape
value_to_ignore = []
for i in range(m):
for j in range(n):
if F1[i, j] == "NA" or F1[i, j] == 0:
value_to_ignore.append((i, j))
print("value to ignore: ", value_to_ignore)
#F1 = np.array([[0, 0, 0, 0, 0.1, 0, 0, 0, 0, 0, 0, 0, 0],
# [0, 0, 0, 0, 0.1, 0, 0, 0, 0, 0, 0, 0, 0],
# [0, 0, 0, 0, 0.1, 0, 0, 0, 0, 0, 0, 0, 0]])
data_label = copy.deepcopy(F1)
for row in data_label:
for column_idx in range(len(row)):
if row[column_idx] == "NA":
row[column_idx] = ""
# print question without interrogative pronoun required for the second part of the analysis:
for key, value in gold_dict.items():
if key in audit_trail_from_question_type[
""] and key in pred_dict.keys():
print("question: ", gold_dict_per_type_of_questions[''])
print("golden answers: ", )
print("prediction: ", pred_dict[key])
print()
fig, ax = plt.subplots()
types_of_questions[types_of_questions.index(
"")] = "Implicit question without interrogative pronoun"
im, cbar = heatmap(F1, positions_in_question, types_of_questions, ax=ax, \
cmap="YlGn", cbarlabel="F1 scores")
texts = annotate_heatmap(im,
data=data_label,
valfmt="{x:.1f}",
ignore=value_to_ignore)
fig.tight_layout()
plt.show()
# Log to console
results_str = ', '.join(f'{k}: {v:05.2f}' for k, v in results.items())
log.info(f'{args.split.title()} {results_str}')
# Log to TensorBoard
tbx = SummaryWriter(args.save_dir)
util.visualize(tbx,
pred_dict=pred_dict,
eval_path=eval_file,
step=0,
split=args.split,
num_visuals=args.num_visuals)
# Write submission file
sub_path = join(args.save_dir, args.split + '_' + args.sub_file)
log.info(f'Writing submission file to {sub_path}...')
with open(sub_path, 'w', newline='', encoding='utf-8') as csv_fh:
csv_writer = csv.writer(csv_fh, delimiter=',')
csv_writer.writerow(['Id', 'Predicted'])
for uuid in sorted(sub_dict):
csv_writer.writerow([uuid, sub_dict[uuid]])
def main(args):
if args.large:
args.train_record_file += '_large'
args.dev_eval_file += '_large'
model_name = "albert-xlarge-v2"
else:
model_name = "albert-base-v2"
if args.xxlarge:
args.train_record_file += '_xxlarge'
args.dev_eval_file += '_xxlarge'
model_name = "albert-xxlarge-v2"
# Set up logging and devices
args.save_dir = util.get_save_dir(args.save_dir, args.name, training=True)
log = util.get_logger(args.save_dir, args.name)
tbx = SummaryWriter(args.save_dir)
device, args.gpu_ids = util.get_available_devices()
log.info(f'Args: {dumps(vars(args), indent=4, sort_keys=True)}')
args.batch_size *= max(1, len(args.gpu_ids))
# Set random seed
log.info(f'Using random seed {args.seed}...')
random.seed(args.seed)
np.random.seed(args.seed)
torch.manual_seed(args.seed)
torch.cuda.manual_seed_all(args.seed)
# Get model
log.info('Building model...')
if args.bidaf:
char_vectors = util.torch_from_json(args.char_emb_file)
if args.model_name == 'albert_highway':
model = models.albert_highway(model_name)
elif args.model_name == 'albert_lstm_highway':
model = models.LSTM_highway(model_name, hidden_size=args.hidden_size)
elif args.model_name == 'albert_bidaf':
model = models.BiDAF(char_vectors=char_vectors,
hidden_size=args.hidden_size,
drop_prob=args.drop_prob)
elif args.model_name == 'albert_bidaf2':
model = models.BiDAF2(model_name=model_name,
char_vectors=char_vectors,
hidden_size=args.hidden_size,
drop_prob=args.drop_prob)
else:
model = AlbertForQuestionAnswering.from_pretrained(args.model_name)
model = nn.DataParallel(model, args.gpu_ids)
if args.load_path:
log.info(f'Loading checkpoint from {args.load_path}...')
model, step = util.load_model(model, args.load_path, args.gpu_ids)
else:
step = 0
model = model.to(device)
model.train()
ema = util.EMA(model, args.ema_decay)
# Get saver
saver = util.CheckpointSaver(args.save_dir,
max_checkpoints=args.max_checkpoints,
metric_name=args.metric_name,
maximize_metric=args.maximize_metric,
log=log)
# Get optimizer and scheduler
optimizer = AdamW(model.parameters(), lr=args.lr, weight_decay=args.l2_wd)
scheduler = sched.LambdaLR(optimizer, lambda s: 1.) # Constant LR
# Get data loader
log.info('Building dataset...')
train_dataset = SQuAD(args.train_record_file, args.use_squad_v2,
args.bidaf)
train_loader = data.DataLoader(train_dataset,
batch_size=args.batch_size,
shuffle=True,
num_workers=args.num_workers)
dev_dataset = SQuAD(args.dev_eval_file, args.use_squad_v2, args.bidaf)
dev_loader = data.DataLoader(dev_dataset,
batch_size=args.batch_size,
shuffle=False,
num_workers=args.num_workers)
with open(args.dev_gold_file) as f:
gold_dict = json.load(f)
tokenizer = AlbertTokenizer.from_pretrained(model_name)
# Train
log.info('Training...')
steps_till_eval = args.eval_steps
epoch = step // len(train_dataset)
while epoch != args.num_epochs:
epoch += 1
log.info(f'Starting epoch {epoch}...')
with torch.enable_grad(), \
tqdm(total=len(train_loader.dataset)) as progress_bar:
for batch in train_loader:
batch = tuple(t.to(device) for t in batch)
inputs = {
"input_ids": batch[0],
"attention_mask": batch[1],
"token_type_ids": batch[2],
'start_positions': batch[3],
'end_positions': batch[4],
}
if args.bidaf:
inputs['char_ids'] = batch[6]
y1 = batch[3]
y2 = batch[4]
# Setup for forward
batch_size = inputs["input_ids"].size(0)
optimizer.zero_grad()
# Forward
# log_p1, log_p2 = model(**inputs)
y1, y2 = y1.to(device), y2.to(device)
outputs = model(**inputs)
loss = outputs[0]
loss = loss.mean()
# loss_fct = nn.CrossEntropyLoss()
# loss = loss_fct(log_p1, y1) + loss_fct(log_p2, y2)
# loss = F.nll_loss(log_p1, y1) + F.nll_loss(log_p2, y2)
loss_val = loss.item()
# Backward
loss.backward()
nn.utils.clip_grad_norm_(model.parameters(),
args.max_grad_norm)
optimizer.step()
scheduler.step(step // batch_size)
ema(model, step // batch_size)
# Log info
step += batch_size
progress_bar.update(batch_size)
progress_bar.set_postfix(epoch=epoch, NLL=loss_val)
tbx.add_scalar('train/NLL', loss_val, step)
tbx.add_scalar('train/LR', optimizer.param_groups[0]['lr'],
step)
steps_till_eval -= batch_size
if steps_till_eval <= 0:
steps_till_eval = args.eval_steps
# Evaluate and save checkpoint
log.info(f'Evaluating at step {step}...')
ema.assign(model)
results, pred_dict = evaluate(args, model, dev_dataset,
dev_loader, gold_dict,
tokenizer, device,
args.max_ans_len,
args.use_squad_v2)
saver.save(step, model, results[args.metric_name], device)
ema.resume(model)
# Log to console
results_str = ', '.join(f'{k}: {v:05.2f}'
for k, v in results.items())
log.info(f'Dev {results_str}')
# Log to TensorBoard
log.info('Visualizing in TensorBoard...')
for k, v in results.items():
tbx.add_scalar(f'dev/{k}', v, step)
def main(course_dir,
text_embedding_size,
audio_embedding_size,
image_embedding_size,
hidden_size,
drop_prob,
max_text_length,
out_heatmaps_dir,
args,
batch_size=3,
num_epochs=100):
# Set up logging and devices
args.save_dir = util.get_save_dir(args.save_dir, args.name, training=True)
log = util.get_logger(args.save_dir, args.name)
tbx = SummaryWriter(args.save_dir)
device, args.gpu_ids = util.get_available_devices()
log.info(f'Args: {dumps(vars(args), indent=4, sort_keys=True)}')
args.batch_size *= max(1, len(args.gpu_ids))
# Set random seed
log.info(f'Using random seed {args.seed}...')
random.seed(args.seed)
np.random.seed(args.seed)
torch.manual_seed(args.seed)
torch.cuda.manual_seed_all(args.seed)
# Create Dataset objects
text_dataset = TextDataset(course_dir, max_text_length)
audio_dataset = AudioDataset(course_dir)
target_dataset = TargetDataset(course_dir)
# Preprocess the image in prescribed format
normalize = transforms.Normalize(mean=[0.485, 0.456, 0.406],
std=[0.229, 0.224, 0.225])
transform = transforms.Compose([
transforms.RandomResizedCrop(256),
transforms.RandomHorizontalFlip(),
transforms.ToTensor(),
normalize,
])
image_dataset = ImageDataset(course_dir, transform)
assert len(text_dataset) == len(audio_dataset) and len(
audio_dataset) == len(image_dataset) and len(image_dataset) == len(
target_dataset), "Unequal dataset lengths"
# Creating data indices for training and validation splits:
train_indices, val_indices = gen_train_val_indices(text_dataset)
# Creating PT data samplers and loaders:
train_sampler = torch.utils.data.SequentialSampler(train_indices)
val_sampler = torch.utils.data.SequentialSampler(val_indices)
# Get sentence embeddings
train_text_loader = torch.utils.data.DataLoader(text_dataset,
batch_size=batch_size,
shuffle=False,
num_workers=2,
collate_fn=collator,
sampler=train_sampler)
val_text_loader = torch.utils.data.DataLoader(text_dataset,
batch_size=batch_size,
shuffle=False,
num_workers=2,
collate_fn=collator,
sampler=val_sampler)
# Get Audio embeddings
train_audio_loader = torch.utils.data.DataLoader(audio_dataset,
batch_size=batch_size,
shuffle=False,
num_workers=2,
collate_fn=collator,
sampler=train_sampler)
val_audio_loader = torch.utils.data.DataLoader(audio_dataset,
batch_size=batch_size,
shuffle=False,
num_workers=2,
collate_fn=collator,
sampler=val_sampler)
# Get images
train_image_loader = torch.utils.data.DataLoader(image_dataset,
batch_size=batch_size,
shuffle=False,
num_workers=2,
collate_fn=collator,
sampler=train_sampler)
val_image_loader = torch.utils.data.DataLoader(image_dataset,
batch_size=batch_size,
shuffle=False,
num_workers=2,
collate_fn=collator,
sampler=val_sampler)
# Load Target text
train_target_loader = torch.utils.data.DataLoader(
target_dataset,
batch_size=batch_size,
shuffle=False,
num_workers=2,
collate_fn=target_collator,
sampler=train_sampler)
val_target_loader = torch.utils.data.DataLoader(target_dataset,
batch_size=batch_size,
shuffle=False,
num_workers=2,
collate_fn=target_collator,
sampler=val_sampler)
# print("lens - train_text_loader {}, val_text_loader {}".format(len(train_text_loader), len(val_text_loader)))
# print("lens - train_audio_loader {}, val_audio_loader {}".format(len(train_audio_loader), len(val_audio_loader)))
# print("lens - train_image_loader {}, val_image_loader {}".format(len(train_image_loader), len(val_image_loader)))
# print("lens - train_target_loader {}, val_target_loader {}".format(len(train_target_loader), len(val_target_loader)))
# Create model
model = MMBiDAF(hidden_size, text_embedding_size, audio_embedding_size,
image_embedding_size, device, drop_prob, max_text_length)
model = nn.DataParallel(model, args.gpu_ids)
if args.load_path:
log.info(f'Loading checkpoint from {args.load_path}...')
model, step = util.load_model(model, args.load_path, args.gpu_ids)
else:
step = 0
model = model.to(device)
model.train()
ema = util.EMA(model, args.ema_decay) # For exponential moving average
# Get saver
saver = util.CheckpointSaver(args.save_dir,
max_checkpoints=args.max_checkpoints,
metric_name=args.metric_name,
maximize_metric=args.maximize_metric,
log=log) # Need to change the metric name
# Get optimizer and scheduler
optimizer = optim.Adadelta(model.parameters(),
args.lr,
weight_decay=args.l2_wd)
scheduler = sched.LambdaLR(optimizer, lambda s: 1.) # Constant LR
# Let's do this!
loss = 0
eps = 1e-8
log.info("Training...")
steps_till_eval = args.eval_steps
epoch = step // len(TextDataset(course_dir, max_text_length))
while epoch != args.num_epochs:
epoch += 1
log.info("Starting epoch {epoch}...")
count_item = 0
loss_epoch = 0
with torch.enable_grad(), tqdm(
total=len(train_text_loader.dataset)) as progress_bar:
for (batch_text, original_text_lengths), (
batch_audio, original_audio_lengths), (
batch_images,
original_img_lengths), (batch_target_indices,
batch_source_paths,
batch_target_paths,
original_target_len) in zip(
train_text_loader,
train_audio_loader,
train_image_loader,
train_target_loader):
loss = 0
max_dec_len = torch.max(
original_target_len
) # TODO check error : max decoder timesteps for each batch
# Transfer tensors to GPU
batch_text = batch_text.to(device)
log.info("Loaded batch text")
batch_audio = batch_audio.to(device)
log.info("Loaded batch audio")
batch_images = batch_images.to(device)
log.info("Loaded batch image")
batch_target_indices = batch_target_indices.to(device)
log.info("Loaded batch targets")
# Setup for forward
batch_size = batch_text.size(0)
optimizer.zero_grad()
log.info("Starting forward pass")
# Forward
batch_out_distributions, loss = model(
batch_text, original_text_lengths, batch_audio,
original_audio_lengths, batch_images, original_img_lengths,
batch_target_indices, original_target_len, max_dec_len)
loss_val = loss.item() # numerical value of loss
loss_epoch = loss_epoch + loss_val
log.info("Starting backward")
# Backward
loss.backward()
nn.utils.clip_grad_norm_(
model.parameters(),
args.max_grad_norm) # To tackle exploding gradients
optimizer.step()
scheduler.step(step // batch_size)
ema(model, step // batch_size)
# Log info
step += batch_size
progress_bar.update(batch_size)
progress_bar.set_postfix(epoch=epoch, NLL=loss_val)
tbx.add_scalar('train/NL', loss_val, step)
tbx.add_scalar('train/LR', optimizer.param_groups[0]['lr'],
step)
steps_till_eval -= batch_size
if steps_till_eval <= 0:
steps_till_eval = args.eval_steps
# Evaluate and save checkpoint
log.info(f'Evaluating at step {step}...')
ema.assign(model)
# TODO
# scores, results = evaluate(model, dev_loader, device,
# args.dev_eval_file,
# args.max_ans_len,
# args.use_squad_v2)
saver.save(step, model, device)
ema.resume(model)
# Generate summary
print('Generated summary for iteration {}: '.format(epoch))
summaries = get_generated_summaries(batch_out_distributions,
original_text_lengths,
batch_source_paths)
print(summaries)
# Evaluation
# rouge = Rouge()
# rouge_scores = rouge.get_scores(batch_source_paths, batch_target_paths, avg=True)
# print('Rouge score at iteration {} is {}: '.format(epoch, rouge_scores))
# Generate Output Heatmaps
# sns.set()
# for idx in range(len(out_distributions)):
# out_distributions[idx] = out_distributions[idx].squeeze(0).detach().numpy() # Converting each timestep distribution to numpy array
# out_distributions = np.asarray(out_distributions) # Converting the timestep list to array
# ax = sns.heatmap(out_distributions)
# fig = ax.get_figure()
# fig.savefig(out_heatmaps_dir + str(epoch) + '.png')
print("Epoch loss is : {}".format(loss_epoch / count_item))
def main():
# define parser and arguments
args = get_train_test_args()
util.set_seed(args.seed)
model = DistilBertForQuestionAnswering.from_pretrained(
"distilbert-base-uncased")
tokenizer = DistilBertTokenizerFast.from_pretrained(
'distilbert-base-uncased')
if args.do_train:
if not os.path.exists(args.save_dir):
os.makedirs(args.save_dir)
if args.resume_training:
checkpoint_path = os.path.join(args.save_dir, 'checkpoint')
model = DistilBertForQuestionAnswering.from_pretrained(
checkpoint_path)
model.to(args.device)
else:
args.save_dir = util.get_save_dir(args.save_dir, args.run_name)
log = util.get_logger(args.save_dir, 'log_train')
log.info(f'Args: {json.dumps(vars(args), indent=4, sort_keys=True)}')
log.info("Preparing Training Data...")
args.device = torch.device(
'cuda') if torch.cuda.is_available() else torch.device('cpu')
trainer = Trainer(args, log)
train_dataset, _ = get_dataset(args, args.train_datasets,
args.train_dir, tokenizer, 'train',
args.outdomain_data_repeat)
log.info("Preparing Validation Data...")
val_dataset, val_dict = get_dataset(args, args.train_datasets,
args.val_dir, tokenizer, 'val',
args.outdomain_data_repeat)
train_loader = DataLoader(train_dataset,
batch_size=args.batch_size,
sampler=RandomSampler(train_dataset))
val_loader = DataLoader(val_dataset,
batch_size=args.batch_size,
sampler=RandomSampler(val_dataset))
best_scores = trainer.train(model, train_loader, val_loader, val_dict)
if args.do_eval:
args.device = torch.device(
'cuda') if torch.cuda.is_available() else torch.device('cpu')
split_name = 'test' if 'test' in args.eval_dir else 'validation'
log = util.get_logger(args.save_dir, f'log_{split_name}')
trainer = Trainer(args, log)
checkpoint_path = os.path.join(args.save_dir, 'checkpoint')
model = DistilBertForQuestionAnswering.from_pretrained(checkpoint_path)
discriminator_input_size = 768
if args.full_adv:
discriminator_input_size = 384 * 768
discriminator = DomainDiscriminator(
input_size=discriminator_input_size)
# discriminator.load_state_dict(torch.load(checkpoint_path + '/discriminator'))
model.to(args.device)
discriminator.to(args.device)
eval_dataset, eval_dict = get_dataset(args, args.eval_datasets,
args.eval_dir, tokenizer,
split_name,
args.outdomain_data_repeat)
eval_loader = DataLoader(eval_dataset,
batch_size=args.batch_size,
sampler=SequentialSampler(eval_dataset))
eval_preds, eval_scores = trainer.evaluate(model,
discriminator,
eval_loader,
eval_dict,
return_preds=True,
split=split_name)
results_str = ', '.join(f'{k}: {v:05.2f}'
for k, v in eval_scores.items())
log.info(f'Eval {results_str}')
# Write submission file
sub_path = os.path.join(args.save_dir,
split_name + '_' + args.sub_file)
log.info(f'Writing submission file to {sub_path}...')
with open(sub_path, 'w', newline='', encoding='utf-8') as csv_fh:
csv_writer = csv.writer(csv_fh, delimiter=',')
csv_writer.writerow(['Id', 'Predicted'])
for uuid in sorted(eval_preds):
csv_writer.writerow([uuid, eval_preds[uuid]])
def main():
args = get_bert_args()
assert not (args.do_output
and args.do_train), 'Don\'t output and train at the same time!'
if args.do_output:
sub_dir_prefix = 'output'
elif args.do_train:
sub_dir_prefix = 'train'
else:
sub_dir_prefix = 'test'
args.save_dir = util.get_save_dir(args.save_dir, args.name, sub_dir_prefix)
args.output_dir = args.save_dir
global logger
logger = util.get_logger(args.save_dir, args.name)
if args.doc_stride >= args.max_seq_length - args.max_query_length:
logger.warning(
"WARNING - You've set a doc stride which may be superior to the document length in some "
"examples. This could result in errors when building features from the examples. Please reduce the doc "
"stride or increase the maximum length to ensure the features are correctly built."
)
if not args.evaluate_during_saving and args.save_best_only:
raise ValueError("No best result without evaluation during saving")
# Use util.get_save_dir, comment this for now
# if (
# os.path.exists(args.output_dir)
# and os.listdir(args.output_dir)
# and args.do_train
# and not args.overwrite_output_dir
# ):
# raise ValueError(
# "Output directory ({}) already exists and is not empty. Use --overwrite_output_dir to overcome.".format(
# args.output_dir
# )
# )
# Setup distant debugging if needed
if args.server_ip and args.server_port:
# Distant debugging - see https://code.visualstudio.com/docs/python/debugging#_attach-to-a-local-script
import ptvsd
print("Waiting for debugger attach")
ptvsd.enable_attach(address=(args.server_ip, args.server_port),
redirect_output=True)
ptvsd.wait_for_attach()
# Setup CUDA, GPU & distributed training
if args.local_rank == -1 or args.no_cuda:
device = torch.device("cuda" if torch.cuda.is_available()
and not args.no_cuda else "cpu")
args.n_gpu = torch.cuda.device_count()
else: # Initializes the distributed backend which will take care of sychronizing nodes/GPUs
torch.cuda.set_device(args.local_rank)
device = torch.device("cuda", args.local_rank)
torch.distributed.init_process_group(backend="nccl")
args.n_gpu = 1
args.device = device
# Setup logging
# logging.basicConfig(
# format="%(asctime)s - %(levelname)s - %(name)s - %(message)s",
# datefmt="%m/%d/%Y %H:%M:%S",
# level=logging.INFO if args.local_rank in [-1, 0] else logging.WARN,
# )
logger.warning(
"Process rank: %s, device: %s, n_gpu: %s, distributed training: %s, 16-bits training: %s",
args.local_rank,
device,
args.n_gpu,
bool(args.local_rank != -1),
args.fp16,
)
# Set seed
set_seed(args)
# Load pretrained model and tokenizer
if args.local_rank not in [-1, 0]:
# Make sure only the first process in distributed training will download model & vocab
torch.distributed.barrier()
args.model_type = args.model_type.lower()
config_class, model_class, tokenizer_class = MODEL_CLASSES[args.model_type]
config = config_class.from_pretrained(
args.config_name if args.config_name else args.model_name_or_path,
cache_dir=args.cache_dir if args.cache_dir else None,
)
tokenizer = tokenizer_class.from_pretrained(
args.tokenizer_name
if args.tokenizer_name else args.model_name_or_path,
do_lower_case=args.do_lower_case,
cache_dir=args.cache_dir if args.cache_dir else None,
)
model = model_class.from_pretrained(
args.model_name_or_path,
from_tf=bool(".ckpt" in args.model_name_or_path),
config=config,
cache_dir=args.cache_dir if args.cache_dir else None,
)
if args.local_rank == 0:
# Make sure only the first process in distributed training will download model & vocab
torch.distributed.barrier()
model.to(args.device)
logger.info("Training/evaluation parameters %s", args)
# Before we do anything with models, we want to ensure that we get fp16 execution of torch.einsum if args.fp16 is set.
# Otherwise it'll default to "promote" mode, and we'll get fp32 operations. Note that running `--fp16_opt_level="O2"` will
# remove the need for this code, but it is still valid.
if args.fp16:
try:
import apex
apex.amp.register_half_function(torch, "einsum")
except ImportError:
raise ImportError(
"Please install apex from https://www.github.com/nvidia/apex to use fp16 training."
)
# Training
if args.do_train:
train_dataset = load_and_cache_examples(args,
tokenizer,
evaluate=False,
output_examples=False)
global_step, tr_loss = train(args, train_dataset, model, tokenizer)
logger.info(" global_step = %s, average loss = %s", global_step,
tr_loss)
# Save the trained model and the tokenizer
if args.do_train and (args.local_rank == -1
or torch.distributed.get_rank() == 0):
# Create output directory if needed
if not os.path.exists(args.output_dir) and args.local_rank in [-1, 0]:
os.makedirs(args.output_dir)
logger.info("Saving model checkpoint to %s", args.output_dir)
# Save a trained model, configuration and tokenizer using `save_pretrained()`.
# They can then be reloaded using `from_pretrained()`
# Take care of distributed/parallel training
model_to_save = model.module if hasattr(model, "module") else model
model_to_save.save_pretrained(args.output_dir)
tokenizer.save_pretrained(args.output_dir)
# Good practice: save your training arguments together with the trained model
torch.save(args, os.path.join(args.output_dir, "training_args.bin"))
# Load a trained model and vocabulary that you have fine-tuned
model = model_class.from_pretrained(
args.output_dir) # , force_download=True)
tokenizer = tokenizer_class.from_pretrained(
args.output_dir, do_lower_case=args.do_lower_case)
model.to(args.device)
# Evaluation - we can ask to evaluate all the checkpoints (sub-directories) in a directory
results = {}
if args.do_eval and args.local_rank in [-1, 0]:
if args.do_train:
logger.info(
"Loading checkpoints saved during training for evaluation")
checkpoints = [args.output_dir]
if args.eval_all_checkpoints:
checkpoints = list(
os.path.dirname(c) for c in sorted(
glob.glob(args.output_dir + "/**/" + WEIGHTS_NAME,
recursive=True)))
# logging.getLogger("transformers.modeling_utils").setLevel(logging.WARN) # Reduce model loading logs
else:
logger.info("Loading checkpoint %s for evaluation",
args.model_name_or_path)
checkpoints = [args.model_name_or_path]
logger.info("Evaluate the following checkpoints: %s", checkpoints)
for checkpoint in checkpoints:
# Reload the model
global_step = checkpoint.split(
"-")[-1] if len(checkpoints) > 1 else ""
model = model_class.from_pretrained(
checkpoint) # , force_download=True)
model.to(args.device)
# Evaluate
result = evaluate(args,
model,
tokenizer,
prefix=global_step,
save_dir=args.output_dir,
save_log_path=os.path.join(
checkpoint, 'eval_result.json'))
result = dict(
(k + ("_{}".format(global_step) if global_step else ""), v)
for k, v in result.items())
results.update(result)
logger.info(
f'Convert format and Writing submission file to directory {args.output_dir}...'
)
util.convert_submission_format_and_save(
args.output_dir,
prediction_file_path=os.path.join(args.output_dir,
'predictions_.json'))
logger.info("Results: {}".format(results))
# Generate output
if args.do_output and args.local_rank in [-1, 0]:
if args.do_train:
logger.info("Loading checkpoints saved during training for output")
checkpoints = [args.output_dir]
if args.eval_all_checkpoints:
checkpoints = list(
os.path.dirname(c) for c in sorted(
glob.glob(args.output_dir + "/**/" + WEIGHTS_NAME,
recursive=True)))
# logging.getLogger("transformers.modeling_utils").setLevel(logging.WARN) # Reduce model loading logs
else:
logger.info("Loading checkpoint %s for output",
args.model_name_or_path)
checkpoints = [args.model_name_or_path]
logger.info("Output the following checkpoints: %s", checkpoints)
for checkpoint in checkpoints:
# Reload the model
global_step = checkpoint.split(
"-")[-1] if len(checkpoints) > 1 else ""
model = model_class.from_pretrained(
checkpoint) # , force_download=True)
model.to(args.device)
generate_model_outputs(args,
model,
tokenizer,
is_dev=True,
prefix=global_step,
save_dir=args.output_dir)
return results
def train(args):
args.save_dir = util.get_save_dir(args.save_dir, args.name, training=True)
log = util.get_logger(args.save_dir, args.name)
tbx = SummaryWriter(args.save_dir)
if args.gpu_ids == 'cpu':
device, args.gpu_ids = torch.device('cpu'), []
else:
device, args.gpu_ids = util.get_available_devices()
log.info('training on device {} with gpu_id {}'.format(str(device), str(args.gpu_ids)))
# Set random seed
log.info('Using random seed {}...'.format(args.seed))
random.seed(args.seed)
np.random.seed(args.seed)
torch.manual_seed(args.seed)
torch.cuda.manual_seed_all(args.seed)
log.info('Building model...')
if args.task == 'tag':
model = SummarizerLinear()
# model = SummarizerLinearAttended(128, 256)
# model = SummarizerRNN(128, 256)
else:
model = SummarizerAbstractive(128, 256, device)
if len(args.gpu_ids) > 0:
model = nn.DataParallel(model, args.gpu_ids)
if args.load_path:
log.info('Loading checkpoint from {}...'.format(args.load_path))
model, step = util.load_model(model, args.load_path, args.gpu_ids)
else:
step = 0
model = model.to(device)
model.train()
## get a saver
saver = util.CheckpointSaver(args.save_dir,
max_checkpoints=args.max_checkpoints,
metric_name=args.metric_name,
maximize_metric=args.maximize_metric,
log=log)
optimizer = optim.Adam(model.parameters(), args.lr,
weight_decay=args.l2_wd)
log.info('Building dataset...')
data_path = PROCESSED_DATA_SUPER_TINY if args.split == 'super_tiny' else PROCESSED_DATA
with open(data_path, 'rb') as f:
all_data = pickle.load(f)
if 'tiny' in args.split:
train_split = all_data['tiny']
dev_split = all_data['tiny']
else:
train_split = all_data['train']
dev_split = all_data['dev']
train_dataset = SummarizationDataset(
train_split['X'], train_split['y'], train_split['gold'])
dev_dataset = SummarizationDataset(
dev_split['X'], dev_split['y'], dev_split['gold'])
collate_fn = tag_collate_fn if args.task == 'tag' else decode_collate_fn
train_loader = data.DataLoader(train_dataset,
batch_size=args.batch_size,
num_workers=args.num_workers,
shuffle=True,
collate_fn=collate_fn)
dev_loader = data.DataLoader(dev_dataset,
batch_size=args.batch_size,
num_workers=args.num_workers,
shuffle=False,
collate_fn=collate_fn)
## Train!
log.info('Training...')
steps_till_eval = args.eval_steps
epoch = step // len(train_dataset)
while epoch != args.num_epochs:
epoch += 1
log.info('Starting epoch {}...'.format(epoch))
batch_num = 0
with torch.enable_grad(), \
tqdm(total=len(train_loader.dataset)) as progress_bar:
for X, y, _ in train_loader:
batch_size = X.size(0)
batch_num += 1
X = X.to(device)
y = y.float().to(device) # (batch_size, max_len) for tag, (batch_size, 110) for decode
optimizer.zero_grad()
if args.task == 'tag':
logits = model(X) # (batch_size, max_len)
mask = (X != PAD_VALUE).float() # 1 for real data, 0 for pad, size of (batch_size, max_len)
loss = (F.binary_cross_entropy_with_logits(logits, y, reduction='none') * mask).mean()
loss_val = loss.item()
else:
logits = model(X, y[:, :-1]) # (batch_size, 109, max_len)
loss = sum(F.cross_entropy(logits[i], y[i, 1:], ignore_index=-1, reduction='mean')\
for i in range(batch_size)) / batch_size
loss_val = loss.item()
loss.backward()
nn.utils.clip_grad_norm_(model.parameters(), args.max_grad_norm)
optimizer.step()
# scheduler.step(step // batch_size)
# Log info
step += args.batch_size
progress_bar.update(args.batch_size)
progress_bar.set_postfix(epoch=epoch,
Loss=loss_val)
tbx.add_scalar('train/Loss', loss_val, step)
tbx.add_scalar('train/LR',
optimizer.param_groups[0]['lr'],
step)
steps_till_eval -= batch_size
if steps_till_eval <= 0:
steps_till_eval = args.eval_steps
# Evaluate and save checkpoint
log.info('Evaluating at step {}...'.format(step))
results, pred_dict = evaluate(args, model, dev_loader, device)
if results is None:
log.info('Selected predicted no select for all in batch')
continue
saver.save(step, model, results[args.metric_name], device)
# # Log to console
results_str = ', '.join('{}: {:05.2f}'.format(k, v)
for k, v in results.items())
log.info('Dev {}'.format(results_str))
# Log to TensorBoard
log.info('Visualizing in TensorBoard...')
for k, v in results.items():
tbx.add_scalar('dev/{}'.format(k), v, step)
def main(args):
# Set up logging and devices
args.save_dir = util.get_save_dir(args.save_dir, args.name, training=True)
log = util.get_logger(args.save_dir, args.name)
tbx = SummaryWriter(args.save_dir) # Writes entries directly to event files in the logdir to be consumed by TensorBoard.
device, args.gpu_ids = util.get_available_devices()
log.info(f'Args: {dumps(vars(args), indent=4, sort_keys=True)}')
args.batch_size *= max(1, len(args.gpu_ids)) # 一个gpu： batch_size=64 看实际情况
# Set random seed
log.info(f'Using random seed {args.seed}...')
random.seed(args.seed)
np.random.seed(args.seed)
torch.manual_seed(args.seed)
torch.cuda.manual_seed_all(args.seed)
# Get embeddings
log.info('Loading embeddings...')
word_vectors = util.torch_from_json(args.word_emb_file)
# Get model
log.info('Building model...')
model = BiDAF(word_vectors=word_vectors,
hidden_size=args.hidden_size,
drop_prob=args.drop_prob)
model = nn.DataParallel(model, args.gpu_ids)
if args.load_path: # default=None
log.info(f'Loading checkpoint from {args.load_path}...')
model, step = util.load_model(model, args.load_path, args.gpu_ids)
else:
step = 0
model = model.to(device)
model.train()
ema = util.EMA(model, args.ema_decay) # ema_decay = 0.999
# ema core => new_average = (1.0 - decay) * param.data + decay * self.shadow[name]
# Get saver
# metric_name: NLL or EM F1
saver = util.CheckpointSaver(args.save_dir,
max_checkpoints=args.max_checkpoints, # max_checkpoints = 5
metric_name=args.metric_name,
maximize_metric=args.maximize_metric,
log=log)
# Get optimizer and scheduler
optimizer = optim.Adadelta(model.parameters(), args.lr,
weight_decay=args.l2_wd) # lr : default=0.5 l2_wd : default=0
scheduler = sched.LambdaLR(optimizer, lambda s: 1.) # Constant LR
# Get data loader
log.info('Building dataset...')
train_dataset = SQuAD(args.train_record_file, args.use_squad_v2) # train_record_file = './data/train.npz'
train_loader = data.DataLoader(train_dataset,
batch_size=args.batch_size, # 64
shuffle=True, # sampler = RandomSampler(dataset) batch_sampler = BatchSampler(sampler, batch_size, drop_last)
num_workers=args.num_workers, # 4
collate_fn=collate_fn) # merges a list of samples to form a mini-batch.
dev_dataset = SQuAD(args.dev_record_file, args.use_squad_v2) # dev_record_file = './data/dev.npz'
dev_loader = data.DataLoader(dev_dataset,
batch_size=args.batch_size,
shuffle=False,
num_workers=args.num_workers,
collate_fn=collate_fn)# Merge examples of different length by padding all examples to the maximum length in the batch.
# Train
log.info('Training...')
steps_till_eval = args.eval_steps # 50000
epoch = step // len(train_dataset) # len(train_dataset)= 7 epoch=0
while epoch != args.num_epochs: # 30
epoch += 1
log.info(f'Starting epoch {epoch}...')
with torch.enable_grad(), \
tqdm(total=len(train_loader.dataset)) as progress_bar:
for cw_idxs, cc_idxs, qw_idxs, qc_idxs, y1, y2, ids in train_loader:
# Setup for forward
cw_idxs = cw_idxs.to(device)
qw_idxs = qw_idxs.to(device)
batch_size = cw_idxs.size(0) # 64
optimizer.zero_grad()
# Forward
log_p1, log_p2 = model(cw_idxs, qw_idxs)
y1, y2 = y1.to(device), y2.to(device)
loss = F.nll_loss(log_p1, y1) + F.nll_loss(log_p2, y2)
loss_val = loss.item()
# Backward
loss.backward()
nn.utils.clip_grad_norm_(model.parameters(), args.max_grad_norm) # max_grad_norm : default=5.0
optimizer.step() # 进行1次optimize
scheduler.step(step // batch_size)# train : step=0
ema(model, step // batch_size) # def __call__(self, model, num_updates):
# Log info
step += batch_size #step: 0 batch_size=64
progress_bar.update(batch_size)
progress_bar.set_postfix(epoch=epoch,
NLL=loss_val)
tbx.add_scalar('train/NLL', loss_val, step) # Add scalar data to summary.
tbx.add_scalar('train/LR',
optimizer.param_groups[0]['lr'],
step)
steps_till_eval -= batch_size
if steps_till_eval <= 0:
steps_till_eval = args.eval_steps # 50000
# Evaluate and save checkpoint
log.info(f'Evaluating at step {step}...')
ema.assign(model) ##
results, pred_dict = evaluate(model, dev_loader, device,
args.dev_eval_file, # './data/dev_eval.json'
args.max_ans_len, # 15
args.use_squad_v2)
saver.save(step, model, results[args.metric_name], device)
ema.resume(model)
# Log to console
results_str = ', '.join(f'{k}: {v:05.2f}' for k, v in results.items())
log.info(f'Dev {results_str}')
# Log to TensorBoard
log.info('Visualizing in TensorBoard...')
for k, v in results.items():
tbx.add_scalar(f'dev/{k}', v, step)
util.visualize(tbx,
pred_dict=pred_dict,
eval_path=args.dev_eval_file,
step=step,
split='dev',
num_visuals=args.num_visuals)
def main(args):
# Set up logging and devices
args.save_dir = util.get_save_dir(args.save_dir, args.name, training=True)
log = util.get_logger(args.save_dir, args.name)
tbx = SummaryWriter(args.save_dir)
device, args.gpu_ids = util.get_available_devices()
log.info('Args: {}'.format(dumps(vars(args), indent=4, sort_keys=True)))
# Comment out to only use 1 GPU on nv12
args.batch_size *= max(1, len(args.gpu_ids))
# Set random seed
log.info('Using random seed {}...'.format(args.seed))
random.seed(args.seed)
np.random.seed(args.seed)
torch.manual_seed(args.seed)
torch.cuda.manual_seed_all(args.seed)
# Get embeddings
log.info('Loading embeddings...')
word_vectors = util.torch_from_json(args.word_emb_file)
# Get model
log.info('Building model...')
model = None
max_context_len, max_question_len = args.para_limit, args.ques_limit
if (args.model_type == "bidaf" or args.model_type == "bert-bidaf"):
model = BiDAF(word_vectors=word_vectors,
hidden_size=args.hidden_size,
drop_prob=args.drop_prob)
elif (args.model_type == "dcn" or args.model_type == "bert-dcn"):
model = DCN(word_vectors=word_vectors,
hidden_size=args.hidden_size,
max_context_len=max_context_len,
max_question_len=max_question_len,
drop_prob=args.drop_prob)
elif (args.model_type == "bert-basic"):
model = BERT(word_vectors=word_vectors,
hidden_size=args.hidden_size,
drop_prob=args.drop_prob)
if model is None:
raise ValueError('Model is unassigned. Please ensure --model_type \
chooses between {bidaf, bert-bidaf, dcn, bert-dcn, bert-basic} ')
model = nn.DataParallel(model, args.gpu_ids)
if args.load_path:
log.info('Loading checkpoint from {}...'.format(args.load_path))
model, step = util.load_model(model, args.load_path, args.gpu_ids)
else:
step = 0
model = model.to(device)
model.train()
ema = util.EMA(model, args.ema_decay)
# Get saver
saver = util.CheckpointSaver(args.save_dir,
max_checkpoints=args.max_checkpoints,
metric_name=args.metric_name,
maximize_metric=args.maximize_metric,
log=log)
# Get optimizer and scheduler
optimizer = optim.Adadelta(model.parameters(),
args.lr,
weight_decay=args.l2_wd)
scheduler = sched.LambdaLR(optimizer, lambda s: 1.) # Constant LR
# Get data loader
log.info('Building dataset...')
train_dataset = SQuAD(args.train_record_file, args.use_squad_v2)
train_loader = data.DataLoader(train_dataset,
batch_size=args.batch_size,
shuffle=True,
num_workers=args.num_workers,
collate_fn=collate_fn)
dev_dataset = SQuAD(args.dev_record_file, args.use_squad_v2)
dev_loader = data.DataLoader(dev_dataset,
batch_size=args.batch_size,
shuffle=False,
num_workers=args.num_workers,
collate_fn=collate_fn)
# Train
log.info('Training...')
steps_till_eval = args.eval_steps
epoch = step // len(train_dataset)
count_skip = 0
while epoch != args.num_epochs:
epoch += 1
log.info('Starting epoch {}...'.format(epoch))
with torch.enable_grad(), \
tqdm(total=len(train_loader.dataset)) as progress_bar:
for cw_idxs, cc_idxs, qw_idxs, qc_idxs, y1, y2, ids in train_loader:
batch_size = cw_idxs.size(0)
count_skip += 1
if (args.skip_examples == True
and (count_skip % 5 == 1 or count_skip % 5 == 2
or count_skip % 5 == 3 or count_skip % 5 == 4)):
step += batch_size
progress_bar.update(batch_size)
steps_till_eval -= batch_size
continue
# Setup for forward
cw_idxs = cw_idxs.to(device)
qw_idxs = qw_idxs.to(device)
batch_size = cw_idxs.size(0)
optimizer.zero_grad()
## Additions for BERT ##
max_context_len, max_question_len = args.para_limit, args.ques_limit
if "bert" in args.model_type:
bert_train_embeddings = get_embeddings(
"train", ids, args.para_limit, args.ques_limit)
else:
bert_train_embeddings = None
# Forward
log_p1, log_p2 = model(cw_idxs, qw_idxs, bert_train_embeddings, \
max_context_len, max_question_len, device)
y1, y2 = y1.to(device), y2.to(device)
loss = F.nll_loss(log_p1, y1) + F.nll_loss(log_p2, y2)
loss_val = loss.item()
# Backward
loss.backward()
nn.utils.clip_grad_norm_(model.parameters(),
args.max_grad_norm)
optimizer.step()
scheduler.step(step // batch_size)
ema(model, step // batch_size)
# Log info
step += batch_size
progress_bar.update(batch_size)
progress_bar.set_postfix(epoch=epoch, NLL=loss_val)
tbx.add_scalar('train/NLL', loss_val, step)
tbx.add_scalar('train/LR', optimizer.param_groups[0]['lr'],
step)
steps_till_eval -= batch_size
if steps_till_eval <= 0:
steps_till_eval = args.eval_steps
# Evaluate and save checkpoint
log.info('Evaluating at step {}...'.format(step))
ema.assign(model)
results, pred_dict = evaluate(model, dev_loader, device,
args.dev_eval_file,
args.max_ans_len,
args.use_squad_v2, args)
saver.save(step, model, results[args.metric_name], device)
ema.resume(model)
# Log to console
results_str = ', '.join('{}: {:05.2f}'.format(k, v)
for k, v in results.items())
log.info('Dev {}'.format(results_str))
# Log to TensorBoard
log.info('Visualizing in TensorBoard...')
for k, v in results.items():
tbx.add_scalar('dev/{}'.format(k), v, step)
util.visualize(tbx,
pred_dict=pred_dict,
eval_path=args.dev_eval_file,
step=step,
split='dev',
num_visuals=args.num_visuals)
def main(args):
# Set up logging and devices
args.save_dir = util.get_save_dir(args.save_dir, args.name, training=True)
log = util.get_logger(args.save_dir, args.name)
tbx = SummaryWriter(args.save_dir)
device, args.gpu_ids = util.get_available_devices()
log.info(f'Args: {dumps(vars(args), indent=4, sort_keys=True)}')
args.batch_size *= max(1, len(args.gpu_ids))
# Set random seed
log.info(f'Using random seed {args.seed}...')
random.seed(args.seed)
np.random.seed(args.seed)
torch.manual_seed(args.seed)
torch.cuda.manual_seed_all(args.seed)
# Get model
log.info('Building model...')
'''
TODO: YOUR MODEL HERE
'''
model = None
model = nn.DataParallel(model, args.gpu_ids)
if args.load_path:
log.info(f'Loading checkpoint from {args.load_path}...')
model, step = util.load_model(model, args.load_path, args.gpu_ids)
else:
step = 0
model = model.to(device)
model.train()
ema = util.EMA(model, args.ema_decay)
# Get saver
saver = util.CheckpointSaver(args.save_dir,
max_checkpoints=args.max_checkpoints,
metric_name=args.metric_name,
maximize_metric=args.maximize_metric,
log=log)
# Get optimizer and scheduler
optimizer = optim.Adadelta(model.parameters(),
args.lr,
weight_decay=args.l2_wd)
scheduler = sched.LambdaLR(optimizer, lambda s: 1.) # Constant LR
# Get data loader
log.info('Building dataset...')
train_dataset = MyDataset(args.train_record_file)
train_loader = data.DataLoader(train_dataset,
batch_size=args.batch_size,
shuffle=True,
num_workers=args.num_workers,
collate_fn=collate_fn)
dev_dataset = MyDataset(args.dev_record_file, args.use_squad_v2)
dev_loader = data.DataLoader(dev_dataset,
batch_size=args.batch_size,
shuffle=False,
num_workers=args.num_workers,
collate_fn=collate_fn)
# Train
log.info('Training...')
steps_till_eval = args.eval_steps
epoch = step // len(train_dataset)
while epoch != args.num_epochs:
epoch += 1
log.info(f'Starting epoch {epoch}...')
with torch.enable_grad(), \
tqdm(total=len(train_loader.dataset)) as progress_bar:
for features, ys, ids in train_loader:
# Setup for forward
batch_size = 1 # TODO:
optimizer.zero_grad()
# Forward
outputs = model(features)
y = y.to(device)
loss = loss_fn(outputs, y) # TODO
loss_val = loss.item()
# Backward
loss.backward()
nn.utils.clip_grad_norm_(model.parameters(),
args.max_grad_norm)
optimizer.step()
scheduler.step(step // batch_size)
ema(model, step // batch_size)
# Log info
step += batch_size
progress_bar.update(batch_size)
progress_bar.set_postfix(epoch=epoch, NLL=loss_val)
tbx.add_scalar('train/NLL', loss_val, step)
tbx.add_scalar('train/LR', optimizer.param_groups[0]['lr'],
step)
steps_till_eval -= batch_size
if steps_till_eval <= 0:
steps_till_eval = args.eval_steps
# Evaluate and save checkpoint
log.info(f'Evaluating at step {step}...')
ema.assign(model)
results, pred_dict = evaluate(model, dev_loader, device,
args.dev_eval_file)
saver.save(step, model, results[args.metric_name], device)
ema.resume(model)
# Log to console
results_str = ', '.join(f'{k}: {v:05.2f}'
for k, v in results.items())
log.info(f'Dev {results_str}')
# Log to TensorBoard
log.info('Visualizing in TensorBoard...')
for k, v in results.items():
tbx.add_scalar(f'dev/{k}', v, step)
util.visualize(tbx,
pred_dict=pred_dict,
eval_path=args.dev_eval_file,
step=step,
split='dev',
num_visuals=args.num_visuals)
def main(args):
# Set up logging and devices
name = "train_exp2"
args.save_dir = util.get_save_dir(args.logging_dir, name, training=True)
log = get_logger(args.save_dir, name)
tbx = SummaryWriter(args.save_dir)
device, gpu_ids = util.get_available_devices()
log.info(f"Args: {dumps(vars(args), indent=4, sort_keys=True)}")
args.batch_size *= max(1, len(gpu_ids))
# Set random seed
log.info(f"Using random seed {args.random_seed}...")
random.seed(args.random_seed)
np.random.seed(args.random_seed)
torch.manual_seed(args.random_seed)
torch.cuda.manual_seed_all(args.random_seed)
# Get embeddings
log.info(f"Loading embeddings from {args.word_emb_file}...")
word_vectors = util.torch_from_json(args.word_emb_file)
# Get model
log.info("Building model...")
model = BiDAF(word_vectors=word_vectors,
hidden_size=args.hidden_size,
drop_prob=args.drop_prob)
model = nn.DataParallel(model, gpu_ids)
if args.load_path:
log.info(f"Loading checkpoint from {args.load_path}...")
model, step = util.load_model(model, args.load_path, gpu_ids)
else:
step = 0
model = model.to(device)
model.train()
ema = util.EMA(model, args.ema_decay)
# Get saver
saver = util.CheckpointSaver(args.save_dir,
max_checkpoints=args.max_checkpoints,
metric_name=args.metric_name,
maximize_metric=args.maximize_metric,
log=log)
# Get optimizer and scheduler
optimizer = optim.Adadelta(model.parameters(), args.learning_rate,
weight_decay=args.learning_rate_decay)
# scheduler = sched.LambdaLR(optimizer, lambda s: 1.) # Constant LR
scheduler = sched.ReduceLROnPlateau(optimizer=optimizer,
mode="min", factor=0.1,
patience=2, verbose=True, cooldown=0
min_lr=0.0005)
for epoch in range(args.num_epochs):
log.info(f"Starting epoch {epoch}...")
for i in range(args.num_train_chunks):
# Get data loader
train_rec_file = f"{args.train_record_file_exp2}_{i}.npz"
log.info(f'Building dataset from {train_rec_file} ...')
train_dataset = SQuAD(train_rec_file, args.exp2_train_topic_contexts, use_v2=True)
train_loader = data.DataLoader(train_dataset,
batch_size=args.batch_size,
shuffle=True,
num_workers=args.num_workers,
collate_fn=collate_fn)
# Train
log.info('Training...')
steps_till_eval = args.eval_steps
epoch = 0
# torch.set_num_threads(7)
with torch.enable_grad(), tqdm(total=len(train_loader.dataset)) as progress_bar:
for cw_idxs, cc_idxs, qw_idxs, qc_idxs, y1, y2, ids in train_loader:
# Setup for forward
cw_idxs = cw_idxs.to(device)
qw_idxs = qw_idxs.to(device)
batch_size = qw_idxs.size(0)
optimizer.zero_grad()
# Forward
log_p1, log_p2 = model(cw_idxs, qw_idxs)
y1, y2 = y1.to(device), y2.to(device)
loss = F.nll_loss(log_p1, y1) + F.nll_loss(log_p2, y2)
loss_val = loss.item()
# Backward
loss.backward()
nn.utils.clip_grad_norm_(model.parameters(), args.max_grad_norm)
optimizer.step()
scheduler.step(step // batch_size)
ema(model, step // batch_size)
# Log info
step += batch_size
progress_bar.update(batch_size)
progress_bar.set_postfix(epoch=epoch,
NLL=loss_val)
tbx.add_scalar('train/NLL', loss_val, step)
tbx.add_scalar('train/LR',
optimizer.param_groups[0]['lr'],
step)
steps_till_eval -= batch_size
if steps_till_eval <= 0:
steps_till_eval = args.eval_steps
# Evaluate and save checkpoint
log.info(f"Evaluating at step {step}...")
ema.assign(model)
for i in range(args.num_dev_chunks):
# Get data loader
all_pred_dicts = {}
all_results = OrderedDict()
dev_rec_file = f"{args.dev_record_file_exp2}_{i}.npz"
log.info(f'Building evaluating dataset from {dev_rec_file} ...')
dev_dataset = SQuAD(dev_rec_file,
args.exp2_dev_topic_contexts,
use_v2=True)
dev_loader = data.DataLoader(dev_dataset,
batch_size=args.batch_size,
shuffle=True,
num_workers=args.num_workers,
collate_fn=collate_fn)
results, pred_dict = evaluate(model, dev_loader, device,
args.dev_eval_file,
args.max_ans_len,
use_squad_v2=True)
all_results.update(results)
all_pred_dicts.update(pred_dict)
del dev_dataset
del dev_loader
del results
del pred_dict
torch.cuda.empty_cache()
saver.save(step, model, all_results[args.metric_name], device)
ema.resume(model)
# Log to console
results_str = ', '.join(f'{k}: {v:05.2f}' for k, v in all_results.items())
log.info(f"Dev {results_str}")
# Log to TensorBoard
log.info('Visualizing in TensorBoard...')
for k, v in all_results.items():
tbx.add_scalar(f"dev/{k}", v, step)
util.visualize(tbx,
pred_dict=all_pred_dicts,
eval_path=args.dev_eval_file,
step=step,
split='dev',
num_visuals=args.num_visuals)
torch.cuda.empty_cache()
del train_dataset
del train_loader
torch.cuda.empty_cache()
def main():
# define parser and arguments
args = get_train_test_args()
util.set_seed(args.seed)
if args.mixture_of_experts and args.do_eval:
model = MoE(load_gate=True)
experts = True
model.gate.eval()
elif args.mixture_of_experts and args.do_train:
model = MoE(load_gate=False)
experts = True
else:
model = DistilBertForQuestionAnswering.from_pretrained("distilbert-base-uncased")
experts = False
if args.reinit > 0:
transformer_temp = getattr(model, 'distilbert')
for layer in transformer_temp.transformer.layer[-args.reinit:]:
for module in layer.modules():
print(type(module))
if isinstance(module, (nn.Linear, nn.Embedding)):
module.weight.data.normal_(mean=0.0, std=transformer_temp.config.initializer_range)
elif isinstance(module, nn.LayerNorm):
module.bias.data.zero_()
module.weight.data.fill_(1.0)
if isinstance(module, nn.Linear) and module.bias is not None:
module.bias.data.zero_()
tokenizer = DistilBertTokenizerFast.from_pretrained('distilbert-base-uncased')
if args.do_train:
if not os.path.exists(args.save_dir):
os.makedirs(args.save_dir)
args.save_dir = util.get_save_dir(args.save_dir, args.run_name)
log = util.get_logger(args.save_dir, 'log_train')
log.info(f'Args: {json.dumps(vars(args), indent=4, sort_keys=True)}')
log.info("Preparing Training Data...")
args.device = torch.device('cuda') if torch.cuda.is_available() else torch.device('cpu')
trainer = Trainer(args, log)
train_dataset, _ = get_dataset(args, args.train_datasets, args.train_dir, tokenizer, 'train')
log.info("Preparing Validation Data...")
val_dataset, val_dict = get_dataset(args, args.train_datasets, args.val_dir, tokenizer, 'val')
train_loader = DataLoader(train_dataset,
batch_size=args.batch_size,
sampler=RandomSampler(train_dataset))
val_loader = DataLoader(val_dataset,
batch_size=args.batch_size,
sampler=SequentialSampler(val_dataset))
best_scores = trainer.train(model, train_loader, val_loader, val_dict, experts)
if args.do_eval:
args.device = torch.device('cuda') if torch.cuda.is_available() else torch.device('cpu')
split_name = 'test' if 'test' in args.eval_dir else 'validation'
log = util.get_logger(args.save_dir, f'log_{split_name}')
trainer = Trainer(args, log)
if args.mixture_of_experts is False:
checkpoint_path = os.path.join(args.save_dir, 'checkpoint')
model = DistilBertForQuestionAnswering.from_pretrained(checkpoint_path)
model.to(args.device)
eval_dataset, eval_dict = get_dataset(args, args.eval_datasets, args.eval_dir, tokenizer, split_name)
eval_loader = DataLoader(eval_dataset,
batch_size=args.batch_size,
sampler=SequentialSampler(eval_dataset))
eval_preds, eval_scores = trainer.evaluate(model, eval_loader,
eval_dict, return_preds=True,
split=split_name, MoE = True)
results_str = ', '.join(f'{k}: {v:05.2f}' for k, v in eval_scores.items())
log.info(f'Eval {results_str}')
# Write submission file
sub_path = os.path.join(args.save_dir, split_name + '_' + args.sub_file)
log.info(f'Writing submission file to {sub_path}...')
with open(sub_path, 'w', newline='', encoding='utf-8') as csv_fh:
csv_writer = csv.writer(csv_fh, delimiter=',')
csv_writer.writerow(['Id', 'Predicted'])
for uuid in sorted(eval_preds):
csv_writer.writerow([uuid, eval_preds[uuid]])
def main():
# define parser and arguments
args = get_train_test_args()
util.set_seed(args.seed)
# model = DistilBertForQuestionAnswering.from_pretrained("distilbert-base-uncased")
model = DomainQA(args.num_classes, args.hidden_size, args.num_layers,
args.dropout, args.dis_lambda, args.concat, args.anneal)
tokenizer = DistilBertTokenizerFast.from_pretrained(
'distilbert-base-uncased')
if args.do_train:
if not os.path.exists(args.save_dir):
os.makedirs(args.save_dir)
args.save_dir = util.get_save_dir(args.save_dir, args.run_name)
log = util.get_logger(args.save_dir, 'log_train')
log.info(f'Args: {json.dumps(vars(args), indent=4, sort_keys=True)}')
log.info("Preparing Training Data...")
args.device = torch.device(
'cuda') if torch.cuda.is_available() else torch.device('cpu')
if args.load_weights != '':
args.load_weights = os.path.join(args.load_weights, 'checkpoint',
model.WEIGHTS_NAME)
model.load_state_dict(torch.load(args.load_weights))
if args.load_distilbert_weights != '':
# args.load_distilbert_weights = os.path.join(args.load_distilbert_weights, 'checkpoint', model.WEIGHTS_NAME)
args.load_distilbert_weights = os.path.join(
args.load_distilbert_weights, 'checkpoint',
'pytorch_model.bin')
model.distilbert.load_state_dict(
torch.load(args.load_distilbert_weights))
print('loaded pretrained distilbert weights from',
args.load_distilbert_weights)
trainer = Trainer(args, log, model)
#target_data_dir, target_dataset, tokenizer, split_name, source_data_dir = None, source_dataset = None
train_dataset, _ = get_train_dataset(args, \
args.target_train_dir,\
args.target_train_datasets,\
tokenizer, 'train', \
source_data_dir=args.source_train_dir, \
source_dataset=args.source_train_datasets)
log.info("Preparing Validation Data...")
val_dataset, val_dict = get_dataset(args, \
args.eval_datasets,\
args.eval_dir,\
tokenizer, 'val')
train_loader = DataLoader(train_dataset,
batch_size=args.batch_size,
sampler=RandomSampler(train_dataset))
val_loader = DataLoader(val_dataset,
batch_size=args.batch_size,
sampler=SequentialSampler(val_dataset))
best_scores = trainer.train(train_loader, val_loader, val_dict)
if args.do_eval:
args.device = torch.device(
'cuda') if torch.cuda.is_available() else torch.device('cpu')
split_name = 'test' if 'test' in args.eval_dir else 'validation'
log = util.get_logger(args.save_dir, f'log_{split_name}')
trainer = Trainer(args, log, model)
config_path = os.path.join(args.save_dir, 'checkpoint', 'config.json')
checkpoint_path = os.path.join(args.save_dir, 'checkpoint',
model.WEIGHTS_NAME)
model.load_state_dict(torch.load(checkpoint_path))
model.to(args.device)
eval_dataset, eval_dict = get_dataset(args, args.eval_datasets,
args.eval_dir, tokenizer,
split_name)
eval_loader = DataLoader(eval_dataset,
batch_size=args.batch_size,
sampler=SequentialSampler(eval_dataset))
eval_preds, eval_scores = trainer.evaluate(eval_loader,
eval_dict,
return_preds=True,
split=split_name)
results_str = ', '.join(f'{k}: {v:05.2f}'
for k, v in eval_scores.items())
log.info(f'Eval {results_str}')
# Write submission file
sub_path = os.path.join(args.save_dir,
split_name + '_' + args.sub_file)
log.info(f'Writing submission file to {sub_path}...')
with open(sub_path, 'w', newline='', encoding='utf-8') as csv_fh:
csv_writer = csv.writer(csv_fh, delimiter=',')
csv_writer.writerow(['Id', 'Predicted'])
for uuid in sorted(eval_preds):
csv_writer.writerow([uuid, eval_preds[uuid]])
def main(args):
# Set up logging
args.save_dir = util.get_save_dir(args.save_dir, args.name, training=False)
log = util.get_logger(args.save_dir, args.name)
device, gpu_ids = util.get_available_devices()
args.batch_size *= max(1, len(gpu_ids))
log.info(f'Args: {dumps(vars(args), indent=4, sort_keys=True)}')
# Get embeddings
log.info('Loading embeddings...')
word_vectors = util.torch_from_json(args.word_emb_file)
models = {}
if args.use_ensemble:
total_models = 0
for model_name in ['bidaf', 'bidafextra', 'fusionnet']:
models_list = []
for model_file in glob.glob(
f'{args.load_path}/{model_name}-*/{args.ensemble_models}'):
# Get model
log.info('Building model...')
if model_name == 'bidaf':
model = BiDAF(word_vectors=word_vectors,
hidden_size=args.hidden_size)
elif model_name == 'bidafextra':
model = BiDAFExtra(word_vectors=word_vectors, args=args)
elif model_name == 'fusionnet':
model = FusionNet(word_vectors=word_vectors, args=args)
model = nn.DataParallel(model, gpu_ids)
log.info(f'Loading checkpoint from {model_file}...')
model = util.load_model(model,
model_file,
gpu_ids,
return_step=False)
# Load each model on CPU (have plenty of RAM ...)
model = model.cpu()
model.eval()
models_list.append(model)
models[model_name] = models_list
total_models += len(models_list)
log.info(f'Using an ensemble of {total_models} models')
else:
device, gpu_ids = util.get_available_devices()
# Get model
log.info('Building model...')
if args.model == 'bidaf':
model = BiDAF(word_vectors=word_vectors,
hidden_size=args.hidden_size)
elif args.model == 'bidafextra':
model = BiDAFExtra(word_vectors=word_vectors, args=args)
elif args.model == 'fusionnet':
model = FusionNet(word_vectors=word_vectors, args=args)
model = nn.DataParallel(model, gpu_ids)
log.info(f'Loading checkpoint from {args.load_path}...')
model = util.load_model(model,
args.load_path,
gpu_ids,
return_step=False)
model = model.to(device)
model.eval()
models[args.model] = [model]
# Get data loader
log.info('Building dataset...')
record_file = vars(args)[f'{args.split}_record_file']
dataset = SQuAD(record_file, args)
data_loader = data.DataLoader(dataset,
batch_size=args.batch_size,
shuffle=False,
num_workers=args.num_workers,
collate_fn=collate_fn)
# Evaluate
log.info(f'Evaluating on {args.split} split...')
nll_meter = util.AverageMeter()
pred_dict = {} # Predictions for TensorBoard
sub_dict = {} # Predictions for submission
eval_file = vars(args)[f'{args.split}_eval_file']
with open(eval_file, 'r') as fh:
gold_dict = json_load(fh)
with torch.no_grad(), \
tqdm(total=len(dataset)) as progress_bar:
for cw_idxs, cc_idxs, qw_idxs, qc_idxs, cw_pos, cw_ner, cw_freq, cqw_extra, y1, y2, ids in data_loader:
# Setup for forward
cw_idxs = cw_idxs.to(device)
qw_idxs = qw_idxs.to(device)
batch_size = cw_idxs.size(0)
p1s = []
p2s = []
for model_name in models:
for model in models[model_name]:
# Move model to GPU to evaluate
model = model.to(device)
# Forward
if model_name == 'bidaf':
log_p1, log_p2 = model.to(device)(cw_idxs, qw_idxs)
else:
log_p1, log_p2 = model.to(device)(cw_idxs, qw_idxs,
cw_pos, cw_ner,
cw_freq, cqw_extra)
log_p1, log_p2 = log_p1.cpu(), log_p2.cpu()
if not args.use_ensemble:
y1, y2 = y1.to(device), y2.to(device)
log_p1, log_p2 = log_p1.to(device), log_p2.to(device)
loss = F.nll_loss(log_p1, y1) + F.nll_loss(log_p2, y2)
nll_meter.update(loss.item(), batch_size)
# Move model back to CPU to release GPU memory
model = model.cpu()
# Get F1 and EM scores
p1, p2 = log_p1.exp().unsqueeze(
-1).cpu(), log_p2.exp().unsqueeze(-1).cpu()
p1s.append(p1), p2s.append(p2)
best_ps = torch.max(
torch.cat([
torch.cat(p1s, -1).unsqueeze(-1),
torch.cat(p2s, -1).unsqueeze(-1)
], -1), -2)[0]
p1, p2 = best_ps[:, :, 0], best_ps[:, :, 1]
starts, ends = util.discretize(p1, p2, args.max_ans_len,
args.use_squad_v2)
# Log info
progress_bar.update(batch_size)
if args.split != 'test':
# No labels for the test set, so NLL would be invalid
progress_bar.set_postfix(NLL=nll_meter.avg)
idx2pred, uuid2pred = util.convert_tokens(gold_dict, ids.tolist(),
starts.tolist(),
ends.tolist(),
args.use_squad_v2)
pred_dict.update(idx2pred)
sub_dict.update(uuid2pred)
# Log results (except for test set, since it does not come with labels)
if args.split != 'test':
results = util.eval_dicts(gold_dict, pred_dict, args.use_squad_v2)
results_list = [('NLL', nll_meter.avg), ('F1', results['F1']),
('EM', results['EM'])]
if args.use_squad_v2:
results_list.append(('AvNA', results['AvNA']))
results = OrderedDict(results_list)
# Log to console
results_str = ', '.join(f'{k}: {v:05.2f}' for k, v in results.items())
log.info(f'{args.split.title()} {results_str}')
# Log to TensorBoard
tbx = SummaryWriter(args.save_dir)
util.visualize(tbx,
pred_dict=pred_dict,
eval_path=eval_file,
step=0,
split=args.split,
num_visuals=args.num_visuals)
# Write submission file
sub_path = join(args.save_dir, args.split + '_' + args.sub_file)
log.info(f'Writing submission file to {sub_path}...')
with open(sub_path, 'w', newline='', encoding='utf-8') as csv_fh:
csv_writer = csv.writer(csv_fh, delimiter=',')
csv_writer.writerow(['Id', 'Predicted'])
for uuid in sorted(sub_dict):
csv_writer.writerow([uuid, sub_dict[uuid]])
def run(args):
# set up args
if args.cuda and torch.cuda.is_available():
device = torch.device('cuda')
args.cuda = True
else:
device = torch.device('cpu')
args.cuda = False
if args.train_mode == 'thermo' or args.train_mode == 'thermo_wake':
partition = util.get_partition(args.num_partitions,
args.partition_type, args.log_beta_min,
device)
util.print_with_time('device = {}'.format(device))
util.print_with_time(str(args))
# save args
save_dir = util.get_save_dir()
args_path = util.get_args_path(save_dir)
util.save_object(args, args_path)
# data
binarized_mnist_train, binarized_mnist_valid, binarized_mnist_test = \
data.load_binarized_mnist(where=args.where)
data_loader = data.get_data_loader(binarized_mnist_train, args.batch_size,
device)
valid_data_loader = data.get_data_loader(binarized_mnist_valid,
args.valid_batch_size, device)
test_data_loader = data.get_data_loader(binarized_mnist_test,
args.test_batch_size, device)
train_obs_mean = torch.tensor(np.mean(binarized_mnist_train, axis=0),
device=device,
dtype=torch.float)
# init models
util.set_seed(args.seed)
generative_model, inference_network = util.init_models(
train_obs_mean, args.architecture, device)
# optim
optim_kwargs = {'lr': args.learning_rate}
# train
if args.train_mode == 'ws':
train_callback = train.TrainWakeSleepCallback(
save_dir, args.num_particles * args.batch_size, test_data_loader,
args.eval_num_particles, args.logging_interval,
args.checkpoint_interval, args.eval_interval)
train.train_wake_sleep(generative_model, inference_network,
data_loader, args.num_iterations,
args.num_particles, optim_kwargs,
train_callback)
elif args.train_mode == 'ww':
train_callback = train.TrainWakeWakeCallback(
save_dir, args.num_particles, test_data_loader,
args.eval_num_particles, args.logging_interval,
args.checkpoint_interval, args.eval_interval)
train.train_wake_wake(generative_model, inference_network, data_loader,
args.num_iterations, args.num_particles,
optim_kwargs, train_callback)
elif args.train_mode == 'reinforce' or args.train_mode == 'vimco':
train_callback = train.TrainIwaeCallback(
save_dir, args.num_particles, args.train_mode, test_data_loader,
args.eval_num_particles, args.logging_interval,
args.checkpoint_interval, args.eval_interval)
train.train_iwae(args.train_mode, generative_model, inference_network,
data_loader, args.num_iterations, args.num_particles,
optim_kwargs, train_callback)
elif args.train_mode == 'thermo':
train_callback = train.TrainThermoCallback(
save_dir, args.num_particles, partition, test_data_loader,
args.eval_num_particles, args.logging_interval,
args.checkpoint_interval, args.eval_interval)
train.train_thermo(generative_model, inference_network, data_loader,
args.num_iterations, args.num_particles, partition,
optim_kwargs, train_callback)
elif args.train_mode == 'thermo_wake':
train_callback = train.TrainThermoWakeCallback(
save_dir, args.num_particles, test_data_loader,
args.eval_num_particles, args.logging_interval,
args.checkpoint_interval, args.eval_interval)
train.train_thermo_wake(generative_model, inference_network,
data_loader, args.num_iterations,
args.num_particles, partition, optim_kwargs,
train_callback)
# eval validation
train_callback.valid_log_p, train_callback.valid_kl = train.eval_gen_inf(
generative_model, inference_network, valid_data_loader,
args.eval_num_particles)
# save models and stats
util.save_checkpoint(save_dir,
iteration=None,
generative_model=generative_model,
inference_network=inference_network)
stats_path = util.get_stats_path(save_dir)
util.save_object(train_callback, stats_path)