def test_abs(args, device_id, pt, step):
""" Implements testing process (meta / non-memta)
Arguments:
device_id (int) : the GPU id to be used
pt() : checkpoint model
step (int) : checkpoint step
Process:
- load checkpoint
- prepare dataloader class
- prepare model class
- prepare predictor
- predictor.translate()
"""
device = "cpu" if args.visible_gpus == '-1' else "cuda"
logger.info('Device ID %d', device_id)
logger.info('Device %s', device)
# Load chekcpoint
if (pt != ''):
test_from = pt
else:
test_from = args.test_from
logger.info('Loading checkpoint from %s' % test_from)
checkpoint = torch.load(test_from,
map_location=lambda storage, loc: storage)
opt = vars(checkpoint['opt'])
for k in opt.keys():
if (k in model_flags):
setattr(args, k, opt[k])
# Prepare dataloader
test_iter = data_loader.Dataloader(args,
load_dataset(args,
'test',
shuffle=False),
args.test_batch_size,
device,
shuffle=False,
is_test=True)
# Prepare model
if (args.meta_mode):
model = MTLAbsSummarizer(args, device, checkpoint)
else:
model = AbsSummarizer(args, device, checkpoint)
model.eval()
# Prepare predictor
tokenizer = BertTokenizer.from_pretrained('bert-base-uncased',
do_lower_case=True,
cache_dir=args.temp_dir)
symbols = {
'BOS': tokenizer.vocab['[unused0]'],
'EOS': tokenizer.vocab['[unused1]'],
'PAD': tokenizer.vocab['[PAD]'],
'EOQ': tokenizer.vocab['[unused2]']
}
predictor = build_predictor(args, tokenizer, symbols, model, logger)
predictor.translate(test_iter, step) # long time
def train_iter_fct():
return data_loader.Dataloader(args,
load_dataset(args,
'train',
shuffle=True),
args.batch_size,
device,
shuffle=True,
is_test=False)
def test_text_abs(args, device_id, pt, step):
device = "cpu" if args.visible_gpus == '-1' else "cuda"
if (pt != ''):
test_from = pt
else:
test_from = args.test_from
logger.info('Loading checkpoint from %s' % test_from)
checkpoint = torch.load(test_from,
map_location=lambda storage, loc: storage)
opt = vars(checkpoint['opt'])
for k in opt.keys():
if (k in model_flags):
setattr(args, k, opt[k])
print(args)
model = AbsSummarizer(args, device, checkpoint)
model.eval()
test_iter = data_loader.Dataloader(args,
load_dataset(args,
'test',
shuffle=False),
args.test_batch_size,
device,
shuffle=False,
is_test=True)
tokenizer = BertTokenizer.from_pretrained('bert-base-uncased',
do_lower_case=True,
cache_dir=args.temp_dir)
symbols = {
'BOS': tokenizer.vocab['[unused0]'],
'EOS': tokenizer.vocab['[unused1]'],
'PAD': tokenizer.vocab['[PAD]'],
'EOQ': tokenizer.vocab['[unused2]']
}
predictor = build_predictor(args, tokenizer, symbols, model, logger)
predictor.translate(test_iter, step)
def validate(args, device_id, pt, step):
''' Implements validation process (meta / non-memta)
Arguments:
device_id (int) : the GPU id to be used
pt() : checkpoint model
step (int) : checkpoint step
Process:
- load checkpoint
- prepare dataloader class
- prepare model class
- prepare loss func, which return loss class
- prepare trainer
- trainer.validate()
Meta vs Normal
- MetaDataloader vs Dataloader
- load_dataset vs load_meta_dataset
- MTLAbsSummarizer vs AbsSummarizer
- build_MTLtrainer vs MTLTrainer
'''
device = "cpu" if args.visible_gpus == '-1' else "cuda"
logger.info('Device ID %d' % device_id)
logger.info('Device %s' % device)
# Fix random seed to control experiement
torch.manual_seed(args.seed)
random.seed(args.seed)
torch.backends.cudnn.deterministic = True
if device_id >= 0:
torch.cuda.set_device(device_id)
torch.cuda.manual_seed(args.seed)
# Load checkpoint ard args
if (pt != ''):
test_from = pt
else:
test_from = args.test_from
logger.info('Loading checkpoint from %s' % test_from)
checkpoint = torch.load(test_from,
map_location=lambda storage, loc: storage)
opt = vars(checkpoint['opt']) # which is self.args
for k in opt.keys():
if (k in model_flags):
setattr(args, k, opt[k])
# Prepare dataloader
if (args.meta_mode):
def valid_iter_fct():
return data_loader.MetaDataloader(args,
load_meta_dataset(args,
'valid',
shuffle=True),
args.batch_size,
device,
shuffle=True,
is_test=False)
else:
valid_iter = data_loader.Dataloader(args,
load_dataset(args,
'valid',
shuffle=False),
args.batch_size,
device,
shuffle=False,
is_test=False)
# Prepare model
if (args.meta_mode):
model = MTLAbsSummarizer(args, device, checkpoint)
else:
model = AbsSummarizer(args, device, checkpoint)
#model.eval()
# Prepare optimizer for inner loop
# The optimizer for each task is seperated
if (args.meta_mode):
optims_inner = []
for i in range(args.num_task):
if (args.sep_optim):
optim_bert_inner = model_builder.build_optim_bert_inner(
args, model, checkpoint, 'maml')
optim_dec_inner = model_builder.build_optim_dec_inner(
args, model, checkpoint, 'maml')
optims_inner.append([optim_bert_inner, optim_dec_inner])
else:
self.optims_inner.append([
model_builder.build_optim_inner(args, model, checkpoint,
'maml')
])
# Prepare optimizer (not actually used, but get the step information)
if (args.sep_optim):
optim_bert = model_builder.build_optim_bert(args, model, checkpoint)
optim_dec = model_builder.build_optim_dec(args, model, checkpoint)
optim = [optim_bert, optim_dec]
else:
optim = [model_builder.build_optim(args, model, checkpoint)]
# Prepare loss
tokenizer = BertTokenizer.from_pretrained('bert-base-uncased',
do_lower_case=True,
cache_dir=args.temp_dir)
symbols = {
'BOS': tokenizer.vocab['[unused0]'],
'EOS': tokenizer.vocab['[unused1]'],
'PAD': tokenizer.vocab['[PAD]'],
'EOQ': tokenizer.vocab['[unused2]']
}
# Prepare loss computation
valid_loss = abs_loss(model.generator,
symbols,
model.vocab_size,
device,
train=False)
# Prepare trainer and perform validation
if (args.meta_mode):
trainer = build_MTLtrainer(args, device_id, model, optim, optims_inner,
valid_loss)
stats = trainer.validate(valid_iter_fct, step)
else:
trainer = build_trainer(args, device_id, model, None, valid_loss)
stats = trainer.validate(valid_iter, step)
return stats.xent()
from data.data_loader import load_dataset
# Training settings
eval_iter = 10
seed = 0
gpu = 0
outf = "test_sde"
droprate = 0.1
batch_size = 512
target_scale = 10.939756
Iter_test = 100
tf.random.set_seed(seed)
X_train, y_train, X_test, y_test = load_dataset("MSD")
in_test_ds = (
tf.data.Dataset.from_tensor_slices((X_train, y_train))
.batch(batch_size)
.prefetch(buffer_size=tf.data.experimental.AUTOTUNE)
)
X_out = load_dataset("boston")
path = "sde_net.h5"
model = tf.keras.models.load_model(path, compile=False)
def mse(y, mean):
import tensorflow_addons as tfa
from warnings import filterwarnings
filterwarnings("ignore")
epochs = 2
lr = 1e-4
lr2 = 0.01
seed = 0
batch_size = 128
target_scale = 10.939756
# Data
print("==> Preparing data..")
tf.random.set_seed(seed)
X_train, y_train, X_test, y_test = data_loader.load_dataset("MSD")
train_ds = (tf.data.Dataset.from_tensor_slices(
(X_train, y_train)).batch(batch_size).prefetch(
buffer_size=tf.data.experimental.AUTOTUNE))
test_ds = (tf.data.Dataset.from_tensor_slices(
(X_train, y_train)).batch(batch_size).prefetch(
buffer_size=tf.data.experimental.AUTOTUNE))
# Model
print("==> Building model..")
model = SDENet(4)
real_label = 0 # training data
fake_label = 1 # out-of-distribution data