def main(args):
"""
run inference for SS-VAE
:param args: arguments for SS-VAE
:return: None
"""
if args.seed is not None:
pyro.set_rng_seed(args.seed)
# batch_size: number of images (and labels) to be considered in a batch
ss_vae = SSVAE(z_dim=args.z_dim,
hidden_layers=args.hidden_layers,
use_cuda=args.cuda,
config_enum=args.enum_discrete,
aux_loss_multiplier=args.aux_loss_multiplier)
# setup the optimizer
adam_params = {"lr": args.learning_rate, "betas": (args.beta_1, 0.999)}
optimizer = Adam(adam_params)
# set up the loss(es) for inference. wrapping the guide in config_enumerate builds the loss as a sum
# by enumerating each class label for the sampled discrete categorical distribution in the model
guide_enum = config_enumerate(guide, args.enum_discrete, expand=True)
elbo = (JitTraceEnum_ELBO if args.jit else TraceEnum_ELBO)(max_plate_nesting=1)
loss_basic = SVI(model, guide_enum, optimizer, loss=elbo)
# build a list of all losses considered
losses = [loss_basic]
# aux_loss: whether to use the auxiliary loss from NIPS 14 paper (Kingma et al)
if args.aux_loss:
elbo = JitTrace_ELBO() if args.jit else Trace_ELBO()
loss_aux = SVI(ss_vae.model_classify, ss_vae.guide_classify, optimizer, loss=elbo)
losses.append(loss_aux)
try:
# setup the logger if a filename is provided
logger = open(args.logfile, "w") if args.logfile else None
data_loaders = setup_data_loaders(MNISTCached, args.cuda, args.batch_size, sup_num=args.sup_num)
# how often would a supervised batch be encountered during inference
# e.g. if sup_num is 3000, we would have every 16th = int(50000/3000) batch supervised
# until we have traversed through the all supervised batches
periodic_interval_batches = int(MNISTCached.train_data_size / (1.0 * args.sup_num))
# number of unsupervised examples
unsup_num = MNISTCached.train_data_size - args.sup_num
# initializing local variables to maintain the best validation accuracy
# seen across epochs over the supervised training set
# and the corresponding testing set and the state of the networks
best_valid_acc, corresponding_test_acc = 0.0, 0.0
# WL: added. =====
print_and_log(logger, args)
print_and_log(logger, "\nepoch\t"+"elbo(sup)\t"+"elbo(unsup)\t"+"time(sec)")
times = [time.time()]
# ================
# run inference for a certain number of epochs
for i in range(0, args.num_epochs):
# get the losses for an epoch
epoch_losses_sup, epoch_losses_unsup = \
run_inference_for_epoch(data_loaders, losses, periodic_interval_batches)
# compute average epoch losses i.e. losses per example
avg_epoch_losses_sup = map(lambda v: v / args.sup_num, epoch_losses_sup)
avg_epoch_losses_unsup = map(lambda v: v / unsup_num, epoch_losses_unsup)
# store the loss and validation/testing accuracies in the logfile
# WL: edited. =====
# str_loss_sup = " ".join(map(str, avg_epoch_losses_sup))
# str_loss_unsup = " ".join(map(str, avg_epoch_losses_unsup))
# str_print = "{} epoch: avg losses {}".format(i, "{} {}".format(str_loss_sup, str_loss_unsup))
times.append(time.time())
str_elbo_sup = " ".join(map(lambda v: f"{-v:.4f}", avg_epoch_losses_sup))
str_elbo_unsup = " ".join(map(lambda v: f"{-v:.4f}", avg_epoch_losses_unsup))
str_print = f"{i:06d}\t"\
f"{str_elbo_sup}\t"\
f"{str_elbo_unsup}\t"\
f"{times[-1]-times[-2]:.3f}"
# =================
validation_accuracy = get_accuracy(data_loaders["valid"], ss_vae.classifier, args.batch_size)
# WL: commented. =====
# str_print += " validation accuracy {}".format(validation_accuracy)
# ====================
# this test accuracy is only for logging, this is not used
# to make any decisions during training
test_accuracy = get_accuracy(data_loaders["test"], ss_vae.classifier, args.batch_size)
# WL: commented. =====
# str_print += " test accuracy {}".format(test_accuracy)
# ====================
# update the best validation accuracy and the corresponding
# testing accuracy and the state of the parent module (including the networks)
if best_valid_acc < validation_accuracy:
best_valid_acc = validation_accuracy
corresponding_test_acc = test_accuracy
print_and_log(logger, str_print)
final_test_accuracy = get_accuracy(data_loaders["test"], ss_vae.classifier, args.batch_size)
# WL: commented. =====
# print_and_log(logger, "best validation accuracy {} corresponding testing accuracy {} "
# "last testing accuracy {}".format(best_valid_acc, corresponding_test_acc, final_test_accuracy))
# ====================
finally:
# close the logger file object if we opened it earlier
if args.logfile:
logger.close()
def main(args):
"""
run inference for CVAE
:param args: arguments for CVAE
:return: None
"""
if args.seed is not None:
set_seed(args.seed, args.cuda)
if os.path.exists('cvae.model.pt'):
print('Loading model %s' % 'cvae.model.pt')
cvae = torch.load('cvae.model.pt')
else:
cvae = CVAE(z_dim=args.z_dim,
y_dim=8,
x_dim=32612,
hidden_dim=args.hidden_dimension,
use_cuda=args.cuda)
print(cvae)
# setup the optimizer
adam_params = {
"lr": args.learning_rate,
"betas": (args.beta_1, 0.999),
"clip_norm": 0.5
}
optimizer = ClippedAdam(adam_params)
guide = config_enumerate(cvae.guide, args.enum_discrete)
# set up the loss for inference.
loss = SVI(cvae.model,
guide,
optimizer,
loss=TraceEnum_ELBO(max_iarange_nesting=1))
try:
# setup the logger if a filename is provided
logger = open(args.logfile, "w") if args.logfile else None
data_loaders = setup_data_loaders(NHANES, args.cuda, args.batch_size)
print(len(data_loaders['train']))
print(len(data_loaders['test']))
print(len(data_loaders['valid']))
# initializing local variables to maintain the best validation acc
# seen across epochs over the supervised training set
# and the corresponding testing set and the state of the networks
best_valid_err, best_test_err = float('inf'), float('inf')
# run inference for a certain number of epochs
for i in range(0, args.num_epochs):
# get the losses for an epoch
epoch_losses = \
run_inference_for_epoch(args.batch_size, data_loaders, loss, args.cuda)
# compute average epoch losses i.e. losses per example
avg_epoch_losses = epoch_losses / NHANES.train_size
# store the losses in the logfile
str_loss = str(avg_epoch_losses)
str_print = "{} epoch: avg loss {}".format(i,
"{}".format(str_loss))
validation_err = get_accuracy(data_loaders["valid"],
cvae.sim_measurements)
str_print += " validation error {}".format(validation_err)
# this test accuracy is only for logging, this is not used
# to make any decisions during training
test_еrr = get_accuracy(data_loaders["test"],
cvae.sim_measurements)
str_print += " test error {}".format(test_еrr)
# update the best validation accuracy and the corresponding
# testing accuracy and the state of the parent module (including the networks)
if best_valid_err > validation_err:
best_valid_err = validation_err
if best_test_err > test_еrr:
best_test_err = test_еrr
print_and_log(logger, str_print)
final_test_accuracy = get_accuracy(data_loaders["test"],
cvae.sim_measurements)
print_and_log(
logger, "best validation error {} corresponding testing error {} "
"last testing error {}".format(best_valid_err, best_test_err,
final_test_accuracy))
torch.save(cvae, 'cvae.model.pt')
#mu, sigma, actuals, lods, masks = get_predictions(data_loaders["prediction"], cvae.sim_measurements)
#torch.save((mu, sigma, actuals, lods, masks), 'cvae.predictions.pt')
finally:
# close the logger file object if we opened it earlier
if args.logfile:
logger.close()
def main():
with WUST_TRAIN_PATH.open('rb') as file:
wust_train = pickle.load(file)
with WUST_TEST_PATH.open('rb') as file:
wust_test = pickle.load(file)
with WEAK_SMALL_ENCODED_PATH.open('rb') as file:
weak_small_x = pickle.load(file)
weak_small_x = torch.FloatTensor(weak_small_x)
wust_train_x = torch.FloatTensor(wust_train[1])
wust_test_x = torch.FloatTensor(wust_test[1])
wust_train_labels = wust_train[2]
wust_test_labels = wust_test[2]
label_encoder = LabelEncoder()
label_encoder.fit(wust_train_labels)
wust_train_y = label_encoder.transform(wust_train_labels)
wust_test_y = label_encoder.transform(wust_test_labels)
wust_train_y = torch.LongTensor(wust_train_y.reshape(-1, 1))
wust_test_y = torch.LongTensor(wust_test_y.reshape(-1, 1))
wust_train_ds = TensorDataset(wust_train_x, wust_train_y)
wust_train_dl = DataLoader(wust_train_ds,
batch_size=BATCH_SIZE,
shuffle=True)
wust_test_ds = TensorDataset(wust_test_x, wust_test_y)
wust_test_dl = DataLoader(wust_test_ds,
batch_size=BATCH_SIZE,
shuffle=True)
weak_train_ds = TensorDataset(weak_small_x)
weak_train_dl = DataLoader(weak_train_ds,
batch_size=BATCH_SIZE,
shuffle=True)
pyro.set_rng_seed(SEED)
# batch_size: number of images (and labels) to be considered in a batch
ss_vae = SSVAE(input_size=wust_train_x.shape[1],
output_size=len(label_encoder.classes_),
z_dim=Z_DIM,
hidden_layers=HIDDEN_LAYERS,
use_cuda=CUDA,
config_enum=ENUM_DISCRETE,
aux_loss_multiplier=AUX_LOSS_MULTIPLIER)
# setup the optimizer
optimizer = Adam({"lr": LEARNING_RATE, "betas": (BETA_1, 0.999)})
# set up the loss(es) for inference. wrapping the guide in config_enumerate builds the loss as a sum
# by enumerating each class label for the sampled discrete categorical distribution in the model
guide = config_enumerate(ss_vae.guide, ENUM_DISCRETE, expand=True)
Elbo = JitTraceEnum_ELBO if USE_JIT else TraceEnum_ELBO
elbo = Elbo(max_plate_nesting=1, strict_enumeration_warning=False)
loss_basic = SVI(ss_vae.model, guide, optimizer, loss=elbo)
# build a list of all losses considered
losses = [loss_basic]
# aux_loss: whether to use the auxiliary loss from NIPS 14 paper (Kingma et al)
if AUX_LOSS:
elbo = JitTrace_ELBO() if USE_JIT else Trace_ELBO()
loss_aux = SVI(ss_vae.model_classify,
ss_vae.guide_classify,
optimizer,
loss=elbo)
losses.append(loss_aux)
try:
# setup the logger if a filename is provided
logger = open(LOGFILE, "w") if LOGFILE else None
# data_loaders = setup_data_loaders(MNISTCached, CUDA, BATCH_SIZE, sup_num=SUP_NUM)
# how often would a supervised batch be encountered during inference
# e.g. if sup_num is 3000, we would have every 16th = int(50000/3000) batch supervised
# until we have traversed through the all supervised batches
# number of unsupervised examples
sup_num = len(wust_train_ds)
unsup_num = len(weak_train_ds)
periodic_interval_batches = int(unsup_num / sup_num)
# initializing local variables to maintain the best validation accuracy
# seen across epochs over the supervised training set
# and the corresponding testing set and the state of the networks
best_valid_acc, corresponding_test_acc = 0.0, 0.0
# run inference for a certain number of epochs
for i in range(0, NUM_EPOCHS):
# get the losses for an epoch
epoch_losses_sup, epoch_losses_unsup = run_inference_for_epoch(
wust_train_dl, weak_train_dl, losses,
periodic_interval_batches)
# compute average epoch losses i.e. losses per example
avg_epoch_losses_sup = map(lambda v: v / sup_num, epoch_losses_sup)
avg_epoch_losses_unsup = map(lambda v: v / unsup_num,
epoch_losses_unsup)
# store the loss and validation/testing accuracies in the logfile
str_loss_sup = " ".join(map(str, avg_epoch_losses_sup))
str_loss_unsup = " ".join(map(str, avg_epoch_losses_unsup))
str_print = f"{i} epoch: avg losses {str_loss_sup} {str_loss_unsup}"
# validation_accuracy = get_accuracy(data_loaders["valid"], ss_vae.classifier, BATCH_SIZE)
# str_print += " validation accuracy {}".format(validation_accuracy)
# this test accuracy is only for logging, this is not used
# to make any decisions during training
test_accuracy = get_accuracy(wust_test_dl, ss_vae.classifier,
BATCH_SIZE)
str_print += " test accuracy {}".format(test_accuracy)
# update the best validation accuracy and the corresponding
# testing accuracy and the state of the parent module (including the networks)
if best_valid_acc < test_accuracy:
best_valid_acc = test_accuracy
corresponding_test_acc = test_accuracy
print_and_log(logger, str_print)
final_test_accuracy = get_accuracy(wust_test_dl, ss_vae.classifier,
BATCH_SIZE)
print_and_log(
logger,
f"best validation accuracy {best_valid_acc} corresponding testing accuracy {corresponding_test_acc} "
f"last testing accuracy {final_test_accuracy}")
finally:
if LOGFILE:
logger.close()
def main():
"""
run inference for SS-VAE
:param args: arguments for SS-VAE
:return: None
"""
pyro.set_rng_seed(12345)
cuda = True
# batch_size: number of images (and labels) to be considered in a batch
ss_vae = TextSSVAE(embed_dim=300,
z_dim=300,
kernels=[3, 4, 5],
filters=[100, 100, 100],
hidden_size=300,
num_rnn_layers=1,
config_enum="parallel",
use_cuda=cuda,
aux_loss_multiplier=46)
ss_vae = ss_vae.cuda()
try:
pyro.get_param_store().load('pyro_param_store.store')
print(
'successfully loaded param store, remove file from directory if undesired'
)
except Exception:
print("failed to load param store, starting over")
try:
ss_vae.load_state_dict(torch.load('ss_vae_model.pth'))
print(
'successfully loaded model parameters, remove file from directory if undesired'
)
except Exception:
print("failed to load model parameters")
# setup the optimizer
adam_params = {"lr": 1e-4, "betas": (0.9, 0.999), "weight_decay": 0.01}
optimizer = Adam(adam_params)
# set up the loss(es) for inference. wrapping the guide in config_enumerate builds the loss as a sum
# by enumerating each class label for the sampled discrete categorical distribution in the model
jit = False
guide = config_enumerate(ss_vae.guide, "parallel", expand=True)
elbo = (JitTraceEnum_ELBO if jit else TraceEnum_ELBO)()
loss_basic = SVI(ss_vae.model, guide, optimizer, loss=elbo)
# build a list of all losses considered
losses = [loss_basic]
# aux_loss: whether to use the auxiliary loss from NIPS 14 paper (Kingma et al)
aux_loss = True
if aux_loss:
elbo = JitTrace_ELBO() if jit else Trace_ELBO()
loss_aux = SVI(ss_vae.model_classify,
ss_vae.guide_classify,
optimizer,
loss=elbo)
losses.append(loss_aux)
batch_size = 32
valid_num = 100
train_data_size = 3409
sup_num = 1163
try:
# setup the logger if a filename is provided
logger = open('./tmp.log', "w") if './tmp.log' else None
data_loaders = setup_data_loaders(IMDBCached,
cuda,
batch_size=32,
sup_num=valid_num)
# how often would a supervised batch be encountered during inference
# e.g. if sup_num is 3000, we would have every 16th = int(50000/3000) batch supervised
# until we have traversed through the all supervised batches
periodic_interval_batches = int(train_data_size / (1.0 * sup_num))
# number of unsupervised examples
unsup_num = train_data_size - sup_num
# initializing local variables to maintain the best validation accuracy
# seen across epochs over the supervised training set
# and the corresponding testing set and the state of the networks
best_valid_acc, corresponding_test_acc = 0.0, 0.0
# run inference for a certain number of epochs
num_epochs = 200
sup_loss_log = []
unsup_loss_log = []
for i in range(0, num_epochs):
# get the losses for an epoch
epoch_losses_sup, epoch_losses_unsup = \
run_inference_for_epoch(data_loaders, losses, periodic_interval_batches)
# compute average epoch losses i.e. losses per example
avg_epoch_losses_sup = map(lambda v: v / sup_num, epoch_losses_sup)
avg_epoch_losses_unsup = map(lambda v: v / unsup_num,
epoch_losses_unsup)
sup_loss_log.append(avg_epoch_losses_sup)
unsup_loss_log.append(avg_epoch_losses_unsup)
# store the loss and validation/testing accuracies in the logfile
str_loss_sup = " ".join(map(str, avg_epoch_losses_sup))
str_loss_unsup = " ".join(map(str, avg_epoch_losses_unsup))
str_print = "{} epoch: avg losses {}".format(
i, "{} {}".format(str_loss_sup, str_loss_unsup))
ss_vae.eval()
validation_accuracy = get_accuracy(data_loaders["valid"],
ss_vae.classifier, batch_size)
str_print += " validation accuracy {}".format(validation_accuracy)
# this test accuracy is only for logging, this is not used
# to make any decisions during training
test_accuracy = get_accuracy(data_loaders["test"],
ss_vae.classifier, batch_size)
str_print += " test accuracy {}".format(test_accuracy)
ss_vae.train()
torch.save(ss_vae.state_dict(), 'ss_vae_model.pth')
pyro.get_param_store().save('pyro_param_store.store')
# update the best validation accuracy and the corresponding
# testing accuracy and the state of the parent module (including the networks)
if best_valid_acc < validation_accuracy:
best_valid_acc = validation_accuracy
corresponding_test_acc = test_accuracy
if i % 10 == 0:
neg_sentences, neg_bleu = generateSentences(
data_loaders["test"],
ss_vae.model,
ss_vae.w2v_model,
sentiment=0)
pos_sentences, pos_bleu = generateSentences(
data_loaders["test"],
ss_vae.model,
ss_vae.w2v_model,
sentiment=1)
str_print += " neg_bleu {}".format(neg_bleu)
str_print += " pos_bleu {}".format(pos_bleu)
pd.DataFrame.from_dict(pos_sentences).to_csv(
'positive_sentences.csv', encoding='utf-8')
pd.DataFrame.from_dict(neg_sentences).to_csv(
'negative_sentences.csv', encoding='utf-8')
cond_neg_sentences, neg_bleu = generateSentences(
data_loaders["test"],
ss_vae.conditioned_generation,
ss_vae.w2v_model,
sentiment=0)
cond_pos_sentences, pos_bleu = generateSentences(
data_loaders["test"],
ss_vae.conditioned_generation,
ss_vae.w2v_model,
sentiment=1)
pd.DataFrame.from_dict(cond_pos_sentences).to_csv(
'cond_positive_sentences.csv', encoding='utf-8')
pd.DataFrame.from_dict(cond_neg_sentences).to_csv(
'cond_negative_sentences.csv', encoding='utf-8')
str_print += " cond_neg_bleu {}".format(neg_bleu)
str_print += " cond_pos_bleu {}".format(pos_bleu)
print_and_log(logger, str_print)
np.save("avg_loss_sup", np.asarray(sup_loss_log))
np.save("avg_loss_unsup", np.asarray(unsup_loss_log))
ss_vae.eval()
final_test_accuracy = get_accuracy(data_loaders["test"],
ss_vae.classifier, batch_size)
print_and_log(
logger,
"best validation accuracy {} corresponding testing accuracy {} "
"last testing accuracy {}".format(best_valid_acc,
corresponding_test_acc,
final_test_accuracy))
finally:
# close the logger file object if we opened it earlier
logfile = True
if logfile:
logger.close()
def main(args):
"""
run inference for CVAE
:param args: arguments for CVAE
:return: None
"""
if args.seed is not None:
set_seed(args.seed, args.cuda)
# batch_size: number of images (and labels) to be considered in a batch
cvae = CVAE(z_dim=args.z_dim,
hidden_dim=args.hidden_dimension,
use_cuda=args.cuda)
# setup the optimizer
adam_params = {"lr": args.learning_rate, "betas": (args.beta_1, 0.999)}
optimizer = ClippedAdam(adam_params)
# set up the loss for inference.
guide = config_enumerate(cvae.guide, args.enum_discrete)
loss = SVI(cvae.model, guide, optimizer, loss=TraceEnum_ELBO(max_iarange_nesting=1))
try:
# setup the logger if a filename is provided
logger = open(args.logfile, "w") if args.logfile else None
data_loaders = setup_data_loaders(NHANES, args.cuda, args.batch_size)
# initializing local variables to maintain the best validation acc
# seen across epochs over the supervised training set
# and the corresponding testing set and the state of the networks
best_valid_acc, corresponding_test_acc = 0.0, 0.0
# run inference for a certain number of epochs
for i in range(0, args.num_epochs):
# get the losses for an epoch
epoch_losses = \
run_inference_for_epoch(data_loaders, loss)
# compute average epoch losses i.e. losses per example
avg_epoch_losses = epoch_losses / NHANES.train_data_size
# store the losses in the logfile
str_loss = str(avg_epoch_losses)
str_print = "{} epoch: avg loss {}".format(i, "{}".format(str_loss))
validation_accuracy = get_accuracy(data_loaders["valid"], cvae.sim_measurements, args.batch_size)
str_print += " validation accuracy {}".format(validation_accuracy)
# this test accuracy is only for logging, this is not used
# to make any decisions during training
test_accuracy = get_accuracy(data_loaders["test"], cvae.sim_measurements, args.batch_size)
str_print += " test accuracy {}".format(test_accuracy)
# update the best validation accuracy and the corresponding
# testing accuracy and the state of the parent module (including the networks)
if best_valid_acc < validation_accuracy:
best_valid_acc = validation_accuracy
corresponding_test_acc = test_accuracy
print_and_log(logger, str_print)
final_test_accuracy = get_accuracy(data_loaders["test"], cvae.sim_measurements, args.batch_size)
print_and_log(logger, "best validation accuracy {} corresponding testing accuracy {} "
"last testing accuracy {}".format(best_valid_acc, corresponding_test_acc,
final_test_accuracy))
finally:
# close the logger file object if we opened it earlier
if args.logfile:
logger.close()
def main():
"""
run inference for SS-VAE
:param args: arguments for SS-VAE
:return: None
"""
pyro.set_rng_seed(12345)
cuda = True
# batch_size: number of images (and labels) to be considered in a batch
ss_vae = SSVAE(
z_dim=50,
hidden_layers=[500],
use_cuda=cuda,
config_enum="sequential", #no idea
aux_loss_multiplier=46)
# setup the optimizer
adam_params = {"lr": 0.00042, "betas": (0.9, 0.999)}
optimizer = Adam(adam_params)
# set up the loss(es) for inference. wrapping the guide in config_enumerate builds the loss as a sum
# by enumerating each class label for the sampled discrete categorical distribution in the model
jit = False
guide = config_enumerate(ss_vae.guide, "sequential") #, expand=True)
elbo = (JitTraceEnum_ELBO if jit else TraceEnum_ELBO)()
loss_basic = SVI(ss_vae.model, guide, optimizer, loss=elbo)
# build a list of all losses considered
losses = [loss_basic]
# aux_loss: whether to use the auxiliary loss from NIPS 14 paper (Kingma et al)
aux_loss = True
if aux_loss:
elbo = JitTrace_ELBO() if jit else Trace_ELBO()
loss_aux = SVI(ss_vae.model_classify,
ss_vae.guide_classify,
optimizer,
loss=elbo)
losses.append(loss_aux)
batch_size = 200
sup_num = 3000
try:
# setup the logger if a filename is provided
logger = open('./tmp.log', "w") if './tmp.log' else None
data_loaders = setup_data_loaders(MNISTCached,
cuda,
batch_size,
sup_num=sup_num)
# how often would a supervised batch be encountered during inference
# e.g. if sup_num is 3000, we would have every 16th = int(50000/3000) batch supervised
# until we have traversed through the all supervised batches
periodic_interval_batches = int(MNISTCached.train_data_size /
(1.0 * sup_num))
# number of unsupervised examples
unsup_num = MNISTCached.train_data_size - sup_num
# initializing local variables to maintain the best validation accuracy
# seen across epochs over the supervised training set
# and the corresponding testing set and the state of the networks
best_valid_acc, corresponding_test_acc = 0.0, 0.0
# run inference for a certain number of epochs
num_epochs = 10
for i in range(0, num_epochs):
# get the losses for an epoch
epoch_losses_sup, epoch_losses_unsup = \
run_inference_for_epoch(data_loaders, losses, periodic_interval_batches)
# compute average epoch losses i.e. losses per example
avg_epoch_losses_sup = map(lambda v: v / sup_num, epoch_losses_sup)
avg_epoch_losses_unsup = map(lambda v: v / unsup_num,
epoch_losses_unsup)
# store the loss and validation/testing accuracies in the logfile
str_loss_sup = " ".join(map(str, avg_epoch_losses_sup))
str_loss_unsup = " ".join(map(str, avg_epoch_losses_unsup))
str_print = "{} epoch: avg losses {}".format(
i, "{} {}".format(str_loss_sup, str_loss_unsup))
validation_accuracy = get_accuracy(data_loaders["valid"],
ss_vae.classifier, batch_size)
str_print += " validation accuracy {}".format(validation_accuracy)
# this test accuracy is only for logging, this is not used
# to make any decisions during training
test_accuracy = get_accuracy(data_loaders["test"],
ss_vae.classifier, batch_size)
str_print += " test accuracy {}".format(test_accuracy)
# update the best validation accuracy and the corresponding
# testing accuracy and the state of the parent module (including the networks)
if best_valid_acc < validation_accuracy:
best_valid_acc = validation_accuracy
corresponding_test_acc = test_accuracy
print_and_log(logger, str_print)
final_test_accuracy = get_accuracy(data_loaders["test"],
ss_vae.classifier, batch_size)
print_and_log(
logger,
"best validation accuracy {} corresponding testing accuracy {} "
"last testing accuracy {}".format(best_valid_acc,
corresponding_test_acc,
final_test_accuracy))
finally:
# close the logger file object if we opened it earlier
logfile = True
if logfile:
logger.close()
def training(args, rel_embeddings, word_embeddings):
if args.seed is not None:
pyro.set_rng_seed(args.seed)
# CUDA for PyTorch
cuda_available = torch.cuda.is_available()
if (cuda_available and args.cuda):
device = torch.device("cuda")
torch.cuda.set_device(0)
print("using gpu acceleration")
print("Generating Config")
config = Config(
word_embeddings=torch.FloatTensor(word_embeddings),
decoder_hidden_dim=args.decoder_hidden_dim,
num_relations=7,
encoder_hidden_dim=args.encoder_hidden_dim,
num_predicates=1000,
batch_size=args.batch_size
)
# initialize the generator model
generator = SimpleGenerator(config)
# setup the optimizer
adam_params = {"lr": args.learning_rate, "betas": (args.beta_1, 0.999)}
optimizer = ClippedAdam(adam_params)
# set up the loss(es) for inference. wrapping the guide in config_enumerate builds the loss as a sum
# by enumerating each class label for the sampled discrete categorical distribution in the model
if args.enumerate:
guide = config_enumerate(generator.guide, args.enum_discrete, expand=True)
else:
guide = generator.guide
elbo = (JitTraceEnum_ELBO if args.jit else TraceEnum_ELBO)(max_plate_nesting=1)
loss_basic = SVI(generator.model, guide, optimizer, loss=elbo)
# build a list of all losses considered
losses = [loss_basic]
# aux_loss: whether to use the auxiliary loss from NIPS 14 paper (Kingma et al)
if args.aux_loss:
elbo = JitTrace_ELBO() if args.jit else Trace_ELBO()
loss_aux = SVI(generator.model_identify, generator.guide_identify, optimizer, loss=elbo)
losses.append(loss_aux)
# prepare data
real_model = RealModel(rel_embeddings, word_embeddings)
data = real_model.generate_data()
sup_train_set = data[:100]
unsup_train_set = data[100:700]
eval_set = data[700:900]
test_set = data[900:]
data_loaders = setup_data_loaders(sup_train_set,
unsup_train_set,
eval_set,
test_set,
batch_size=args.batch_size)
num_train = len(sup_train_set) + len(unsup_train_set)
num_eval = len(eval_set)
num_test = len(test_set)
# how often would a supervised batch be encountered during inference
# e.g. if sup_num is 3000, we would have every 16th = int(50000/3000) batch supervised
# until we have traversed through the all supervised batches
periodic_interval_batches = int(1.0 * num_train / len(sup_train_set))
# setup the logger if a filename is provided
log_fn = "./logs/" + args.experiment_type + '/' + args.experiment_name + '.log'
logger = open(log_fn, "w")
# run inference for a certain number of epochs
for i in tqdm(range(0, args.num_epochs)):
# get the losses for an epoch
epoch_losses_sup, epoch_losses_unsup = \
train_epoch(data_loaders=data_loaders,
models=losses,
periodic_interval_batches=periodic_interval_batches)
# compute average epoch losses i.e. losses per example
avg_epoch_losses_sup = map(lambda v: v / len(sup_train_set), epoch_losses_sup)
avg_epoch_losses_unsup = map(lambda v: v / len(unsup_train_set), epoch_losses_unsup)
# store the loss and validation/testing accuracies in the logfile
str_loss_sup = " ".join(map(str, avg_epoch_losses_sup))
str_loss_unsup = " ".join(map(str, avg_epoch_losses_unsup))
str_print = "{} epoch: avg losses {}".format(i, "{} {}".format(str_loss_sup, str_loss_unsup))
print_and_log(logger, str_print)
# save trained models
torch.save(generator.state_dict(), './models/test_generator_state_dict.pth')
return generator
best_valid_kappa, cor_test_kappa, cor_epoch_1 = 0.0, 0.0, 0
best_test_kappa, cor_valid_kappa, cor_epoch_2 = 0.0, 0.0, 0
for epoch in range(num_epochs):
ssvae.eval()
epoch_losses_sup, epoch_losses_aux, epoch_losses_unsup = get_evaluate_for_epoch(
train_dataloader, test_dataloader, loss_basic, loss_aux, use_cuda)
epoch_losses_sup = epoch_losses_sup / len(train_dataset)
epoch_losses_aux = epoch_losses_aux / len(train_dataset)
epoch_losses_unsup = epoch_losses_unsup / len(test_dataset)
ratio = np.abs(epoch_losses_sup / epoch_losses_aux)
str_print = 'Epoch {0} (eval mode): epoch_losses_sup, {1:.3f}; epoch_losses_aux, {2:.3f}; ' \
'epoch_losses_unsup, {3:.3f}; ratio, {4:.4f}'\
.format(epoch, epoch_losses_sup, epoch_losses_aux, epoch_losses_unsup, ratio)
print_and_log(logger, str_print)
writer.add_scalar('Eval mode/epoch_losses_sup', epoch_losses_sup, epoch)
writer.add_scalar('Eval mode/epoch_losses_aux', epoch_losses_aux, epoch)
writer.add_scalar('Eval mode/epoch_losses_unsup', epoch_losses_unsup,
epoch)
writer.add_scalar('Eval mode/ratio', ratio, epoch)
'''Train'''
ssvae.train()
if epoch_losses_aux < 0.0012 and ratio > 400000:
epoch_losses_sup, epoch_losses_unsup = run_inference_for_epoch_ncls(
train_dataloader, test_dataloader, loss_basic, use_cuda)
# elif ratio>10000:
# epoch_losses_sup, epoch_losses_unsup = run_inference_for_epoch(train_dataloader, test_dataloader,
# loss, use_cuda, ratio * factor)
else:
