def train_ae(dataloaders_dict, device=0, num_epochs=5):
ae = AutoEncoder()
ae = ae.to(device)
distance = nn.MSELoss()
optimizer = optim.Adam(ae.parameters(), lr=0.001)
for epoch in range(num_epochs):
print('\nEpoch {}/{}'.format(epoch, num_epochs - 1))
print('-' * 10)
for phase in ["train", "val", "test"]:
if phase == "train":
ae.train() # Set ae to training mode
else:
ae.eval()
for data in dataloaders_dict[phase]:
inputs, _ = data
inputs = inputs.to(device)
with torch.set_grad_enabled(phase == "train"):
output = ae(inputs)
loss = distance(output, inputs)
optimizer.zero_grad()
if phase == "train":
loss.backward()
optimizer.step()
print("{} Loss: {:.4f}".format(phase, loss.item()))
return ae
def train_ae(encoder_sizes,
decoder_sizes,
train_loader,
num_epochs=1,
bw=False):
"""
Pre-train Autoencoder on condition sketches
:return trained encoder nn.Module
"""
AE = AutoEncoder(encoder_sizes, decoder_sizes, bw=bw).to(device)
optimizer = torch.optim.Adam(AE.parameters(), lr=0.0005, weight_decay=3e-6)
for epoch in tqdm(range(num_epochs), "Encoder pretraining"):
epoch_loss = 0
for batch, (sketch, real,
label) in enumerate(tqdm(dataloader_train, "Batch")):
optimizer.zero_grad()
sketch, real, label = sketch.to(device), real.to(device), label.to(
device)
recon = AE(sketch)
loss = torch.mean((sketch - recon)**2)
loss.backward()
epoch_loss += loss.item() / len(dataloader_train)
optimizer.step()
print("AutoEncoder pretraining: Epoch {} Loss: {}".format(
epoch, epoch_loss))
del AE.decoder
optimizer.zero_grad()
del optimizer
return AE.encoder
def main(options):
if options.num_classes == 2:
TRAINING_PATH = 'train_2classes.txt'
else:
TRAINING_PATH = 'train.txt'
IMG_PATH = '/Users/waz/JHU/CV-ADNI/ImageNoSkull'
dset_train = AD_3DRandomPatch(IMG_PATH, TRAINING_PATH)
train_loader = DataLoader(dset_train,
batch_size=options.batch_size,
shuffle=True,
num_workers=4,
drop_last=True)
sparsity = 0.05
beta = 0.5
mean_square_loss = nn.MSELoss()
kl_div_loss = nn.KLDivLoss()
use_gpu = len(options.gpuid) >= 1
autoencoder = AutoEncoder()
autoencoder = autoencoder.cpu()
optimizer = torch.optim.Adam(autoencoder.parameters(),
lr=options.learning_rate,
weight_decay=options.weight_decay)
train_loss = 0.
for epoch in range(options.epochs):
print("At {0}-th epoch.".format(epoch))
for i, patches in enumerate(train_loader):
for b, batch in enumerate(patches):
batch = Variable(batch)
output, mean_activitaion = autoencoder(batch)
loss1 = mean_square_loss(output, batch)
loss2 = kl_div_loss(mean_activitaion,
Variable(torch.Tensor([sparsity])))
print("loss1", loss1)
print("loss2", loss2)
loss = loss1 + loss2
train_loss += loss
logging.info(
"batch {0} training loss is : {1:.5f}, {1:.5f}".format(
b, loss1.data[0], loss2.data[0]))
optimizer.zero_grad()
loss.backward()
optimizer.step()
train_avg_loss = train_loss / len(train_loader * 1000)
print(
"Average training loss is {0:.5f} at the end of epoch {1}".format(
train_avg_loss.data[0], epoch))
torch.save(model.state_dict(), open("autoencoder_model", 'wb'))
def main():
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
model = AutoEncoder(channels_list, latent_dim).to(device)
optimizer = optim.Adam(model.parameters(), lr=lr)
train_loader = torch.utils.data.DataLoader(datasets.MNIST(
'./data', train=True, download=True, transform=transforms.ToTensor()),
batch_size=batch_size,
shuffle=True)
test_loader = torch.utils.data.DataLoader(datasets.MNIST(
'./data', train=False, transform=transforms.ToTensor()),
batch_size=batch_size,
shuffle=True)
runner = AERunner(model, optimizer, train_loader, test_loader, device)
for epoch in range(1, epochs + 1):
runner.train(epoch)
runner.test(epoch)
with torch.no_grad():
sample = model.sample(80, device)
save_image(
sample, './results_ae_' + str(latent_dim) + '/sample_' +
str(epoch) + '.png')
def main(options):
if options.num_classes == 2:
TRAINING_PATH = 'train_2classes.txt'
else:
TRAINING_PATH = 'train.txt'
IMG_PATH = './Image'
dset_train = AD_3DRandomPatch(IMG_PATH, TRAINING_PATH)
train_loader = DataLoader(dset_train,
batch_size=options.batch_size,
shuffle=True,
num_workers=4,
drop_last=True)
sparsity = 0.05
beta = 0.5
mean_square_loss = nn.MSELoss()
kl_div_loss = nn.KLDivLoss(reduce=False)
use_gpu = len(options.gpuid) >= 1
autoencoder = AutoEncoder()
if (use_gpu):
autoencoder = autoencoder.cuda()
else:
autoencoder = autoencoder.cpu()
optimizer = torch.optim.Adam(autoencoder.parameters(),
lr=options.learning_rate,
weight_decay=options.weight_decay)
train_loss = 0.
for epoch in range(options.epochs):
print("At {0}-th epoch.".format(epoch))
for i, patches in enumerate(train_loader):
print(i)
print(len(patches))
def startLearning(bs,me,f,p,l):
#Init Tensorboard
writer = SummaryWriter()
batch_size = bs
max_epochs = me
factor = f
patience = p
lr = l
#Define batch size the number of epoch
print("load dataset")
#Load Dataset
training_loader = torch.utils.data.DataLoader(dataset=Dataset('training_dataset_pack.h5',"std_training.png","mean_training.png"), batch_size=batch_size, shuffle=True,num_workers=0)
validation_loader = torch.utils.data.DataLoader(dataset=Dataset('validation_dataset_pack.h5',"std_validation.png","mean_validation.png"), batch_size=batch_size, shuffle=True,num_workers=0)
print("Done")
#Make model
model = AutoEncoder(training_loader.dataset.getInputSize()).cuda()
#Define loss type
criterion_expressions = nn.CrossEntropyLoss().cuda()
criterion_landmarks = nn.MSELoss().cuda()
#Define the optimizer
optimizer = torch.optim.Adam(model.parameters(), lr=lr, betas=(0.9,0.999))
#Define the scheduler
scheduler = torch.optim.lr_scheduler.ReduceLROnPlateau(optimizer,mode='min',factor=factor,patience=patience)
best_loss = None
#Main loop (epoch)
for epoch in range(1,max_epochs+1):
is_best= False
print("Training...")
#Init progress bar for training
pbart = tqdm.tqdm(total=int(len(training_loader.dataset)/batch_size),postfix={"loss":None,"accuracy":None},desc="Epoch: {}/{}".format(epoch,max_epochs))
#Init metrics
loss = 0.
loss_lm = 0.
loss_expr = 0.
acc = 0.
val_loss = 0.
val_acc =0.
val_loss_lm = 0.
val_loss_expr = 0.
#Training loop
for i, data in enumerate(training_loader, 0):
#Zero the parameter gradients
optimizer.zero_grad()
#Get the inputs
images, landmarks, expressions = data
images = images.to(device)
landmarks = landmarks.to(device).float()
expressions = expressions.to(device).long()
#Get the outputs
outputs = model(images)
#Calculate metrics
#Loss
loss_landmarks = criterion_landmarks(outputs[0], landmarks.float())
loss_expressions = criterion_expressions(outputs[1], expressions)
current_loss = loss_landmarks + loss_expressions
loss_expr += loss_expressions.item()
loss_lm += loss_landmarks.item()
loss += current_loss.item()
#Accuracy
_,predicted_expressions = torch.max(outputs[1],1)
acc += (predicted_expressions == expressions).sum().float()/batch_size
#Backpropagation
current_loss.backward()
#Reduce learning rate if we are on a plateu
optimizer.step()
#Update
pbart.update(1)
pbart.set_postfix({"loss": loss/(i+1),"e_loss": loss_expr/(i+1),"l_loss": loss_lm/(i+1),"acc_e":acc.item()/(i+1)})
pbart.close()
#Calculate metrics on one epoch
loss /= (len(training_loader.dataset)/batch_size)
loss_lm /= (len(validation_loader.dataset)/batch_size)
loss_expr /= (len(validation_loader.dataset)/batch_size)
acc /= (len(training_loader.dataset)/batch_size)
#Save metrics in a log file
with open("log/training_hourglass5.0.3.log","a") as f:
f.write("epoch: {} / {} loss: {} e_loss:{} l_loss: {} accuracy: {}\n".format(epoch,max_epochs,loss,loss_expr,loss_lm,acc))
f.close()
print("Validation...")
#Init progress bar for validation
pbarv = tqdm.tqdm(total=int(len(validation_loader.dataset)/batch_size),postfix={"loss":None,"accuracy":None},desc="Epoch: {}/{}".format(epoch,max_epochs))
#Validation loop
with torch.no_grad():
for i, data in enumerate(validation_loader, 0):
#Get the inputs
images, landmarks, expressions = data
images = images.to(device)
landmarks = landmarks.to(device).float()
expressions = expressions.to(device).long()
#Get the outputs
outputs = model(images)
#Calculate metrics
#Loss
loss_landmarks = criterion_landmarks(outputs[0], landmarks.float())
loss_expressions = criterion_expressions(outputs[1], expressions)
loss = loss_landmarks + loss_expressions
val_loss += loss.item()
val_loss_expr += loss_expressions.item()
val_loss_lm += loss_landmarks.item()
#Accuracy
_,predicted_expressions = torch.max(outputs[1],1)
val_acc += (predicted_expressions == expressions).sum().float()/batch_size
#Uptate validation progress bar
pbarv.update(1)
pbarv.set_postfix({"loss": val_loss/(i+1),"e_loss": val_loss_expr/(i+1),"l_loss": val_loss_lm/(i+1),"acc_e":val_acc.item()/(i+1)})
pbarv.close()
#Calculate metrics on one epoch
val_loss /= (len(validation_loader.dataset)/batch_size)
val_loss_lm /= (len(validation_loader.dataset)/batch_size)
val_loss_expr /= (len(validation_loader.dataset)/batch_size)
val_acc /= (len(validation_loader.dataset)/batch_size)
#Save the weights of the model
if best_loss == None or val_loss < best_loss:
best_loss = val_loss
is_best = True
save_checkpoint({
'epoch': epoch + 1,
'state_dict': model.state_dict(),
'loss': val_loss,
'loss_expressions': val_loss_expr,
'loss_landmarks' : val_loss_lm,
'accuracy': val_acc,
'optimizer' : optimizer.state_dict()}, is_best,"save_model/checkpoint_hourglass5.0.3.pth","save_model/best_model_validation5.0.3.pth")
is_best = False
scheduler.step(val_loss)
#Save metrics in a log file
with open("log/validation_hourglass5.0.3.log","a") as f:
f.write("epoch: {} / {} loss: {} e_loss:{} l_loss: {} accuracy: {}\n".format(epoch,max_epochs,val_loss,val_loss_expr,val_loss_lm,val_acc))
f.close()
#Construct tensorboard graph
writer.add_scalar('data/Loss training', loss, epoch)
writer.add_scalar('data/Loss landmarks training', loss_lm, epoch)
writer.add_scalar('data/Loss expressions training', loss_expr, epoch)
writer.add_scalar('data/Loss validation', val_loss, epoch)
writer.add_scalar('data/Loss landmarks validation', val_loss_lm, epoch)
writer.add_scalar('data/Loss expressions validation', val_loss_expr, epoch)
writer.add_scalar('data/Accuracy training', acc, epoch)
writer.add_scalar('data/Accuracy validation', val_acc, epoch)
if (epoch%5 == 1 or epoch == max_epochs):
desc = dict()
desc["bs"] = batch_size
desc["lr"] = lr
desc["f"] = factor
desc["p"] = patience
desc["d"] = 0
desc["weights"] = [1,1]
desc["epoch"] = epoch
desc["nbepochs"] = max_epochs
try:
send.sendInfos("3 (Hourglass 5.0.3)",desc,loss,acc,loss_lm,"...",loss_expr,"...",val_loss,val_acc,val_loss_lm,"...",val_loss_expr,val_acc)
except Exception as e:
pass
def main():
parser = argparse.ArgumentParser()
parser.add_argument(
"--dataset_ratio",
type=float,
default=1,
help="the purcentage of data used in the dataset (default: 1)")
parser.add_argument("--image_size",
type=int,
default=512,
help="size of the input image (default: 512)")
parser.add_argument("--depth",
type=int,
default=6,
help="depth of the autoencoder (default: 6)")
parser.add_argument(
"--num_block",
type=int,
default=3,
help="number of blocks of the autoencoder (default: 3)")
parser.add_argument("--epoch",
type=int,
default=1,
help="number of epoch (default: 1)")
parser.add_argument("--batch",
type=int,
default=100,
help="number of batch (default: 100)")
parser.add_argument("--valpct",
type=float,
default=0.2,
help="proportion of test data (default: 0.2)")
parser.add_argument("--num_threads",
type=int,
default=1,
help="number of thread used (default: 1)")
parser.add_argument("--create_csv",
type=bool,
default=False,
help="create or not csv file (default: False)")
parser.add_argument("--log",
default=False,
action='store_true',
help="Write log or not (default: False)")
parser.add_argument("--l2_reg",
type=int,
default=0.001,
help="L2 regularisation (default: 0.001)")
args = parser.parse_args()
# if args.create_csv:
# g_csv.generate_csv(DATA_PATH + CSV_NAME, args.num_var,
# args.num_const, args.num_prob)
valid_ratio = args.valpct # Going to use 80%/20% split for train/valid
data_transforms = transforms.Compose([ToTensor(), Normalize()])
full_dataset = ImageLoader(csv_file_path=LABEL_FILE_PATH,
image_directory=IMAGE_FOLDER_PATH,
mask_directory=MASK_FOLDER_PATH,
dataset_size=int(args.dataset_ratio *
DATASET_SIZE),
image_size=args.image_size,
transform=data_transforms)
nb_train = int((1.0 - valid_ratio) * len(full_dataset))
nb_test = len(full_dataset) - nb_train
print("Size of full data set: ", len(full_dataset))
print("Size of training data: ", nb_train)
print("Size of testing data: ", nb_test)
train_dataset, test_dataset = torch.utils.data.dataset.random_split(
full_dataset, [nb_train, nb_test])
train_loader = DataLoader(dataset=train_dataset,
batch_size=args.batch,
shuffle=True,
num_workers=args.num_threads)
test_loader = DataLoader(dataset=test_dataset,
batch_size=args.batch,
shuffle=True,
num_workers=args.num_threads)
# TODO params
# num_param = args.num_var + args.num_const + (args.num_var*args.num_const)
model = AutoEncoder(num_block=args.num_block, depth=args.depth)
# print("Network architechture:\n", model)
use_gpu = torch.cuda.is_available()
if use_gpu:
device = torch.device('cuda')
else:
device = torch.device('cpu')
model.to(device)
# f_loss = nn.BCEWithLogitsLoss()
f_loss = loss.Custom_loss()
# define optimizer
optimizer = torch.optim.Adam(model.parameters(), weight_decay=args.l2_reg)
# Make run directory
run_name = "run-"
LogManager = lw.LogManager(LOG_DIR, run_name)
run_dir_path, num_run = LogManager.generate_unique_dir()
# setup model checkpoint
path_model_check_point = run_dir_path + MODEL_DIR
if not os.path.exists(path_model_check_point):
os.mkdir(path_model_check_point)
model_checkpoint = ModelCheckpoint(path_model_check_point + BEST_MODELE,
model)
# top_logdir = LOG_DIR + FC1
# if not os.path.exists(top_logdir):
# os.mkdir(top_logdir)
# model_checkpoint = ModelCheckpoint(top_logdir + BEST_MODELE, model)
if args.log:
print("Writing log")
# generate unique folder for new run
tensorboard_writer = SummaryWriter(log_dir=run_dir_path,
filename_suffix=".log")
LogManager.set_tensorboard_writer(tensorboard_writer)
LogManager.summary_writer(model, optimizer)
# write short description of the run
run_desc = "Epoch{}".format(args.epoch)
log_file_path = LOG_DIR + run_desc + "Run{}".format(num_run) + ".log"
# LogManager.summary_writer(model, optimizer)
# write short description of the run
# run_desc = "Epoch{}Reg{}Var{}Const{}CLoss{}Dlayer{}Alpha{}".format(
# args.num_epoch,
# args.l2_reg,
# args.num_var,
# args.num_const,
# args.custom_loss,
# args.num_deep_layer,
# args.alpha)
# log_file_path = LOG_DIR + "Run{}".format(num_run) + run_desc + ".log"
# log_file_path = lw.generate_unique_logpath(LOG_DIR, "Linear")
with tqdm(total=args.epoch) as pbar:
for t in range(args.epoch):
pbar.update(1)
pbar.set_description("Epoch {}".format(t))
# print(DIEZ + "Epoch Number: {}".format(t) + DIEZ)
train_loss, train_acc = train(model, train_loader, f_loss,
optimizer, device, LogManager)
progress(train_loss, train_acc)
# time.sleep(0.5)
val_loss, val_acc = test(model,
test_loader,
f_loss,
device,
log_manager=LogManager)
print(" Validation : Loss : {:.4f}, Acc : {:.4f}".format(
val_loss, val_acc))
model_checkpoint.update(val_loss)
# lw.write_log(log_file_path, val_acc, val_loss, train_acc, train_loss)
if args.log:
tensorboard_writer.add_scalars("Loss/", {
'train_loss': train_loss,
'val_loss': val_loss
}, t)
# tensorboard_writer.add_scalar(METRICS + 'train_loss', train_loss, t)
# tensorboard_writer.add_scalar(METRICS + 'train_acc', train_acc, t)
# tensorboard_writer.add_scalar(METRICS + 'val_loss', val_loss, t)
# tensorboard_writer.add_scalar(METRICS + 'val_acc', val_acc, t)
LogManager.write_log(log_file_path, val_acc, val_loss, train_acc,
train_loss)
model.load_state_dict(torch.load(path_model_check_point + BEST_MODELE))
print(DIEZ + " Final Test " + DIEZ)
_, _ = test(model,
train_loader,
f_loss,
device,
final_test=True,
log_manager=LogManager,
txt="training")
test_loss, test_acc = test(model,
test_loader,
f_loss,
device,
log_manager=LogManager,
final_test=True)
print(" Test : Loss : {:.4f}, Acc : {:.4f}".format(
test_loss, test_acc))
plt.show(block=True)
# Initialize Loss function
MSE = nn.MSELoss(reduction='sum')
#MSE = nn.L1Loss()
BCE1 = nn.BCELoss()
BCE2 = nn.BCELoss()
# Establish convention for real and fake labels during training
# For the Discriminator
real_label = 1
fake_label = 0
# Setup Adam optimizers for both AE and D
optimizerAE = optim.Adam(netAE.parameters(),\
lr=lr, betas=(beta1, beta2),\
amsgrad = True)
optimizerD = optim.Adam(netD.parameters(),\
lr=lr, betas=(beta1, beta2),\
amsgrad = True)
optimizerSD = optim.Adam(netD.parameters(),\
lr=lr, betas=(beta1, beta2),\
amsgrad = True)
# Grab a batch of real images from the dataloader
real_batch = next(iter(dataloader))
print("Starting Training Loop...")
# For each epoch
for k, v in pretrained_dict.items() if k in model_dict
} #load the layer only same with the target model
model_dict.update(pretrained_dict)
print('===================================')
print('load pre_train weight successfully')
print('===================================')
except:
print('===================================')
print(' random init the weight ')
print('===================================')
autoencoder.load_state_dict(model_dict)
autoencoder.cuda()
autoencoder.train()
'''opt setting'''
optimizer = torch.optim.Adam(
autoencoder.parameters(),
lr=args.learning_rate) # optimize all cnn parameters
loss_func = nn.MSELoss()
'''folder for saving weight and loss history'''
save_path = args.save_weight_dir
'''training code'''
for epoch in range(EPOCH):
loss_iter = 0
for step, (x, b_label) in enumerate(train_data):
x_in = torch.tensor(x).cuda()
decoded = autoencoder(x_in)
loss = loss_func(decoded, x_in)
optimizer.zero_grad() # clear gradients for this training step
batch_size = 20
learning_rate = 1e-3
if args.fit:
print("Autoencoder ccs item handler has started...")
mv = MovieLens()
mv.create_cold_start_items(n_ratings_threshold=5)
dataloader = DataLoader(mv,
batch_size=batch_size,
shuffle=True,
drop_last=True)
model = AutoEncoder(input_dim=21, latent_dim=5)
criterion = nn.MSELoss()
optimizer = torch.optim.Adam(model.parameters(),
lr=learning_rate,
weight_decay=1e-5)
AutoEncoder.fit(model, num_epochs, dataloader, criterion, optimizer)
for ccs_item in tqdm(mv.ccs_items()):
print('ccs item:', ccs_item)
while mv.is_ccs(ccs_item):
u = mv.pick_random_user()
print('user:', u)
rated_ncs_items_by_u = mv.rated_ncs_items(u)
u_rated_latents = [
model.encode(mv.features(m)) for m in rated_ncs_items_by_u
]
ccs_latent = model.encode(mv.features(ccs_item))
def train(block=200,
data_name="bookcorpus",
downsample=-1,
dropout_rate=0.2,
history=None,
device="cuda:0",
params=None):
# version 1 - tfidf as feature
if data_name == "bookcorpus":
if history is None:
x_train, y_train = load_tfidf("train",
block,
verbose=True,
redo=False)
x_test, y_test = load_tfidf("test",
block,
verbose=True,
redo=False)
x_valid, y_valid = load_tfidf("valid",
block,
verbose=True,
redo=False)
else:
x_train, y_train = load_tfidf_long("train",
block,
verbose=True,
redo=False,
history=history)
x_test, y_test = load_tfidf_long("test",
block,
verbose=True,
redo=False,
history=history)
x_valid, y_valid = load_tfidf_long("valid",
block,
verbose=True,
redo=False,
history=history)
elif data_name == "coda19":
x_train, y_train = coda_load_tfidf("train",
block,
verbose=True,
redo=False)
x_test, y_test = coda_load_tfidf("test",
block,
verbose=True,
redo=False)
x_valid, y_valid = coda_load_tfidf("valid",
block,
verbose=True,
redo=False)
else:
print("Not supported yet")
quit()
if downsample != -1:
random_index = np.random.RandomState(5516).permutation(
x_train.shape[0])[:downsample]
x_train, y_train = x_train[random_index], y_train[random_index]
# parameter setting
vocab_size = x_train.shape[1]
output_size = y_train.shape[1]
hidden_size = 512 if params is None else params.hidden_size
epoch_num = 2000 if params is None else params.epoch_num
batch_size = 512 if params is None else params.batch_size
layer_num = 5 if params is None else params.layer_num
learning_rate = 1e-4 if params is None else params.learning_rate
early_stop_epoch = 20 if params is None else params.early_stop
device = device
if downsample == -1:
note = f"cosine - auto2 - {dropout_rate}"
else:
note = f"cosine - auto2 - {dropout_rate} - downsample"
# build dataset
training_dataset = TFIDF_Dataset(x_train, y_train)
training = data.DataLoader(training_dataset,
batch_size=batch_size,
shuffle=True,
num_workers=2)
testing_dataset = TFIDF_Dataset(x_test, y_test)
testing = data.DataLoader(testing_dataset,
batch_size=batch_size,
shuffle=False,
num_workers=2)
valid_dataset = TFIDF_Dataset(x_valid, y_valid)
valid = data.DataLoader(valid_dataset,
batch_size=batch_size,
shuffle=False,
num_workers=2)
# build model
model = AutoEncoder(
vocab_size=vocab_size,
hidden_size=hidden_size,
output_size=output_size,
dropout_rate=dropout_rate,
device=device,
layer_num=layer_num,
).to(device)
optimizer = torch.optim.Adam(model.parameters(), lr=learning_rate)
loss_function = lambda y_pred, y_batch: 1 - F.cosine_similarity(
y_pred, y_batch).mean()
# first evaluation
evaluate(model, valid, loss_function=loss_function)
best_epoch = 0
best_cosine = 0
best_model = copy.deepcopy(model.state_dict())
stopper = EarlyStop(mode="max", history=early_stop_epoch)
# train model
for epoch in range(1, epoch_num + 1):
# train
model.train()
total_loss = 0
total_count = np.ceil(x_train.shape[0] / batch_size)
total_cosine = 0
for count, (x_batch, y_batch) in enumerate(training, 1):
x_batch = x_batch.squeeze()
y_batch = y_batch.squeeze()
x_batch = x_batch.to(device)
y_batch = y_batch.to(device)
optimizer.zero_grad()
y_pred = model(x_batch)
loss = loss_function(y_pred, y_batch)
loss.backward()
optimizer.step()
total_loss += loss.item()
cosine = F.cosine_similarity(y_batch, y_pred)
total_cosine += cosine.mean().item()
print(
"\x1b[2K\rEpoch: {} / {} [{:.2f}%] Loss: {:.4f} Cosine: {:.4f}"
.format(epoch, epoch_num, 100.0 * count / total_count,
total_loss / count, total_cosine / count),
end="")
print()
# valid
if epoch % 1 == 0 or epoch == epoch_num:
cosine, _ = evaluate(model, valid, loss_function=loss_function)
if cosine > best_cosine:
best_model = copy.deepcopy(model.state_dict())
best_epoch = epoch
best_cosine = cosine
# check early stopping
if stopper.check(cosine):
print("Early Stopping at Epoch = ", epoch)
break
# load best model & test & save
print("loading model from epoch {}".format(best_epoch))
torch.save(
best_model,
os.path.join(model_dir, data_name,
"{}_autoencoder_{}.pt".format(note, block)))
model.load_state_dict(best_model)
cosine, y_pred = evaluate(model,
testing,
device=device,
loss_function=loss_function)
print("testing cosine:", cosine)
# config filename
if history is None:
filename = os.path.join(result_dir, f"{data_name}_dl_baseline.json")
prediction_filename = os.path.join(
predict_dir, "bookcorpus",
f"block{block}_autoencoder_{note.replace(' ', '')}.h5")
else:
filename = os.path.join(result_dir,
f"history_exp_{data_name}_dl_baseline.json")
prediction_filename = os.path.join(
predict_dir, "bookcorpus",
f"history_block{block}_autoencoder_{note.replace(' ', '')}.h5")
print_tfidf_metric(
{
"cosine": float(cosine),
"block": block,
"model": "autoencoder",
"note": "clean - autoencoder - tfidf - deep - {}".format(note)
},
filename=filename)
save_prediction(prediction_filename, y_pred)
#embed_batch1 = all_embeddings[:, 260:280]
#embed_batch2 = all_embeddings[:, 280:300]
#embed_batches = [embed_batch1, embed_batch2]
for d, embeds in enumerate(embed_batches):
dim_start = d * batch_size
dim_end = (d+1) * batch_size
print("Initializing AutoEncoder for dims %i to %i"\
% (dim_start, dim_end))
model = AutoEncoder(embeds, word2class, class2word, USE_CUDA)
model = model.cuda() if USE_CUDA else model
# Ignore any parameters with requires_grad = False
# aka the pretrained embeddings
params = filter(lambda x: x.requires_grad, model.parameters())
optimizer = torch.optim.SGD(params, lr=lr)
num_dims = model.embedding_dims
num_words = model.num_words
num_classes = model.num_classes
last_loss = float("inf")
for i in range(epochs):
print("Epoch %i" % i)
optimizer.zero_grad()
W_out, C = model.forward()
# Transpose to match n x W
def main(args):
## load datasets
train_dataset = dataloader('dogs_cats', 'train')
## split train and validation
num_train = len(train_dataset)
indices = list(range(num_train))
split = 5000
validation_idx = np.random.choice(indices, size=split, replace=False)
train_idx = list(set(indices) - set(validation_idx))
train_sampler = SubsetRandomSampler(train_idx)
validation_sampler = SubsetRandomSampler(validation_idx)
## train and validation loader
train_loader = torch.utils.data.DataLoader(
train_dataset,
batch_size=args.batch_size,
sampler=train_sampler)
valid_loader = torch.utils.data.DataLoader(
train_dataset,
batch_size=args.batch_size,
sampler=validation_sampler)
## debug
if args.debug:
images, _ = next(iter(train_loader))
grid = torchvision.utils.make_grid(images[:25], nrow=5)
imshow(grid, 'train')
images, _ = next(iter(valid_loader))
grid = torchvision.utils.make_grid(images[:25], nrow=5)
imshow(grid, 'valid')
## define model
model = AutoEncoder()
use_gpu = torch.cuda.is_available()
if use_gpu:
print('cuda is available!')
model.cuda()
## loss and optimizer
criterion = nn.MSELoss()
optimizer = torch.optim.SGD(
model.parameters(), lr=0.001, momentum=0.9, weight_decay=1e-5)
## log
log_dir = 'logs'
if not os.path.isdir('logs'):
os.mkdir('logs')
## train and valid
best_val = 5
loss_list = []
val_loss_list = []
for epoch in range(args.n_epochs):
loss = train(model, criterion, optimizer, train_loader, use_gpu)
val_loss = valid(model, criterion, valid_loader, use_gpu)
print('epoch {:d}, loss: {:.4f} val_loss: {:.4f}'.format(epoch, loss, val_loss))
if val_loss < best_val:
print('val_loss improved from {:.5f} to {:.5f}!'.format(best_val, val_loss))
best_val = val_loss
model_file = 'epoch{:03d}-{:.3f}.pth'.format(epoch, val_loss)
torch.save(model.state_dict(), os.path.join(log_dir, model_file))
loss_list.append(loss)
val_loss_list.append(val_loss)
def main(options):
if options.num_classes == 3:
TRAINING_PATH = '/media/ailab/Backup Plus/ADNI/data/train_3classes.txt'
else:
TRAINING_PATH = '/media/ailab/Backup Plus/ADNI/data/train_3classes.txt'
IMG_PATH = '/media/ailab/Backup Plus/ADNI/data/image'
dset_train = AD_3DRandomPatch(IMG_PATH, TRAINING_PATH)
train_loader = DataLoader(dset_train,
batch_size=options.batch_size,
shuffle=True,
num_workers=0,
drop_last=True)
sparsity = 0.05
beta = 0.5
mean_square_loss = nn.MSELoss()
#kl_div_loss = nn.KLDivLoss(reduce=False)
use_gpu = len(options.gpuid) >= 1
autoencoder = AutoEncoder()
if (use_gpu):
autoencoder = autoencoder.cuda()
else:
autoencoder = autoencoder.cpu()
#autoencoder.load_state_dict(torch.load("./autoencoder_pretrained_model19"))
optimizer = torch.optim.Adam(autoencoder.parameters(),
lr=options.learning_rate,
weight_decay=options.weight_decay)
last_train_loss = 1e-4
f = open("autoencoder_loss", 'a')
for epoch in range(options.epochs):
train_loss = 0.
print("At {0}-th epoch.".format(epoch))
for i, patches in enumerate(train_loader):
patch = patches['patch']
for b, batch in enumerate(patch):
batch = Variable(batch).cuda()
#batch = out.view(-1, 343)
output, s_ = autoencoder(batch)
batch = batch.view(-1, 343)
loss1 = mean_square_loss(output, batch)
s = Variable(torch.ones(s_.shape) * sparsity).cuda()
loss2 = (s * torch.log(s / (s_ + 1e-8)) + (1 - s) * torch.log(
(1 - s) / ((1 - s_ + 1e-8)))).sum() / options.batch_size
#kl_div_loss(mean_activitaion, sparsity)
loss = loss1 + beta * loss2
train_loss += loss
logging.info(
"batch {0} training loss is : {1:.5f}, {2:.5f}".format(
i * 1000 + b, loss1.data[0], loss2.data[0]))
f.write("batch {0} training loss is : {1:.3f}\n".format(
i * 1000 + b, loss.data[0]))
optimizer.zero_grad()
loss.backward()
optimizer.step()
train_avg_loss = train_loss / (len(train_loader) * 1000)
print(
"Average training loss is {0:.5f} at the end of epoch {1}".format(
train_avg_loss.data[0], epoch))
if (abs(train_avg_loss.data[0] - last_train_loss) <=
options.estop) or ((epoch + 1) % 20 == 0):
torch.save(autoencoder.state_dict(),
open("autoencoder_pretrained_model" + str(epoch), 'wb'))
last_train_loss = train_avg_loss.data[0]
f.close()
