"""Testing A Simple Prediction"""
#print("Feature vector: %s" % X_test[:1])
print("Label: %s" % str(y_test[0]))
print("Predicted: %s" % str(net0.predict(X_test[:1])))
"""Metrics"""
# layer_info = PrintLayerInfo()
# net0.verbose = 3
# net0.initialize()
#print layer_info(net0)
print "[Classification Report]: "
print classification_report(y_test, predicted)
print "[Train dataset] Score: ", net0.score(X_train, y_train)
print "[Test dataset] Score: ", net0.score(X_test, y_test)
plot_matrix(net0, X_test, y_test, filename)
valid_accuracies = np.array([i["valid_accuracy"] for i in net0.train_history_])
plot_accuracy(valid_accuracies, filename)
train_loss = [row['train_loss'] for row in net0.train_history_]
valid_loss = [row['valid_loss'] for row in net0.train_history_]
plot_loss(valid_loss, train_loss, filename)
y_score = net0.predict_proba(X_test) #[:, 1]
y_test_bin = np.array(label_binarize(y_test, classes=np.unique(y)))
n_classes = y_test_bin.shape[1]
plot_roc_curve(n_classes, y_test_bin, y_score, filename=filename)
#print("reconstruction loss", reconstruction_loss.item(), "sparsity_loss", sparsity_loss.item())
lambda1 = float(config["stain_separation"]["sparsity_weight"])
loss = reconstruction_loss + (lambda1 * sparsity_loss)
loss.backward()
optimiser_tdss.step()
loss_log["reconstruction"].append([epoch, batch_id, reconstruction_loss.item()])
loss_log["sparsity"].append([epoch, batch_id, sparsity_loss.item()])
batch_id += 1
# plot losses
plot_loss(loss_log["reconstruction"], "./output/training/reconstruction_loss.png")
plot_loss(loss_log["sparsity"], "./output/training/absorption_loss.png")
if epoch % 5 == 0:
# save images
for s, stain_od in enumerate(reconstructed_stains):
stain_rgb = OpticalDensity2RGB(stain_od)
vutils.save_image(
stain_rgb.detach(),
"./output/training/reconstructed_stain_{}.png".format(s),
normalize=False
)
vutils.save_image(
batch_size,
shuffle_data,
X_train,
Y_train,
X_val,
Y_val,
)
train_history_64_layers, val_history_64_layers = trainer_shuffle.train(
num_epochs)
plt.figure(figsize=(20, 12))
plt.subplot(1, 2, 1)
plt.ylim([0.00, 1.00])
utils.plot_loss(train_history_64_layers["loss"],
"Training Loss 10 hidden layers",
npoints_to_average=10)
plt.ylabel("Training Loss")
utils.plot_loss(val_history_64_layers["loss"],
"Validation Loss 10 hidden layers")
plt.ylabel("Validation Loss")
utils.plot_loss(train_history["loss"],
"Training Loss 2 hidden layers",
npoints_to_average=10)
utils.plot_loss(val_history["loss"], "Validation Loss 2 hidden layers")
plt.legend()
plt.subplot(1, 2, 2)
plt.ylim([0.92, .97])
utils.plot_loss(val_history_64_layers["accuracy"],
trainer_two = SoftmaxTrainer(
momentum_gamma,
use_momentum,
model_two,
learning_rate,
batch_size,
shuffle_data,
X_train,
Y_train,
X_val,
Y_val,
)
train_history, val_history = trainer.train(num_epochs)
train_history_two, val_history_two = trainer_two.train(num_epochs)
plt.figure(figsize=(20, 10))
plt.subplot(1, 2, 1)
utils.plot_loss(train_history["accuracy"], "Base model task 3 ")
utils.plot_loss(train_history_two["accuracy"], "Ten layer model")
plt.ylim([0.85, 1.0])
plt.ylabel("Training Accuracy")
plt.legend()
plt.subplot(1, 2, 2)
plt.ylim([0.85, 1.0])
utils.plot_loss(val_history["accuracy"], "Base model task 3")
utils.plot_loss(val_history_two["accuracy"], "Ten layer model")
plt.ylabel("Accuracy")
plt.legend()
plt.savefig("task4e_train_loss.png", dpi=120)
def train(self):
# load networks************************************************************************
self.Choose_Model(self.Model_index)
utils.print_network(self.model)
# optimizer
self.momentum = 0.9
self.optimizer = optim.SGD(self.model.parameters(),
lr=self.lr,
momentum=self.momentum)
self.model.float()
# loss function
if self.loss_func == 'mse':
self.loss = nn.MSELoss()
elif self.loss_func == 'ssim':
self.loss = pytorch_ssim.SSIM(window_size=11)
if self.gpu_mode:
self.model = nn.DataParallel(self.model)
self.model.cuda()
self.loss.cuda()
# load dataset
train_data_loader = self.load_dataset(dataset='train')
#val_data_loader = self.load_dataset(dataset='test')
# set the logger
log_dir = os.path.join(self.save_dir, 'logs')
if not os.path.exists(log_dir):
os.mkdir(log_dir)
logger = Logger(log_dir)
################# Train start#################
print('Training is started.')
avg_loss = []
step = 0
self.model.train()
### debug ###
### debug end ###
for epoch in range(self.num_epochs):
epoch_loss = 0
for iter, data in enumerate(train_data_loader):
LR = data['img_LR']
HR = data['img_HR']
#only use Y channel
input_Y = LR[:, 0:1, :, :]
target_Y = HR[:, 0:1, :, :]
if self.scale_factor == 4:
target_Y = utils.shave(
target_Y, border_size=2 * self.scale_factor
) #according to size of the output image passed the network
elif self.scale_factor == 6:
target_Y = utils.shave(target_Y,
border_size=2 * self.scale_factor -
1)
elif self.scale_factor == 2:
target_Y = utils.shave(target_Y,
border_size=2 * self.scale_factor -
1)
else:
target_Y = utils.shave(target_Y,
border_size=2 * self.scale_factor -
2)
if self.save_inImg == True: #save the net input image
saveinY = (input_Y.numpy()[0, :, :, :].transpose(1, 2, 0) *
255).astype(numpy.uint8)
scipy.misc.imsave('lrin.png', saveinY[:, :, 0])
savetarY = (
target_Y.numpy()[0, :, :, :].transpose(1, 2, 0) *
255).astype(numpy.uint8)
scipy.misc.imsave('tarin.png', savetarY[:, :, 0])
if self.gpu_mode:
target = Variable(target_Y.cuda())
#print("target.size()")
#print(target.size())
input = Variable(input_Y.cuda())
#print("input.size():", input.size())
else:
target = Variable(target_Y)
# target = Variable(utils.shave(target_Y, border_size=2*self.scale_factor))
input = Variable(input_Y)
############## ORIGINAL ###############
self.optimizer.zero_grad()
recon_image = self.model(input)
#print("recon_image.size(): ")
#print(recon_image.size())
# if self.scale_factor ==2:
# recon_image = recon_image[:,:,1:-1,1:-1]
# elif self.scale_factor == 3:
# recon_image = recon_image[:, :, 0:-1, 0:-1]
#### SSIM loss ##############
# loss = 1-self.loss(recon_image, target)
loss = self.loss(recon_image, target)
# print loss.data
loss.backward()
self.optimizer.step()
# log
epoch_loss += loss.data
# tensorboard logging
logger.scalar_summary('loss', loss.data, step + 1)
step += 1
if epoch % self.save_epochs == 0:
self.save_model(epoch)
# self.validation(epoch, val_data_loader)
avg_loss.append(epoch_loss / len(train_data_loader))
print("Epoch: [%2d] [%4d/%4d] loss: %.8f" %
((epoch + 1),
(iter + 1), len(train_data_loader), epoch_loss))
# Plot avg. loss
utils.plot_loss([avg_loss], self.num_epochs, save_dir=self.save_dir)
print("Training is finished.")
# Save final trained parameters of model
self.save_model(epoch=None)
def train(self):
# load networks************************************************************************
self.Choose_Model(self.Model_index)
utils.print_network(self.model)
# optimizer
self.momentum = 0.9
self.optimizer = optim.SGD(self.model.parameters(),
lr=self.lr,
momentum=self.momentum)
self.model.float()
# loss function
if self.loss_func == 'mse':
self.loss = nn.MSELoss()
elif self.loss_func == 'ssim':
self.loss = pytorch_ssim.SSIM(window_size=11)
elif self.loss_func == 'ssim&mse':
self.Mseloss = nn.MSELoss()
self.SSIMloss = pytorch_ssim.SSIM(window_size=11)
# self.loss = 0.97 * nn.MSELoss() + 0.03 * pytorch_ssim.SSIM(window_size = 11)
if self.gpu_mode:
self.model.cuda()
self.Mseloss.cuda()
self.SSIMloss.cuda()
# load dataset
train_data_loader = self.load_dataset(dataset='train')
val_data_loader = self.load_dataset(dataset='test')
# set the logger
log_dir = os.path.join(self.save_dir, 'logs')
if not os.path.exists(log_dir):
os.mkdir(log_dir)
logger = Logger(log_dir)
# ckpt_dir = self.ckpt_dir
# if not os.path.exists(ckpt_dir):
# os.makedirs(ckpt_dir, mode=0o777)
################# Train start#################
print('Training is started.')
avg_loss = []
step = 0
self.model.train()
### debug ###
### debug end ###
for epoch in range(self.num_epochs):
epoch_loss = 0
epoch_mseloss = 0
epoch_ssimloss = 0
for iter, data in enumerate(train_data_loader):
LR = data['img_LR']
HR = data['img_HR']
#only use Y channel
input_Y = LR[:, 0:1, :, :]
#print(input_Y.size())
w = input_Y.size()[2]
h = input_Y.size()[3]
C = input_Y.size()[0]
#print(C, w, h)
w_new = w + 4
h_new = h + 4
a = numpy.zeros([C, 1, w_new, h_new], dtype='f')
numpy_Y = input_Y.numpy()
a[:, :, 2:-2, 2:-2] = numpy_Y
input_Y_new = torch.from_numpy(a)
#print(input_Y_new.size())
target_Y = HR[:, 0:1, :, :]
# if self.scale_factor==4:
# target_Y = utils.shave(target_Y, border_size=2 * self.scale_factor) #according to size of the output image passed the network
# elif self.scale_factor==6:
# target_Y = utils.shave(target_Y, border_size=2 * self.scale_factor - 1)
# elif self.scale_factor==2:
# target_Y = utils.shave(target_Y, border_size=2 * self.scale_factor - 1)
# else:
# target_Y = utils.shave(target_Y, border_size=2 * self.scale_factor - 2)
if self.save_inImg == True: #save the net input image
saveinY = (input_Y.numpy()[0, :, :, :].transpose(1, 2, 0) *
255).astype(numpy.uint8)
scipy.misc.imsave('lrin.png', saveinY[:, :, 0])
savetarY = (
target_Y.numpy()[0, :, :, :].transpose(1, 2, 0) *
255).astype(numpy.uint8)
scipy.misc.imsave('tarin.png', savetarY[:, :, 0])
if self.gpu_mode:
target = Variable(target_Y.cuda())
input = Variable(input_Y_new.cuda())
else:
target = Variable(target_Y)
# target = Variable(utils.shave(target_Y, border_size=2*self.scale_factor))
input = Variable(input_Y_new)
############## ORIGINAL ###############
self.optimizer.zero_grad()
recon_image = self.model(input)
# if self.scale_factor ==2:
# recon_image = recon_image[:,:,1:-1,1:-1]
# elif self.scale_factor == 3:
# recon_image = recon_image[:, :, 0:-1, 0:-1]
#### SSIM loss ##############
# loss = 1-self.loss(recon_image, target)
#print('target.size()', target.size())
#print('recon_image.size()', recon_image.size())
# Caculate total_loss = alpha*mseloss + (1-alpha)*ssimloss
alpha = 0.97
mseloss = self.Mseloss(recon_image, target)
ssimloss = self.SSIMloss(recon_image, target)
loss = alpha * mseloss + (1 - alpha) * ssimloss
# loss = self.loss(recon_image, target)
# print loss.data
loss.backward()
self.optimizer.step()
# log
epoch_mseloss += mseloss.data
epoch_ssimloss += ssimloss.data
epoch_loss += loss.data
# tensorboard logging
#print(' loss ', loss.data, ' mseloss ', mseloss.data, ' ssimloss ', ssimloss.data)
logger.scalar_summary('loss', loss.data, step + 1)
step += 1
print(step, ' loss ', epoch_loss, ' mseloss ', epoch_mseloss,
' ssimloss ', epoch_ssimloss)
if epoch % self.save_epochs == 0:
#onnx_name = 'x' + str(self.scale_factor) + '_' + self.model_name + '_epoch_' + str(epoch) + '.onnx'
#torch.onnx.export(self.model, input, onnx_name, export_params=True, verbose=True)
self.save_model(epoch)
# save_path = os.path.join(ckpt_dir, "{}_{}.pth".format(self.ckpt_name, epoch))
# torch.save(self.model.state_dict(), save_path)
#self.validation(epoch, val_data_loader)
avg_loss.append(epoch_loss / len(train_data_loader))
print("Epoch: [%2d] [%4d/%4d] loss: %.8f" %
((epoch + 1),
(iter + 1), len(train_data_loader), epoch_loss))
# Plot avg. loss
utils.plot_loss([avg_loss], self.num_epochs, save_dir=self.save_dir)
print("Training is finished.")
# Save final trained parameters of model
self.save_model(epoch=None)
def train_model(ques_train_map, ans_train_map, img_train_map, ques_train_ids,
ques_to_img_train, ques_val_map, ans_val_map, img_val_map,
ques_val_ids, ques_to_img_val, id_to_ans, train_dim, val_dim,
ans_types, params):
# training parameters
num_epochs = params['num_epochs']
batch_size = params['batch_size']
num_batches_train = train_dim // batch_size
num_batches_val = val_dim // batch_size
eval_every = 5
train_loss, train_acc = [], []
val_loss, val_acc = [], []
eval_acc = []
print "Loading model"
model = get_model(dropout_rate=float(params['dropout_rate']),
regularization_rate=float(params['regularization_rate']),
embedding_size=int(params['embedding_size']),
num_classes=int(params['num_answers']),
model_name=params['model'])
if not ans_types:
savedir = "models/%s_%s" % (params['model'], str(
params['num_answers']))
else:
savedir = "models/%s_%s_%s" % (params['model'],
ans_types.replace("/", ""),
str(params['num_answers']))
if not os.path.exists(savedir):
os.mkdir(savedir)
for k in range(num_epochs):
loss, acc = train_epoch(k + 1, model, num_batches_train, batch_size,
ques_train_map, ans_train_map, img_train_map,
ques_train_ids, ques_to_img_train)
train_loss.append(loss)
train_acc.append(acc)
loss, acc = val_epoch(k + 1, model, num_batches_val, batch_size,
ques_val_map, ans_val_map, img_val_map,
ques_val_ids, ques_to_img_val)
val_loss.append(loss)
val_acc.append(acc)
if (k + 1) % eval_every == 0:
model.save_weights("%s/%s_epoch_%d_weights.h5" %
(savedir, params['model'], (k + 1)),
overwrite=True)
eval_accuracy = evaluate(model, vqa_val, batch_size, ques_val_map,
img_val_map, id_to_ans,
params['ans_types'])
print("Eval accuracy: %.2f" % eval_accuracy)
eval_acc.append(eval_accuracy)
plot_loss(train_loss, val_loss, savedir)
plot_accuracy(train_acc, val_acc, savedir)
plot_eval_accuracy(eval_acc, savedir)
best_epoch = (1 + np.argmax(np.array(eval_acc))) * eval_every
print "Best accuracy %.02f on epoch %d" % (max(eval_acc), best_epoch)
model.load_weights("%s/%s_epoch_%d_weights.h5" %
(savedir, params['model'], best_epoch))
evaluate(model,
vqa_val,
batch_size,
ques_val_map,
img_val_map,
id_to_ans,
params['ans_types'],
verbose=True)
# 任务4：理解utils中cross_entropy的实现代码
loss1 = utils.cross_entropy(y_hat, batchy)
trainloss.append(loss1)
"""从后往前迭代更新残差值，并利用残差值来更新各输出层和隐藏层的权重"""
error = y_hat - batchy
model.backward(error)
# 评估模型性能
loss2 = utils.cross_entropy(model.forward(validX),
utils.make_onehot(validy, 10))
validloss.append(loss2)
#print("iteration:{0}/{1} trainloss:{2:.2f}, validloss:{3:.2f}".format(
#j, i, loss1, loss2))
utils.plot_loss(trainloss, validloss)
#%% 评估
prediction = np.argmax(model.forward(testX), axis=1)
cmatrix = utils.confusion_matrix(prediction, testy.flatten(), 10)
print(cmatrix)
# accuracy
acc = np.diag(cmatrix).sum() / testy.shape[0]
print("accuracy: %.4f" % (acc))
# heat map
plt.figure()
plt.imshow(cmatrix, cmap='gray', interpolation='none')
plt.show()
# 观察错误
shuffle_data,
X_train,
Y_train,
X_val,
Y_val,
)
start = time.time()
train_history_new, val_history_new = new_trainer.train(num_epochs)
end = time.time()
print("Elapsed training time (s): ", end - start)
plt.subplot(1, 2, 1)
utils.plot_loss(train_history["loss"],
"Task 2 Model",
npoints_to_average=10)
utils.plot_loss(train_history_new["loss"],
"Task 2 Model - Improved weight, sigmoid and momentum",
npoints_to_average=10)
plt.ylim([0, .4])
plt.ylabel("Training Cross Entropy Loss")
plt.legend()
plt.subplot(1, 2, 2)
plt.ylim([0.8, .98])
utils.plot_loss(val_history["accuracy"], "Task 2 Model")
utils.plot_loss(val_history_new["accuracy"],
"Task 2 Model - Improved weight, sigmoid and momentum")
plt.ylabel("Validation Accuracy")
plt.legend()
plt.show()
train_steps_per_epoch,
EPOCHS,
validation_generator,
validation_steps_per_epoch,
save_model_filepath=model_file)
# Training KerasModel for pretrained models
# model_history = k_model.train_learned_model_with_generator(train_generator,
# train_steps_per_epoch,
# EPOCHS,
# validation_generator,
# validation_steps_per_epoch,
# save_model_filepath='model_transfer_Inceptionv3.h5')
# Plotting the model Loss
utils.plot_loss(model_history=model_history)
# Uncomment these lines to visualize the model's first and second Convolutions for track 1 and
# track 2 test images
# Track 1 layers visualization
# k_model = KerasModel(load=True, model_file='model.h5')
# test_image = np.asarray(Image.open(
# './assets/Layer_visualization/Track1/center_2018_05_07_18_39_19_350.jpg'))
# print(np.array(test_image).shape)
# k_model.visualize_layer(test_image, 'Track1 Model First Convolution')
# k_model.visualize_layer(test_image, 'Track1 Model Second Convolution', layer_num=4)
# Track 2 layers visualization
# test_image = np.asarray(Image.open(
def main(args):
"""Experiment logic"""
# Get file separator and construct paths
sep = "\t" if args.file_type == "tsv" else ","
train_path = os.path.join(args.data_dir, "train.{}".format(args.file_type))
test_path = os.path.join(args.data_dir, "test.{}".format(args.file_type))
# Read column headings
headings = pd.read_csv(train_path, sep=sep, nrows=1).columns
text, label = "text", "gold_label_{}".format(args.task_type)
if args.elmo:
from elmo import TabularReader, ElmoLoader
# Pretrained urls
options_file = "https://s3-us-west-2.amazonaws.com/allennlp/models/elmo/2x4096_512_2048cnn_2xhighway/elmo_2x4096_512_2048cnn_2xhighway_options.json"
weight_file = "https://s3-us-west-2.amazonaws.com/allennlp/models/elmo/2x4096_512_2048cnn_2xhighway/elmo_2x4096_512_2048cnn_2xhighway_weights.hdf5"
# Read dataset
reader = TabularReader(text, label, sep)
loader = ElmoLoader(reader, train_path, test_path, args.batch_dims)
# Build model
label_map = loader.label_map
embedding_size = 1024
model = DAN(to_int(args.layers),
len(label_map),
embedding_size=embedding_size,
elmo_config=(options_file, weight_file))
else:
# Build data loader
loader = DataLoader(
args.data_dir,
args.file_type,
headings,
text,
label,
to_int(args.batch_dims),
(args.glove_type, args.glove_dim),
args.temp_dir,
)
# Build model
vocab, label_map = loader.vocab, loader.label_map
model = DAN(to_int(args.layers),
len(label_map),
vocab_size=len(vocab),
embedding_size=args.glove_dim,
pretrained_vecs=vocab.vectors)
# Define training functions
optimiser = optim.SGD(model.parameters(), lr=args.lr)
loss_fn = nn.CrossEntropyLoss()
# Train
logging.info("\n\nStarting training...\n\n")
if args.num_processes > 1:
model.share_memory()
processes = []
for pid in range(args.num_processes):
p = mp.Process(target=run.training_process,
args=(pid, loader, model, optimiser, loss_fn,
(args.num_steps // args.num_processes)))
p.start()
processes.append(p)
for p in processes:
p.join()
else:
report_every = int(args.num_steps * 0.01)
losses = run.train(loader, model, optimiser, loss_fn, label_map,
args.num_steps, report_every)
if args.plot:
logging.info("\n\nPlotting training schedule...\n\n")
plot_loss(losses, report_every, args.temp_dir)
# Save the trained model
logging.info("\n\nNow saving...\n\n")
torch.save(model, os.path.join(args.temp_dir, "saved_model.pt"))
# Test
model_acc = run.test(loader, label_map, args.temp_dir)
if args.baseline:
logging.info(
"\n\nComparing with multinomial naive bayes baseline...\n\n")
from bayes import multi_nb
train, test = pd.read_csv(train_path, sep=sep), pd.read_csv(test_path,
sep=sep)
train_txt, test_txt = (train[text], train[label]), (test[text],
test[label])
base_acc = multi_nb(train_txt, test_txt)
logging.info("Model accuracy: {:.6g}".format(model_acc))
logging.info("Baseline accuracy: {:.6g}".format(base_acc))
logging.info("{}".format(
"Model wins!" if model_acc > base_acc else "Baseline wins!"))
def run_main(args):
################################################# START SECTION OF LOADING PARAMETERS #################################################
# Read parameters
epochs = args.epochs
dim_au_out = args.bottleneck #8, 16, 32, 64, 128, 256,512
na = args.missing_value
data_path = DATA_MAP[args.target_data]
test_size = args.test_size
select_drug = args.drug
freeze = args.freeze_pretrain
valid_size = args.valid_size
g_disperson = args.var_genes_disp
min_n_genes = args.min_n_genes
max_n_genes = args.max_n_genes
source_model_path = args.source_model_path
target_model_path = args.target_model_path
log_path = args.logging_file
batch_size = args.batch_size
encoder_hdims = args.source_h_dims.split(",")
encoder_hdims = list(map(int, encoder_hdims))
source_data_path = args.source_data
pretrain = args.pretrain
prediction = args.predition
data_name = args.target_data
label_path = args.label_path
reduce_model = args.dimreduce
predict_hdims = args.p_h_dims.split(",")
predict_hdims = list(map(int, predict_hdims))
leiden_res = args.cluster_res
load_model = bool(args.load_target_model)
# Misc
now = time.strftime("%Y-%m-%d-%H-%M-%S")
# Initialize logging and std out
out_path = log_path + now + ".err"
log_path = log_path + now + ".log"
out = open(out_path, "w")
sys.stderr = out
#Logging infomaion
logging.basicConfig(
level=logging.INFO,
filename=log_path,
filemode='a',
format=
'%(asctime)s - %(pathname)s[line:%(lineno)d] - %(levelname)s: %(message)s'
)
logging.getLogger('matplotlib.font_manager').disabled = True
logging.info(args)
# Save arguments
args_df = ut.save_arguments(args, now)
################################################# END SECTION OF LOADING PARAMETERS #################################################
################################################# START SECTION OF SINGLE CELL DATA REPROCESSING #################################################
# Load data and preprocessing
adata = pp.read_sc_file(data_path)
if data_name == 'GSE117872':
adata = ut.specific_process(adata,
dataname=data_name,
select_origin=args.batch_id)
elif data_name == 'GSE122843':
adata = ut.specific_process(adata, dataname=data_name)
elif data_name == 'GSE110894':
adata = ut.specific_process(adata, dataname=data_name)
elif data_name == 'GSE112274':
adata = ut.specific_process(adata, dataname=data_name)
elif data_name == 'GSE116237':
adata = ut.specific_process(adata, dataname=data_name)
elif data_name == 'GSE108383':
adata = ut.specific_process(adata, dataname=data_name)
elif data_name == 'GSE140440':
adata = ut.specific_process(adata, dataname=data_name)
elif data_name == 'GSE129730':
adata = ut.specific_process(adata, dataname=data_name)
elif data_name == 'GSE149383':
adata = ut.specific_process(adata, dataname=data_name)
else:
adata = adata
sc.pp.filter_cells(adata, min_genes=200)
sc.pp.filter_genes(adata, min_cells=3)
adata = pp.cal_ncount_ngenes(adata)
# Show statisctic after QX
sc.pl.violin(adata,
['n_genes_by_counts', 'total_counts', 'pct_counts_mt-'],
jitter=0.4,
multi_panel=True,
save=data_name,
show=False)
sc.pl.scatter(adata, x='total_counts', y='pct_counts_mt-', show=False)
sc.pl.scatter(adata, x='total_counts', y='n_genes_by_counts', show=False)
if args.remove_genes == 0:
r_genes = []
else:
r_genes = REMOVE_GENES
#Preprocess data by filtering
if data_name not in ['GSE112274', 'GSE140440']:
adata = pp.receipe_my(adata,
l_n_genes=min_n_genes,
r_n_genes=max_n_genes,
filter_mincells=args.min_c,
filter_mingenes=args.min_g,
normalize=True,
log=True,
remove_genes=r_genes)
else:
adata = pp.receipe_my(adata,
l_n_genes=min_n_genes,
r_n_genes=max_n_genes,
filter_mincells=args.min_c,
percent_mito=100,
filter_mingenes=args.min_g,
normalize=True,
log=True,
remove_genes=r_genes)
# Select highly variable genes
sc.pp.highly_variable_genes(adata,
min_disp=g_disperson,
max_disp=np.inf,
max_mean=6)
sc.pl.highly_variable_genes(adata, save=data_name, show=False)
adata.raw = adata
adata = adata[:, adata.var.highly_variable]
# Preprocess data if spcific process is required
data = adata.X
# PCA
# Generate neighbor graph
sc.tl.pca(adata, svd_solver='arpack')
sc.pp.neighbors(adata, n_neighbors=10)
# Generate cluster labels
sc.tl.leiden(adata, resolution=leiden_res)
sc.tl.umap(adata)
sc.pl.umap(adata,
color=['leiden'],
save=data_name + 'umap' + now,
show=False)
adata.obs['leiden_origin'] = adata.obs['leiden']
adata.obsm['X_umap_origin'] = adata.obsm['X_umap']
data_c = adata.obs['leiden'].astype("long").to_list()
################################################# END SECTION OF SINGLE CELL DATA REPROCESSING #################################################
################################################# START SECTION OF LOADING SC DATA TO THE TENSORS #################################################
#Prepare to normailize and split target data
mmscaler = preprocessing.MinMaxScaler()
try:
data = mmscaler.fit_transform(data)
except:
logging.warning("Only one class, no ROC")
# Process sparse data
data = data.todense()
data = mmscaler.fit_transform(data)
# Split data to train and valid set
# Along with the leiden conditions for CVAE propose
Xtarget_train, Xtarget_valid, Ctarget_train, Ctarget_valid = train_test_split(
data, data_c, test_size=valid_size, random_state=42)
# Select the device of gpu
device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")
# Assuming that we are on a CUDA machine, this should print a CUDA device:
logging.info(device)
try:
torch.cuda.set_device(device)
except:
logging.warning("No GPU detected, will apply cpu to process")
# Construct datasets and data loaders
Xtarget_trainTensor = torch.FloatTensor(Xtarget_train).to(device)
Xtarget_validTensor = torch.FloatTensor(Xtarget_valid).to(device)
# Use leiden label if CVAE is applied
Ctarget_trainTensor = torch.LongTensor(Ctarget_train).to(device)
Ctarget_validTensor = torch.LongTensor(Ctarget_valid).to(device)
X_allTensor = torch.FloatTensor(data).to(device)
C_allTensor = torch.LongTensor(data_c).to(device)
train_dataset = TensorDataset(Xtarget_trainTensor, Ctarget_trainTensor)
valid_dataset = TensorDataset(Xtarget_validTensor, Ctarget_validTensor)
Xtarget_trainDataLoader = DataLoader(dataset=train_dataset,
batch_size=batch_size,
shuffle=True)
Xtarget_validDataLoader = DataLoader(dataset=valid_dataset,
batch_size=batch_size,
shuffle=True)
dataloaders_pretrain = {
'train': Xtarget_trainDataLoader,
'val': Xtarget_validDataLoader
}
################################################# START SECTION OF LOADING SC DATA TO THE TENSORS #################################################
################################################# START SECTION OF LOADING BULK DATA #################################################
# Read source data
data_r = pd.read_csv(source_data_path, index_col=0)
label_r = pd.read_csv(label_path, index_col=0)
label_r = label_r.fillna(na)
# Extract labels
selected_idx = label_r.loc[:, select_drug] != na
label = label_r.loc[selected_idx, select_drug]
label = label.values.reshape(-1, 1)
if prediction == "regression":
lbscaler = preprocessing.MinMaxScaler()
label = lbscaler.fit_transform(label)
dim_model_out = 1
else:
le = preprocessing.LabelEncoder()
label = le.fit_transform(label)
dim_model_out = 2
# Process source data
mmscaler = preprocessing.MinMaxScaler()
source_data = mmscaler.fit_transform(data_r)
# Split source data
Xsource_train_all, Xsource_test, Ysource_train_all, Ysource_test = train_test_split(
source_data, label, test_size=test_size, random_state=42)
Xsource_train, Xsource_valid, Ysource_train, Ysource_valid = train_test_split(
Xsource_train_all,
Ysource_train_all,
test_size=valid_size,
random_state=42)
# Transform source data
# Construct datasets and data loaders
Xsource_trainTensor = torch.FloatTensor(Xsource_train).to(device)
Xsource_validTensor = torch.FloatTensor(Xsource_valid).to(device)
if prediction == "regression":
Ysource_trainTensor = torch.FloatTensor(Ysource_train).to(device)
Ysource_validTensor = torch.FloatTensor(Ysource_valid).to(device)
else:
Ysource_trainTensor = torch.LongTensor(Ysource_train).to(device)
Ysource_validTensor = torch.LongTensor(Ysource_valid).to(device)
sourcetrain_dataset = TensorDataset(Xsource_trainTensor,
Ysource_trainTensor)
sourcevalid_dataset = TensorDataset(Xsource_validTensor,
Ysource_validTensor)
Xsource_trainDataLoader = DataLoader(dataset=sourcetrain_dataset,
batch_size=batch_size,
shuffle=True)
Xsource_validDataLoader = DataLoader(dataset=sourcevalid_dataset,
batch_size=batch_size,
shuffle=True)
dataloaders_source = {
'train': Xsource_trainDataLoader,
'val': Xsource_validDataLoader
}
################################################# END SECTION OF LOADING BULK DATA #################################################
################################################# START SECTION OF MODEL CUNSTRUCTION #################################################
# Construct target encoder
if reduce_model == "AE":
encoder = AEBase(input_dim=data.shape[1],
latent_dim=dim_au_out,
h_dims=encoder_hdims)
loss_function_e = nn.MSELoss()
elif reduce_model == "VAE":
encoder = VAEBase(input_dim=data.shape[1],
latent_dim=dim_au_out,
h_dims=encoder_hdims)
elif reduce_model == "CVAE":
# Number of condition is equal to the number of clusters
encoder = CVAEBase(input_dim=data.shape[1],
n_conditions=len(set(data_c)),
latent_dim=dim_au_out,
h_dims=encoder_hdims)
if torch.cuda.is_available():
encoder.cuda()
logging.info("Target encoder structure is: ")
logging.info(encoder)
encoder.to(device)
optimizer_e = optim.Adam(encoder.parameters(), lr=1e-2)
loss_function_e = nn.MSELoss()
exp_lr_scheduler_e = lr_scheduler.ReduceLROnPlateau(optimizer_e)
# Load source model before transfer
if prediction == "regression":
dim_model_out = 1
else:
dim_model_out = 2
# Load AE model
if reduce_model == "AE":
source_model = PretrainedPredictor(input_dim=Xsource_train.shape[1],
latent_dim=dim_au_out,
h_dims=encoder_hdims,
hidden_dims_predictor=predict_hdims,
output_dim=dim_model_out,
pretrained_weights=None,
freezed=freeze)
source_model.load_state_dict(torch.load(source_model_path))
source_encoder = source_model
# Load VAE model
elif reduce_model in ["VAE", "CVAE"]:
source_model = PretrainedVAEPredictor(
input_dim=Xsource_train.shape[1],
latent_dim=dim_au_out,
h_dims=encoder_hdims,
hidden_dims_predictor=predict_hdims,
output_dim=dim_model_out,
pretrained_weights=None,
freezed=freeze,
z_reparam=bool(args.VAErepram))
source_model.load_state_dict(torch.load(source_model_path))
source_encoder = source_model
logging.info("Load pretrained source model from: " + source_model_path)
source_encoder.to(device)
################################################# END SECTION OF MODEL CUNSTRUCTION #################################################
################################################# START SECTION OF SC MODEL PRETRAININIG #################################################
# Pretrain target encoder
# Pretain using autoencoder is pretrain is not False
if (str(pretrain) != '0'):
# Pretrained target encoder if there are not stored files in the harddisk
train_flag = True
pretrain = str(pretrain)
if (os.path.exists(pretrain) == True):
try:
encoder.load_state_dict(torch.load(pretrain))
logging.info("Load pretrained target encoder from " + pretrain)
train_flag = False
except:
logging.warning("Loading failed, procceed to re-train model")
if train_flag == True:
if reduce_model == "AE":
encoder, loss_report_en = t.train_AE_model(
net=encoder,
data_loaders=dataloaders_pretrain,
optimizer=optimizer_e,
loss_function=loss_function_e,
n_epochs=epochs,
scheduler=exp_lr_scheduler_e,
save_path=pretrain)
elif reduce_model == "VAE":
encoder, loss_report_en = t.train_VAE_model(
net=encoder,
data_loaders=dataloaders_pretrain,
optimizer=optimizer_e,
n_epochs=epochs,
scheduler=exp_lr_scheduler_e,
save_path=pretrain)
elif reduce_model == "CVAE":
encoder, loss_report_en = t.train_CVAE_model(
net=encoder,
data_loaders=dataloaders_pretrain,
optimizer=optimizer_e,
n_epochs=epochs,
scheduler=exp_lr_scheduler_e,
save_path=pretrain)
logging.info("Pretrained finished")
# Before Transfer learning, we test the performance of using no transfer performance:
# Use vae result to predict
if (args.dimreduce != "CVAE"):
embeddings_pretrain = encoder.encode(X_allTensor)
else:
embeddings_pretrain = encoder.encode(X_allTensor, C_allTensor)
pretrain_prob_prediction = source_model.predict(
embeddings_pretrain).detach().cpu().numpy()
adata.obs["sens_preds_pret"] = pretrain_prob_prediction[:, 1]
adata.obs["sens_label_pret"] = pretrain_prob_prediction.argmax(axis=1)
# # Use umap result to predict
## This section is removed because the dim problem and the performance problem
# sc.tl.pca(adata, n_comps=max(50,2*dim_au_out),svd_solver='arpack')
# sc.tl.umap(adata, n_components=dim_au_out)
# embeddings_umap = torch.FloatTensor(adata.obsm["X_umap"]).to(device)
# umap_prob_prediction = source_model.predict(embeddings_umap).detach().cpu().numpy()
# adata.obs["sens_preds_umap"] = umap_prob_prediction[:,1]
# adata.obs["sens_label_umap"] = umap_prob_prediction.argmax(axis=1)
# # Use tsne result to predict
# #sc.tl.tsne(adata, n_pcs=dim_au_out)
# X_pca = adata.obsm["X_pca"]
# # Replace tsne by pac beacause TSNE is very slow
# X_tsne = adata.obsm["X_umap"]
# #X_tsne = TSNE(n_components=dim_au_out,method='exact').fit_transform(X_pca)
# embeddings_tsne = torch.FloatTensor(X_tsne).to(device)
# tsne_prob_prediction = source_model.predict(embeddings_tsne).detach().cpu().numpy()
# adata.obs["sens_preds_tsne"] = tsne_prob_prediction[:,1]
# adata.obs["sens_label_tsne"] = tsne_prob_prediction.argmax(axis=1)
# adata.obsm["X_tsne_pret"] = X_tsne
# Add embeddings to the adata object
embeddings_pretrain = embeddings_pretrain.detach().cpu().numpy()
adata.obsm["X_pre"] = embeddings_pretrain
################################################# END SECTION OF SC MODEL PRETRAININIG #################################################
################################################# START SECTION OF TRANSFER LEARNING TRAINING #################################################
# Using ADDA transfer learning
if args.transfer == 'ADDA':
# Set discriminator model
discriminator = Predictor(input_dim=dim_au_out, output_dim=2)
discriminator.to(device)
loss_d = nn.CrossEntropyLoss()
optimizer_d = optim.Adam(encoder.parameters(), lr=1e-2)
exp_lr_scheduler_d = lr_scheduler.ReduceLROnPlateau(optimizer_d)
# Adversairal trainning
discriminator, encoder, report_, report2_ = t.train_ADDA_model(
source_encoder,
encoder,
discriminator,
dataloaders_source,
dataloaders_pretrain,
loss_d,
loss_d,
# Should here be all optimizer d?
optimizer_d,
optimizer_d,
exp_lr_scheduler_d,
exp_lr_scheduler_d,
epochs,
device,
target_model_path)
logging.info("Transfer ADDA finished")
# DaNN model
elif args.transfer == 'DaNN':
# Set predictor loss
loss_d = nn.CrossEntropyLoss()
optimizer_d = optim.Adam(encoder.parameters(), lr=1e-2)
exp_lr_scheduler_d = lr_scheduler.ReduceLROnPlateau(optimizer_d)
# Set DaNN model
DaNN_model = DaNN(source_model=source_encoder, target_model=encoder)
DaNN_model.to(device)
def loss(x, y, GAMMA=args.GAMMA_mmd):
result = mmd.mmd_loss(x, y, GAMMA)
return result
loss_disrtibution = loss
# Tran DaNN model
DaNN_model, report_ = t.train_DaNN_model(
DaNN_model,
dataloaders_source,
dataloaders_pretrain,
# Should here be all optimizer d?
optimizer_d,
loss_d,
epochs,
exp_lr_scheduler_d,
dist_loss=loss_disrtibution,
load=load_model,
weight=args.mmd_weight,
save_path=target_model_path + "_DaNN.pkl")
encoder = DaNN_model.target_model
source_model = DaNN_model.source_model
logging.info("Transfer DaNN finished")
if (load_model == False):
ut.plot_loss(report_[0], path="figures/train_loss_" + now + ".pdf")
ut.plot_loss(report_[1],
path="figures/mmd_loss_" + now + ".pdf",
set_ylim=False)
if (args.dimreduce != 'CVAE'):
# Attribute test using integrated gradient
# Generate a target model including encoder and predictor
target_model = TargetModel(source_model, encoder)
# Allow require gradients and process label
X_allTensor.requires_grad_()
# Run integrated gradient check
# Return adata and feature integrated gradient
ytarget_allPred = target_model(X_allTensor).detach().cpu().numpy()
ytarget_allPred = ytarget_allPred.argmax(axis=1)
adata, attrp1, senNeu_c0_genes, senNeu_c1_genes = ut.integrated_gradient_differential(
net=target_model,
input=X_allTensor,
clip="positive",
target=ytarget_allPred,
adata=adata,
ig_fc=1,
save_name=reduce_model + args.predictor + prediction +
select_drug + "sensNeuron" + now)
adata, attrn1, resNeu_c0_genes, resNeu_c1_genes = ut.integrated_gradient_differential(
net=target_model,
input=X_allTensor,
clip="negative",
target=ytarget_allPred,
adata=adata,
ig_fc=1,
save_name=reduce_model + args.predictor + prediction +
select_drug + "restNeuron" + now)
sc.pl.heatmap(attrp1,
senNeu_c0_genes,
groupby='sensitive',
cmap='RdBu_r',
save=data_name + args.transfer + args.dimreduce +
"_seNc0_" + now,
show=False)
sc.pl.heatmap(attrp1,
senNeu_c1_genes,
groupby='sensitive',
cmap='RdBu_r',
save=data_name + args.transfer + args.dimreduce +
"_seNc1_" + now,
show=False)
sc.pl.heatmap(attrn1,
resNeu_c0_genes,
groupby='sensitive',
cmap='RdBu_r',
save=data_name + args.transfer + args.dimreduce +
"_reNc0_" + now,
show=False)
sc.pl.heatmap(attrn1,
resNeu_c1_genes,
groupby='sensitive',
cmap='RdBu_r',
save=data_name + args.transfer + args.dimreduce +
"_reNc1_" + now,
show=False)
# CHI2 Test on predictive features
SFD = SelectFdr(chi2)
SFD.fit(adata.raw.X, ytarget_allPred)
adata.raw.var['chi2_pval'] = SFD.pvalues_
adata.raw.var['chi2_score'] = SFD.scores_
df_chi2_genes = adata.raw.var[
(SFD.pvalues_ < 0.05) & (adata.raw.var.highly_variable == True)
& (adata.raw.var.n_cells > args.min_c)]
df_chi2_genes.sort_values(by="chi2_pval",
ascending=True,
inplace=True)
df_chi2_genes.to_csv("saved/results/chi2_pval_genes" +
args.predictor + prediction + select_drug +
now + '.csv')
else:
print()
################################################# END SECTION OF TRANSER LEARNING TRAINING #################################################
################################################# START SECTION OF PREPROCESSING FEATURES #################################################
# Extract feature embeddings
# Extract prediction probabilities
if (args.dimreduce != "CVAE"):
embedding_tensors = encoder.encode(X_allTensor)
else:
embedding_tensors = encoder.encode(X_allTensor, C_allTensor)
prediction_tensors = source_model.predictor(embedding_tensors)
embeddings = embedding_tensors.detach().cpu().numpy()
predictions = prediction_tensors.detach().cpu().numpy()
# Transform predict8ion probabilities to 0-1 labels
if (prediction == "regression"):
adata.obs["sens_preds"] = predictions
else:
adata.obs["sens_preds"] = predictions[:, 1]
adata.obs["sens_label"] = predictions.argmax(axis=1)
adata.obs["sens_label"] = adata.obs["sens_label"].astype('category')
adata.obs["rest_preds"] = predictions[:, 0]
adata.write("saved/adata/before_ann" + data_name + now + ".h5ad")
################################################# END SECTION OF PREPROCESSING FEATURES #################################################
################################################# START SECTION OF ANALYSIS AND POST PROCESSING #################################################
# Pipeline of scanpy
# Add embeddings to the adata package
adata.obsm["X_Trans"] = embeddings
#sc.tl.umap(adata)
sc.pp.neighbors(adata, n_neighbors=10, use_rep="X_Trans")
# Use t-sne on transfer learning features
sc.tl.tsne(adata, use_rep="X_Trans")
# Leiden on the data
# sc.tl.leiden(adata)
# Plot tsne
sc.pl.tsne(adata, save=data_name + now, color=["leiden"], show=False)
# Differenrial expression genes
sc.tl.rank_genes_groups(adata, 'leiden', method='wilcoxon')
sc.pl.rank_genes_groups(adata,
n_genes=args.n_DE_genes,
sharey=False,
save=data_name + now,
show=False)
# Differenrial expression genes across 0-1 classes
sc.tl.rank_genes_groups(adata, 'sens_label', method='wilcoxon')
adata = ut.de_score(adata, clustername='sens_label')
# save DE genes between 0-1 class
for label in [0, 1]:
try:
df_degs = get_de_dataframe(adata, label)
df_degs.to_csv("saved/results/DEGs_class_" + str(label) +
args.predictor + prediction + select_drug + now +
'.csv')
except:
logging.warning("Only one class, no two calsses critical genes")
# Generate reports of scores
report_df = args_df
# Data specific benchmarking
sens_pb_pret = adata.obs['sens_preds_pret']
lb_pret = adata.obs['sens_label_pret']
# sens_pb_umap = adata.obs['sens_preds_umap']
# lb_umap = adata.obs['sens_label_umap']
# sens_pb_tsne = adata.obs['sens_preds_tsne']
# lb_tsne = adata.obs['sens_label_tsne']
if ('sensitive' in adata.obs.keys()):
report_df = report_df.T
Y_test = adata.obs['sensitive']
sens_pb_results = adata.obs['sens_preds']
lb_results = adata.obs['sens_label']
le_sc = LabelEncoder()
le_sc.fit(['Resistant', 'Sensitive'])
label_descrbie = le_sc.inverse_transform(Y_test)
adata.obs['sens_truth'] = label_descrbie
color_list = ["sens_truth", "sens_label", 'sens_preds']
color_score_list = [
"Sensitive_score", "Resistant_score", "1_score", "0_score"
]
sens_score = pearsonr(adata.obs["sens_preds"],
adata.obs["Sensitive_score"])[0]
resistant_score = pearsonr(adata.obs["rest_preds"],
adata.obs["Resistant_score"])[0]
report_df['prob_sens_pearson'] = sens_score
report_df['prob_rest_pearson'] = resistant_score
try:
cluster_score_sens = pearsonr(adata.obs["1_score"],
adata.obs["Sensitive_score"])[0]
report_df['sens_pearson'] = cluster_score_sens
except:
logging.warning(
"Prediction score 1 not exist, fill adata with 0 values")
adata.obs["1_score"] = np.zeros(len(adata))
try:
cluster_score_resist = pearsonr(adata.obs["0_score"],
adata.obs["Resistant_score"])[0]
report_df['rest_pearson'] = cluster_score_resist
except:
logging.warning(
"Prediction score 0 not exist, fill adata with 0 values")
adata.obs["0_score"] = np.zeros(len(adata))
#if (data_name in ['GSE110894','GSE117872']):
ap_score = average_precision_score(Y_test, sens_pb_results)
ap_pret = average_precision_score(Y_test, sens_pb_pret)
# ap_umap = average_precision_score(Y_test, sens_pb_umap)
# ap_tsne = average_precision_score(Y_test, sens_pb_tsne)
report_dict = classification_report(Y_test,
lb_results,
output_dict=True)
f1score = report_dict['weighted avg']['f1-score']
report_df['f1_score'] = f1score
classification_report_df = pd.DataFrame(report_dict).T
classification_report_df.to_csv("saved/results/clf_report_" +
reduce_model + args.predictor +
prediction + select_drug + now +
'.csv')
# report_dict_umap = classification_report(Y_test, lb_umap, output_dict=True)
# classification_report_umap_df = pd.DataFrame(report_dict_umap).T
# classification_report_umap_df.to_csv("saved/results/clf_umap_report_" + reduce_model + args.predictor+ prediction + select_drug+now + '.csv')
report_dict_pret = classification_report(Y_test,
lb_pret,
output_dict=True)
classification_report_pret_df = pd.DataFrame(report_dict_pret).T
classification_report_pret_df.to_csv("saved/results/clf_pret_report_" +
reduce_model + args.predictor +
prediction + select_drug + now +
'.csv')
# report_dict_tsne = classification_report(Y_test, lb_tsne, output_dict=True)
# classification_report_tsne_df = pd.DataFrame(report_dict_tsne).T
# classification_report_tsne_df.to_csv("saved/results/clf_tsne_report_" + reduce_model + args.predictor+ prediction + select_drug+now + '.csv')
try:
auroc_score = roc_auc_score(Y_test, sens_pb_results)
auroc_pret = average_precision_score(Y_test, sens_pb_pret)
# auroc_umap = average_precision_score(Y_test, sens_pb_umap)
# auroc_tsne = average_precision_score(Y_test, sens_pb_tsne)
except:
logging.warning("Only one class, no ROC")
auroc_pret = auroc_umap = auroc_tsne = auroc_score = 0
report_df['auroc_score'] = auroc_score
report_df['ap_score'] = ap_score
report_df['auroc_pret'] = auroc_pret
report_df['ap_pret'] = ap_pret
# report_df['auroc_umap'] = auroc_umap
# report_df['ap_umap'] = ap_umap
# report_df['auroc_tsne'] = auroc_tsne
# report_df['ap_tsne'] = ap_tsne
ap_title = "ap: " + str(Decimal(ap_score).quantize(Decimal('0.0000')))
auroc_title = "roc: " + str(
Decimal(auroc_score).quantize(Decimal('0.0000')))
title_list = ["Ground truth", "Prediction", "Probability"]
else:
color_list = ["leiden", "sens_label", 'sens_preds']
title_list = ['Cluster', "Prediction", "Probability"]
color_score_list = color_list
# Simple analysis do neighbors in adata using PCA embeddings
#sc.pp.neighbors(adata)
# Run UMAP dimension reduction
sc.pp.neighbors(adata)
sc.tl.umap(adata)
# Run leiden clustering
# sc.tl.leiden(adata,resolution=leiden_res)
# Plot uamp
# sc.pl.umap(adata,color=[color_list[0],'sens_label_umap','sens_preds_umap'],save=data_name+args.transfer+args.dimreduce+now,show=False,title=title_list)
# Plot transfer learning on umap
sc.pl.umap(adata,
color=color_list + color_score_list,
save=data_name + args.transfer + args.dimreduce + "umap_all" +
now,
show=False)
sc.settings.set_figure_params(dpi=100,
frameon=False,
figsize=(4, 3),
facecolor='white')
sc.pl.umap(adata,
color=['sensitivity', 'leiden', 'sens_label', 'sens_preds'],
title=[
'Cell sensitivity', 'Cell clusters',
'Transfer learning prediction', 'Prediction probability'
],
save=data_name + args.transfer + args.dimreduce + "umap_pred" +
now,
show=False,
ncols=4)
sc.pl.umap(adata,
color=color_score_list,
title=[
'Sensitive gene score', 'Resistant gene score',
'Sensitive gene score (prediction)',
'Resistant gene score (prediction)'
],
save=data_name + args.transfer + args.dimreduce +
"umap_scores" + now,
show=False,
ncols=2)
# sc.pl.umap(adata,color=['Sample name'],
# save=data_name+args.transfer+args.dimreduce+"umap_sm"+now,show=False,ncols=4)
try:
sc.pl.umap(adata,
color=adata.var.sort_values(
"integrated_gradient_sens_class0").head().index,
save=data_name + args.transfer + args.dimreduce +
"_cgenes0_" + now,
show=False)
sc.pl.umap(adata,
color=adata.var.sort_values(
"integrated_gradient_sens_class1").head().index,
save=data_name + args.transfer + args.dimreduce +
"_cgenes1_" + now,
show=False)
# c0_genes = df_11_genes.loc[df_11_genes.pval<0.05].head().index
# c1_genes = df_00_genes.loc[df_00_genes.pval<0.05].head().index
# sc.pl.umap(adata,color=c0_genes,neighbors_key="Trans",save=data_name+args.transfer+args.dimreduce+"_cgenes0_TL"+now,show=False)
# sc.pl.umap(adata,color=c1_genes,neighbors_key="Trans",save=data_name+args.transfer+args.dimreduce+"_cgenes1_TL"+now,show=False)
except:
logging.warning("IG results not avaliable")
# Run embeddings using transfered embeddings
sc.pp.neighbors(adata, use_rep='X_Trans', key_added="Trans")
sc.tl.umap(adata, neighbors_key="Trans")
sc.tl.leiden(adata,
neighbors_key="Trans",
key_added="leiden_trans",
resolution=leiden_res)
sc.pl.umap(adata,
color=color_list,
neighbors_key="Trans",
save=data_name + args.transfer + args.dimreduce + "_TL" + now,
show=False,
title=title_list)
# Plot cell score on umap
sc.pl.umap(adata,
color=color_score_list,
neighbors_key="Trans",
save=data_name + args.transfer + args.dimreduce + "_score_TL" +
now,
show=False,
title=color_score_list)
# This tsne is based on transfer learning feature
sc.pl.tsne(adata,
color=color_list,
neighbors_key="Trans",
save=data_name + args.transfer + args.dimreduce + "_TL" + now,
show=False,
title=title_list)
# Use tsne origianl version to visualize
sc.tl.tsne(adata)
# This tsne is based on transfer learning feature
# sc.pl.tsne(adata,color=[color_list[0],'sens_label_tsne','sens_preds_tsne'],save=data_name+args.transfer+args.dimreduce+"_original_tsne"+now,show=False,title=title_list)
# Plot tsne of the pretrained (autoencoder) embeddings
sc.pp.neighbors(adata, use_rep='X_pre', key_added="Pret")
sc.tl.umap(adata, neighbors_key="Pret")
sc.tl.leiden(adata,
neighbors_key="Pret",
key_added="leiden_Pret",
resolution=leiden_res)
sc.pl.umap(adata,
color=[color_list[0], 'sens_label_pret', 'sens_preds_pret'],
neighbors_key="Pret",
save=data_name + args.transfer + args.dimreduce +
"_umap_Pretrain_" + now,
show=False)
# Ari between two transfer learning embedding and sensitivity label
ari_score_trans = adjusted_rand_score(adata.obs['leiden_trans'],
adata.obs['sens_label'])
ari_score = adjusted_rand_score(adata.obs['leiden'],
adata.obs['sens_label'])
pret_ari_score = adjusted_rand_score(adata.obs['leiden_origin'],
adata.obs['leiden_Pret'])
transfer_ari_score = adjusted_rand_score(adata.obs['leiden_origin'],
adata.obs['leiden_trans'])
sc.pl.umap(adata,
color=['leiden_origin', 'leiden_trans', 'leiden_Pret'],
save=data_name + args.transfer + args.dimreduce +
"_comp_Pretrain_" + now,
show=False)
#report_df = args_df
report_df['ari_score'] = ari_score
report_df['ari_trans_score'] = ari_score_trans
report_df['ari_pre_umap'] = pret_ari_score
report_df['ari_trans_umap'] = transfer_ari_score
# Trajectory of adata
adata, corelations = trajectory(adata,
root_key='sensitive',
genes_vis=senNeu_c0_genes[:5],
root=1,
now=now,
plot=True)
gene_cor = {}
# Trajectory
for g in np.array(senNeu_c0_genes):
gene = g
express_vec = adata[:, gene].X
corr = pearsonr(
np.array(express_vec).ravel(),
np.array(adata.obs["dpt_pseudotime"]))[0]
gene_cor[gene] = corr
try:
for k in corelations.keys():
report_df['cor_dpt_' + k] = corelations[k][0]
report_df['cor_pvl_' + k] = corelations[k][1]
except:
logging.warning(
"Some of the coorelation cannot be reterived from the dictional")
################################################# END SECTION OF ANALYSIS AND POST PROCESSING #################################################
################################################# START SECTION OF ANALYSIS FOR BULK DATA #################################################
# bdata = sc.AnnData(data_r)
# bdata.obs = label_r
# bulk_degs={}
# sc.tl.rank_genes_groups(bdata, select_drug, method='wilcoxon')
# bdata = ut.de_score(bdata,select_drug)
# for label in set(label_r.loc[:,select_drug]):
# try:
# df_degs = get_de_dataframe(bdata,label)
# bulk_degs[label] = df_degs.iloc[:50,:].names
# df_degs.to_csv("saved/results/DEGs_bulk_" +str(label)+ args.predictor+ prediction + select_drug+now + '.csv')
# except:
# logging.warning("Only one class, no two calsses critical genes")
# Xsource_allTensor = torch.FloatTensor(data_r.values).to(device)
# Ysource_preTensor = source_model(Xsource_allTensor)
# Ysource_prediction = Ysource_preTensor.detach().cpu().numpy()
# bdata.obs["sens_preds"] = Ysource_prediction[:,1]
# bdata.obs["sens_label"] = Ysource_prediction.argmax(axis=1)
# bdata.obs["sens_label"] = bdata.obs["sens_label"].astype('category')
# bdata.obs["rest_preds"] = Ysource_prediction[:,0]
# sc.tl.score_genes(adata, bulk_degs['sensitive'],score_name="bulk_sens_score" )
# sc.tl.score_genes(adata, bulk_degs['resistant'],score_name="bulk_rest_score" )
# sc.pl.umap(adata,color=['bulk_sens_score','bulk_rest_score'],save=data_name+args.transfer+args.dimreduce+"umap_bg_all"+now,show=False)
# try:
# bulk_score_sens = pearsonr(adata.obs["1_score"],adata.obs["bulk_sens_score"])[0]
# report_df['bulk_sens_pearson'] = bulk_score_sens
# cluster_score_resist = pearsonr(adata.obs["0_score"],adata.obs["bulk_rest_score"])[0]
# report_df['bulk_rest_pearson'] = cluster_score_resist
# except:
# logging.warning("Bulk level gene score not exist")
# Save adata
adata.write("saved/adata/" + data_name + now + ".h5ad")
# Save report
report_df = report_df.T
report_df.to_csv("saved/results/report" + reduce_model + args.predictor +
prediction + select_drug + now + '.csv')
def train(self):
# networks
self.model = Net(num_channels=self.num_channels,
base_filter=64,
num_residuals=16)
# weigh initialization
self.model.weight_init()
# optimizer
self.optimizer = optim.Adam(self.model.parameters(),
lr=self.lr,
betas=(0.9, 0.999),
eps=1e-8)
# loss function
if self.gpu_mode:
self.model.cuda()
self.L1_loss = nn.L1Loss().cuda()
else:
self.L1_loss = nn.L1Loss()
print('---------- Networks architecture -------------')
utils.print_network(self.model)
print('----------------------------------------------')
# load dataset
train_data_loader = self.load_dataset(dataset=self.train_dataset,
is_train=True)
test_data_loader = self.load_dataset(dataset=self.test_dataset[0],
is_train=False)
# set the logger
log_dir = os.path.join(self.save_dir, 'logs')
if not os.path.exists(log_dir):
os.makedirs(log_dir)
logger = Logger(log_dir)
################# Train #################
print('Training is started.')
avg_loss = []
step = 0
# test image
test_lr, test_hr, test_bc = test_data_loader.dataset.__getitem__(2)
test_lr = test_lr.unsqueeze(0)
test_hr = test_hr.unsqueeze(0)
test_bc = test_bc.unsqueeze(0)
self.model.train()
for epoch in range(self.num_epochs):
# learning rate is decayed by a factor of 2 every 40 epochs
if (epoch + 1) % 40 == 0:
for param_group in self.optimizer.param_groups:
param_group['lr'] /= 2.0
print('Learning rate decay: lr={}'.format(
self.optimizer.param_groups[0]['lr']))
epoch_loss = 0
for iter, (lr, hr, _) in enumerate(train_data_loader):
# input data (low resolution image)
if self.num_channels == 1:
x_ = Variable(hr[:, 0].unsqueeze(1))
y_ = Variable(lr[:, 0].unsqueeze(1))
else:
x_ = Variable(hr)
y_ = Variable(lr)
if self.gpu_mode:
x_ = x_.cuda()
y_ = y_.cuda()
# update network
self.optimizer.zero_grad()
recon_image = self.model(y_)
loss = self.L1_loss(recon_image, x_)
loss.backward()
self.optimizer.step()
# log
epoch_loss += loss.data[0]
print('Epoch: [%2d] [%4d/%4d] loss: %.8f' %
((epoch + 1),
(iter + 1), len(train_data_loader), loss.data[0]))
# tensorboard logging
logger.scalar_summary('loss', loss.data[0], step + 1)
step += 1
# avg. loss per epoch
avg_loss.append(epoch_loss / len(train_data_loader))
# prediction
if self.num_channels == 1:
y_ = Variable(test_lr[:, 0].unsqueeze(1))
else:
y_ = Variable(test_lr)
if self.gpu_mode:
y_ = y_.cuda()
recon_img = self.model(y_)
sr_img = recon_img[0].cpu().data
# save result image
save_dir = os.path.join(self.save_dir, 'train_result')
utils.save_img(sr_img,
epoch + 1,
save_dir=save_dir,
is_training=True)
print('Result image at epoch %d is saved.' % (epoch + 1))
# Save trained parameters of model
if (epoch + 1) % self.save_epochs == 0:
self.save_model(epoch + 1)
# calculate psnrs
if self.num_channels == 1:
gt_img = test_hr[0][0].unsqueeze(0)
lr_img = test_lr[0][0].unsqueeze(0)
bc_img = test_bc[0][0].unsqueeze(0)
else:
gt_img = test_hr[0]
lr_img = test_lr[0]
bc_img = test_bc[0]
bc_psnr = utils.PSNR(bc_img, gt_img)
recon_psnr = utils.PSNR(sr_img, gt_img)
# plot result images
result_imgs = [gt_img, lr_img, bc_img, sr_img]
psnrs = [None, None, bc_psnr, recon_psnr]
utils.plot_test_result(result_imgs,
psnrs,
self.num_epochs,
save_dir=save_dir,
is_training=True)
print('Training result image is saved.')
# Plot avg. loss
utils.plot_loss([avg_loss], self.num_epochs, save_dir=save_dir)
print('Training is finished.')
# Save final trained parameters of model
self.save_model(epoch=None)
0, 2, 3, 1), # convert to BxHxWxC
}
for key, images in info.items():
img_logger[i].image_summary(key, images, epoch + 1)
# Save trained parameters of model
torch.save(
G_list[i].state_dict(), model_dir + 'generator_' +
upsample_list[i] + '_param_epoch_%d.pkl' % (epoch + 1))
torch.save(
D_list[i].state_dict(), model_dir + 'discriminator' +
upsample_list[i] + '_param_epoch_%d.pkl' % (epoch + 1))
avg_losses = []
for i in range(model_len):
# Plot average losses
avg_losses.append(D_avg_losses[i])
avg_losses.append(G_avg_losses[i])
utils.plot_loss(avg_losses,
num_epochs,
upsample_list[i],
save=True,
save_dir=save_dir)
# Save trained parameters of model
torch.save(G_list[i].state_dict(),
model_dir + 'generator_' + upsample_list[i] + '_param.pkl')
torch.save(D_list[i].state_dict(),
model_dir + 'discriminator' + upsample_list[i] + '_param.pkl')
def trainNet(
datapath='.',
nepochs=1,
learning_rate=0.001,
batch_size=64,
cuda=False,
savedir='./',
lossPlotName='loss.png',
num_workers=24,
):
'''
Our basic training file
'''
if "/" not in savedir[-1]:
savedir += "/"
if cuda:
print(f"Running on GPU")
device = torch.device('cuda')
net = model.VGG().to(device)
else:
print(f"Running on CPU")
device = torch.device('cpu')
net = model.VGG()
# Dataset
#imagenet = datasets.ImageNet('/research/imgnet/ILSVRC2013_DET_train/',
# split='train')
t = transforms.Compose([
transforms.Resize(256),
transforms.CenterCrop(224),
transforms.ToTensor(),
])
train_images = datasets.CIFAR10('.',
train=True,
download=True,
transform=t)
train_data = torch.utils.data.DataLoader(train_images,
batch_size=batch_size,
shuffle=True,
num_workers=num_workers)
#criterion = nn.MSELoss()
criterion = nn.CrossEntropyLoss()
optimizer = optim.Adam(net.parameters(), lr=learning_rate)
epoch_loss_array = torch.zeros(nepochs)
print(f"Train data {len(train_images)}")
for epoch in range(nepochs):
net.train()
epoch_loss = 0
for i, (img, label) in enumerate(train_data):
optimizer.zero_grad()
out = net(img.to(device))
loss = criterion(out.to(device), label.to(device))
epoch_loss += loss.item()
loss.backward()
optimizer.step()
epoch_loss /= (i + 1)
print(f"Epoch {epoch} loss {epoch_loss}")
epoch_loss_array[epoch] = epoch_loss
utils.plot_loss(epoch_loss_array, savedir + lossPlotName)
net.load_state_dict(torch.load('final_model.pt'))
# Testing
with torch.no_grad():
net.eval()
test_loss = 0.
test_images = datasets.CIFAR10('.',
train=False,
download=True,
transform=t)
test_data = torch.utils.data.DataLoader(test_images,
batch_size=batch_size,
shuffle=True,
num_workers=num_workers)
classes = [
'plane', 'car', 'bird', 'cat', 'deer', 'dog', 'frog', 'horse',
'ship', 'truck'
]
corrects = np.zeros(len(classes), dtype=np.int)
totals = np.zeros(len(classes), dtype=np.int)
trues = np.zeros(len(test_images))
preds = np.zeros(len(test_images))
for i, (img, label) in enumerate(test_data):
optimizer.zero_grad()
out = net(img.to(device))
trues[i * batch_size:(i + 1) *
batch_size] = label.to('cpu').numpy()
preds[i * batch_size:(i + 1) *
batch_size] = [np.argmax(o) for o in out.to('cpu').numpy()]
for o, l in zip(out, label):
o = o.to('cpu').numpy()
l = l.to('cpu').numpy()
totals[l] += 1
if np.argmax(o) == l:
corrects[l] += 1
loss = criterion(out.to(device), label.to(device))
test_loss += loss.item()
if i == 0:
utils.plot_examples(img[:9].to('cpu'),
out[:9].to('cpu').numpy(), classes,
label[:9].to('cpu').numpy(),
savedir + "examples.png")
test_loss /= (i + 1)
print(f"Test loss {test_loss}")
utils.confusionMatrix(trues, preds, classes)
torch.save(net.state_dict(), "final_model.pt")
#print(f"Corrects {corrects}")
#print(f"Totals {totals}")
print("Accuracy")
print(f"Name\tCorrects\tAccuracy")
for i in range(len(classes)):
print(f"{classes[i]}\t{corrects[i]}\t\t{corrects[i]/totals[i]}")
print(30 * "-")
print(f"Sum\t{np.sum(corrects)}\t\t{np.sum(corrects)/np.sum(totals)}")
plt.savefig("task4b_softmax_weight_" + str(i) + "_" + str(l) +
"_.png")
if show_plots:
plt.show()
else:
plt.close()
# Save the norm for plotting later
norm_list.append(np.linalg.norm(model.w))
# Plotting of accuracy for differente values of lambdas (task 4c)
for l, train_history, val_history in zip(l2_lambdas, train_history_list,
val_history_list):
plt.ylim([0.7, .93])
plt.xlim([-100, 10000])
utils.plot_loss(train_history["accuracy"],
"Training Accuracy $\lambda$ = " + str(l))
utils.plot_loss(val_history["accuracy"],
"Validation Accuracy $\lambda$ = " + str(l))
plt.xlabel("Number of Training Steps")
plt.ylabel("Accuracy")
plt.legend()
plt.savefig("task4c_l2_reg_accuracy.png")
if show_plots:
plt.show()
else:
plt.close()
# Task 4d - Plotting of the l2 norm for each weight
plt.plot(l2_lambdas, norm_list)
plt.title("Plot of length of w vs $\lambda$")
valid_loss, valid_acc, mcc = evaluate(model,
valid_iterator,
criterion,
fields=fields,
metric=binary_accuracy)
print(f'\t Val. Loss: {valid_loss:.6f} | Val. Accuracy: {valid_acc}')
train_accs += [train_acc]
train_losses += [train_loss]
valid_accs += [valid_acc]
valid_losses += [valid_loss]
# scheduler.step()
early_stopping(valid_loss, model)
if valid_acc > 0.55 and mcc > 0.056:
torch.save({
'net': model.state_dict(),
'text': TEXT,
}, f'roberta-model-acc-{valid_acc}-{mcc}.pth')
if early_stopping.early_stop:
print(f"Epochs: {epoch} - Early Stopping...")
break
plot_acc(train_accs,
valid_accs,
fname=f"roberta-epochs-{epochs}-acc-{valid_acc}.png")
plot_loss(train_losses,
valid_losses,
fname=f"roberta-epochs-{epochs}-loss-{valid_acc}.png")
model,
learning_rate,
batch_size,
shuffle_dataset,
X_train,
Y_train,
X_val,
Y_val,
)
train_history, val_history = trainer.train(num_epochs)
val_acc.append(val_history["accuracy"])
weights.append(model.w)
# Plot accuracy
utils.plot_loss(val_history["accuracy"],
"Validation Accuracy with $\lambda$ ={}".format(i))
plt.ylim([0.60, .95])
plt.xlabel("Number of Training Steps")
plt.ylabel("Accuracy")
plt.legend()
plt.savefig("task4c_l2_reg_accuracy.png")
plt.show()
# Plotting of softmax weights (Task 4b)
#weight = visualModeltraining() - task4b.py
#plt.imsave("task4b_softmax_weight_2.png", weight, cmap="gray")
# Plotting of accuracy for difference values of lambdas (task 4c)
#plt.savefig("task4c_l2_reg_accuracy.png")
# Task 4d - Plotting of the l2 norm for each weight. Could be more elegant, but it's just a bar plot so frankly
D_avg_loss = torch.mean(torch.FloatTensor(D_losses))
G_avg_loss = torch.mean(torch.FloatTensor(G_losses))
# avg loss values for plot
D_avg_losses.append(D_avg_loss)
G_avg_losses.append(G_avg_loss)
# Show result for test image
gen_image = G(Variable(test_input.cuda()))
gen_image = gen_image.cpu().data
utils.plot_test_result(test_input,
test_target,
gen_image,
epoch,
save=True,
save_dir=save_dir)
# Plot average losses
utils.plot_loss(D_avg_losses,
G_avg_losses,
params.num_epochs,
save=True,
save_dir=save_dir)
# Make gif
utils.make_gif(params.dataset, params.num_epochs, save_dir=save_dir)
# Save trained parameters of model
torch.save(G.state_dict(), model_dir + 'generator_param.pkl')
torch.save(D.state_dict(), model_dir + 'discriminator_param.pkl')
l2_reg_lambda=l2_reg_lambda)
print("Final Train Cross Entropy Loss:",
cross_entropy_loss(Y_train, model.forward(X_train)))
print("Final Validation Cross Entropy Loss:",
cross_entropy_loss(Y_test, model.forward(X_test)))
print("Final Test Cross Entropy Loss:",
cross_entropy_loss(Y_val, model.forward(X_val)))
print("Train accuracy:", calculate_accuracy(X_train, Y_train, model))
print("Validation accuracy:", calculate_accuracy(X_val, Y_val, model))
print("Test accuracy:", calculate_accuracy(X_test, Y_test, model))
# Plot loss
utils.plot_loss(train_loss, "Training Loss")
utils.plot_loss(val_loss, "Validation Loss")
plt.ylim([0, .4])
plt.legend()
plt.savefig("binary_train_loss_stopping=%s_epochs=%s.png" %
(with_stopping, str(num_epochs)))
plt.show()
# Plot accuracy
plt.ylim([0.93, .99])
utils.plot_loss(train_accuracy, "Training Accuracy")
utils.plot_loss(val_accuracy, "Validation Accuracy")
plt.legend()
plt.savefig("binary_train_accuracy_stopping=%s_epochs=%s.png" %
(with_stopping, str(num_epochs)))
os.mkdir(path)
except FileExistsError:
pass
# Model loading and training
model_path = os.path.join(path, MODEL_NAME + ".mdl")
try:
print("Loading model ", MODEL_NAME)
model.load_state_dict(torch.load(model_path))
except FileNotFoundError:
print("No such model. Training a new one!")
train(train_loader1, 1, lf)
train(train_loader2, 2, lf)
train(train_loader3, 3, lf)
train(train_loader4, 4, lf)
plot_loss(total_loss, path)
torch.save(model.state_dict(), model_path)
model.eval()
test_loss = 0
results_path = os.path.join(path, "results")
try:
os.mkdir(results_path)
except FileExistsError:
pass
with torch.no_grad():
for i in range(4):
h_views, v_views, i_views, d_views, center, gt, mask, index = test_set.get_scene(i)
data_h = torch.tensor(h_views, device=device).float()
data_v = torch.tensor(v_views, device=device).float()
data_d = torch.tensor(d_views, device=device).float()
momentum_gamma,
use_momentum,
model_128,
learning_rate,
batch_size,
shuffle_data,
X_train,
Y_train,
X_val,
Y_val,
)
train_history_128, val_history_128 = trainer_128.train(num_epochs)
plt.subplot(1, 2, 1)
utils.plot_loss(train_history_32["loss"],
"Task 4 Model - 32 Neurons",
npoints_to_average=10)
utils.plot_loss(train_history_64["loss"],
"Task 4 Model - 64 Neurons",
npoints_to_average=10)
utils.plot_loss(train_history_128["loss"],
"Task 4 Model - 128 Neurons",
npoints_to_average=10)
plt.xlabel("Number of Training Steps")
plt.ylabel("Training loss")
plt.legend()
plt.ylim([0, .4])
plt.subplot(1, 2, 2)
plt.ylim([0.9, 1.01])
utils.plot_loss(val_history_32["accuracy"],
def train(self):
# networks
self.model = Net(num_channels=self.num_channels, scale_factor=self.scale_factor, d=56, s=12, m=4)
# weigh initialization
self.model.weight_init(mean=0.0, std=0.02)
# optimizer
self.momentum = 0.9
self.optimizer = optim.SGD(self.model.parameters(), lr=self.lr, momentum=self.momentum)
# loss function
if self.gpu_mode:
self.model.cuda()
self.MSE_loss = nn.MSELoss().cuda()
else:
self.MSE_loss = nn.MSELoss()
print('---------- Networks architecture -------------')
utils.print_network(self.model)
print('----------------------------------------------')
# load dataset
train_data_loader = self.load_dataset(dataset='train')
test_data_loader = self.load_dataset(dataset='test')
# set the logger
log_dir = os.path.join(self.save_dir, 'logs')
if not os.path.exists(log_dir):
os.mkdir(log_dir)
logger = Logger(log_dir)
################# Train #################
print('Training is started.')
avg_loss = []
step = 0
# test image
test_input, test_target = test_data_loader.dataset.__getitem__(2)
test_input = test_input.unsqueeze(0)
test_target = test_target.unsqueeze(0)
self.model.train()
for epoch in range(self.num_epochs):
epoch_loss = 0
for iter, (input, target) in enumerate(train_data_loader):
# input data (low resolution image)
if self.gpu_mode:
x_ = Variable(utils.shave(target, border_size=2*self.scale_factor).cuda())
y_ = Variable(input.cuda())
else:
x_ = Variable(utils.shave(target, border_size=2*self.scale_factor))
y_ = Variable(input)
# update network
self.optimizer.zero_grad()
recon_image = self.model(y_)
loss = self.MSE_loss(recon_image, x_)
loss.backward()
self.optimizer.step()
# log
epoch_loss += loss.data[0]
print("Epoch: [%2d] [%4d/%4d] loss: %.8f" % ((epoch + 1), (iter + 1), len(train_data_loader), loss.data[0]))
# tensorboard logging
logger.scalar_summary('loss', loss.data[0], step + 1)
step += 1
# avg. loss per epoch
avg_loss.append(epoch_loss / len(train_data_loader))
# prediction
recon_imgs = self.model(Variable(test_input.cuda()))
recon_img = recon_imgs[0].cpu().data
gt_img = utils.shave(test_target[0], border_size=2 * self.scale_factor)
lr_img = utils.shave(test_input[0], border_size=2)
bc_img = utils.shave(utils.img_interp(test_input[0], self.scale_factor), border_size=2 * self.scale_factor)
# calculate psnrs
bc_psnr = utils.PSNR(bc_img, gt_img)
recon_psnr = utils.PSNR(recon_img, gt_img)
# save result images
result_imgs = [gt_img, lr_img, bc_img, recon_img]
psnrs = [None, None, bc_psnr, recon_psnr]
utils.plot_test_result(result_imgs, psnrs, epoch + 1, save_dir=self.save_dir, is_training=True)
print("Saving training result images at epoch %d" % (epoch + 1))
# Save trained parameters of model
if (epoch + 1) % self.save_epochs == 0:
self.save_model(epoch + 1)
# Plot avg. loss
utils.plot_loss([avg_loss], self.num_epochs, save_dir=self.save_dir)
print("Training is finished.")
# Save final trained parameters of model
self.save_model(epoch=None)
# Initiate the train and test generators with data augumentation
train_generator, valid_generator, test_generator = load_data('./data/')
# Save the model according to certain conditions
# checkpoint = ModelCheckpoint("vgg16_1.h5", monitor='val_acc', verbose=1, save_best_only=True, save_weights_only=False, mode='auto', period=1)
early = EarlyStopping(monitor='val_acc',
min_delta=0,
patience=10,
verbose=1,
mode='auto')
# Train the model
history = model_final.fit_generator(
train_generator,
steps_per_epoch=train_generator.n // batch_size,
epochs=epochs,
validation_data=valid_generator,
validation_steps=valid_generator.n // batch_size,
callbacks=[early]) # ,checkpoint])
STEP_SIZE_TEST = test_generator.n // test_generator.batch_size
score = model_final.evaluate_generator(generator=test_generator,
steps=STEP_SIZE_TEST)
print('Test loss:', score[0])
print('Test accuracy:', score[1])
plot_accuracy(history, path.join('./result/', out_file_name))
plot_loss(history, path.join('./result/', out_file_name))
save_model(model_final, path.join('./out/', out_file_name))
save_history(history, path.join('./out/', out_file_name))
criterion,
fields=fields,
metric=binary_accuracy)
print(f'\t Val. Loss: {valid_loss:.6f} | Val. Accuracy: {valid_acc}')
train_accs += [train_acc]
train_losses += [train_loss]
valid_accs += [valid_acc]
valid_losses += [valid_loss]
# scheduler.step()
early_stopping(valid_loss, model)
if valid_acc > 0.55 and mcc > 0.056:
torch.save(
{
'net': model.state_dict(),
'text': TEXT,
'dim': embed_dim,
}, f'te-model-acc-{valid_acc}-{mcc}.pth')
if early_stopping.early_stop:
print(f"Epochs: {epoch} - Early Stopping...")
break
plot_acc(train_accs,
valid_accs,
fname=f"ternn-epochs-{epochs}-acc-{valid_acc}.png")
plot_loss(train_losses,
valid_losses,
fname=f"ternn-epochs-{epochs}-loss-{valid_acc}.png")
l1 = DenseLayer((784, num_hidden_neuron), optimizer)
bn1 = BatchNorm((num_hidden_neuron,), optimizer)
l2 = DenseLayer((num_hidden_neuron, num_hidden_neuron), optimizer)
l3 = DenseLayer((num_hidden_neuron, 10), optimizer)
def calc_model(x, y=None, train_mode=False):
a1 = op.relu(bn1.compute(l1.compute(x), train_mode=train_mode))
a2 = op.relu(l2.compute(a1))
z3 = l3.compute(a2)
if not train_mode:
return op.softmax(z3)
else:
return op.softmax_cross_entropy(z3, y)
def loss_function(y, m, train_mode=False):
if train_mode:
return op.sum(m)
else:
return op.sum(-y * op.log(m))
train_loss_values, test_loss_values, test_score_values = learn(mnist_in_data, mnist_out_data, mnist_test_in_data,
mnist_test_out_data, calc_model, loss_function,
optimizer, score_func=mnist.score_result,
batch_size=128, epoch_number=10)
plot_loss(train_loss_values, test_loss_values)
)
train_history, val_history = trainer.train(num_epochs)
print("Final Train Cross Entropy Loss:",
cross_entropy_loss(Y_train, model.forward(X_train)))
print("Final Validation Cross Entropy Loss:",
cross_entropy_loss(Y_val, model.forward(X_val)))
print("Train accuracy:", calculate_accuracy(X_train, Y_train, model))
print("Validation accuracy:", calculate_accuracy(X_val, Y_val, model))
# Plot loss for first model (task 2c)
plt.figure(figsize=(16, 10))
plt.subplot(1, 2, 1)
plt.ylim([0., .5])
utils.plot_loss(train_history["loss"],
"Training Loss",
npoints_to_average=10)
utils.plot_loss(val_history["loss"], "Validation Loss")
plt.legend()
plt.xlabel("Number of Training Steps")
plt.ylabel("Cross Entropy Loss - Average")
# Plot accuracy
plt.subplot(1, 2, 2)
plt.ylim([0.90, .99])
utils.plot_loss(train_history["accuracy"], "Training Accuracy")
utils.plot_loss(val_history["accuracy"], "Validation Accuracy")
plt.xlabel("Number of Training Steps")
plt.ylabel("Accuracy")
plt.legend()
plt.savefig("task2c_train_loss.png")
# construct model and ship to GPU
hidden_list = list(map(int, args.hiddens.split(',')))
model = MADE(xtr.size(1),
hidden_list,
xtr.size(1) * 2,
num_masks=args.num_masks)
print("number of model parameters:",
sum([np.prod(p.size()) for p in model.parameters()]))
model.cuda()
# set up the optimizer
opt = torch.optim.Adam(model.parameters(),
args.learning_rate,
weight_decay=args.weight_decay)
scheduler = torch.optim.lr_scheduler.StepLR(opt, step_size=45, gamma=0.1)
# list to store loss
loss_tr = []
loss_te = []
loss_od = []
# start the training
for epoch in range(args.epoch):
scheduler.step(epoch)
loss_tr.append(run_epoch('train'))
loss_te.append(run_epoch('test')) # run validation, which is pos
loss_od.append(run_epoch('ood')) # run test, which is ood
print("optimization done")
plot_loss(file_name, loss_tr, loss_te, loss_od)
# run_epoch('test')
def main():
batch_size = args.batch_size
iterations = args.iterations
device = args.device
# hair_classes, eye_classes = 12, 10
# num_classes = hair_classes + eye_classes
hair_class, eye_class, face_class, glass_class = 6, 4, 3, 2
num_classes = hair_class + eye_class + face_class + glass_class
latent_dim = 100
smooth = 0.9
config = 'WGANGP_batch{}_steps{}'.format(batch_size, iterations)
print('Configuration: {}'.format(config))
root_dir = './{}/images'.format(args.train_dir)
tags_file = './{}/cartoon_attr.txt'.format(args.train_dir)
# hair_prior = np.load('../{}/hair_prob.npy'.format(args.train_dir))
# eye_prior = np.load('../{}/eye_prob.npy'.format(args.train_dir))
random_sample_dir = '{}/{}/random_generation'.format(args.sample_dir, config)
fixed_attribute_dir = '{}/{}/fixed_attributes'.format(args.sample_dir, config)
checkpoint_dir = '{}/{}'.format(args.checkpoint_dir, config)
if not os.path.exists(random_sample_dir):
os.makedirs(random_sample_dir)
if not os.path.exists(fixed_attribute_dir):
os.makedirs(fixed_attribute_dir)
if not os.path.exists(checkpoint_dir):
os.makedirs(checkpoint_dir)
########## Start Training ##########
transform = Transform.Compose([Transform.ToTensor(),
Transform.Normalize((0.5, 0.5, 0.5), (0.5, 0.5, 0.5))])
dataset = Anime(root_dir = root_dir, tags_file = tags_file, transform = transform)
shuffler = Shuffler(dataset = dataset, batch_size = args.batch_size)
G = Generator(latent_dim = latent_dim, class_dim = num_classes).to(device)
D = Discriminator(hair_classes=hair_class, eye_classes=eye_class, face_classes=face_class, glass_classes=glass_class).to(device)
G_optim = optim.Adam(G.parameters(), betas = [args.beta, 0.999], lr = args.lr)
D_optim = optim.Adam(D.parameters(), betas = [args.beta, 0.999], lr = args.lr)
d_log, g_log, classifier_log = [], [], []
criterion = torch.nn.BCELoss()
# criterion = torch.nn.NLLLoss()
for step_i in range(1, iterations + 1):
real_label = torch.ones(batch_size).to(device)
fake_label = torch.zeros(batch_size).to(device)
soft_label = torch.Tensor(batch_size).uniform_(smooth, 1).to(device)
# we need gradient when training descriminator
for p in D.parameters():
p.requires_grad = True
for d_iter in range(5):
D_optim.zero_grad()
# D.zero_grad()
# Train discriminator
real_img, hair_tags, eye_tags, face_tags, glass_tags = shuffler.get_batch()
real_img, hair_tags, eye_tags, face_tags, glass_tags = real_img.to(device), hair_tags.to(device), eye_tags.to(device), face_tags.to(device), glass_tags.to(device)
# real_tag = torch.cat((hair_tags, eye_tags), dim = 1)
with torch.no_grad(): # totally freeze G , training D
z = torch.randn(batch_size, latent_dim).to(device)
fake_tag = get_random_label(batch_size = batch_size,
hair_classes = hair_class,
eye_classes = eye_class,
face_classes = face_class,
glass_classes = glass_class).to(device)
real_img.requires_grad = True
real_score, real_hair_predict, real_eye_predict, real_face_predict, real_glass_predict = D(real_img)
real_discrim_loss = real_score.mean()
# real_discrim_loss = criterion(real_score, soft_label)
# fake_discrim_loss = criterion(fake_score, fake_label)
real_hair_aux_loss = criterion(real_hair_predict, hair_tags)
real_eye_aux_loss = criterion(real_eye_predict, eye_tags)
real_face_aux_loss = criterion(real_face_predict, face_tags)
real_glass_aux_loss = criterion(real_glass_predict, glass_tags)
real_classifier_loss = real_hair_aux_loss + real_eye_aux_loss + real_face_aux_loss + real_glass_aux_loss
fake_img = G(z, fake_tag).to(device)
fake_score, _ , _ , _ , _ = D(fake_img)
fake_discrim_loss = fake_score.mean()
gradient_penalty = calculate_gradient_penalty(D, real_img.detach(), fake_img.detach(), batch_size, device)
# discrim_loss = real_discrim_loss + fake_discrim_loss
discrim_loss = fake_discrim_loss - real_discrim_loss + gradient_penalty
classifier_loss = real_classifier_loss * args.classification_weight
classifier_log.append(classifier_loss.item())
D_loss = discrim_loss + classifier_loss
# D_optim.zero_grad()
D_loss.backward()
D_optim.step()
# Train generator
for p in D.parameters():
p.requires_grad = False
G_optim.zero_grad()
# G.zero_grad()
z = torch.randn(batch_size, latent_dim).to(device)
z.requires_grad = True
fake_tag = get_random_label(batch_size = batch_size,
hair_classes = hair_class,
eye_classes = eye_class,
face_classes = face_class,
glass_classes = glass_class).to(device)
hair_tag = fake_tag[:, :hair_class]
eye_tag = fake_tag[:, 6:10]
face_tag = fake_tag[:, 10:13]
glass_tag = fake_tag[:, 13:15]
fake_img = G(z, fake_tag).to(device)
fake_score, hair_predict, eye_predict, face_predict, glass_predict = D(fake_img)
discrim_loss = fake_score.mean()
G_discrim_loss = -discrim_loss
# discrim_loss = criterion(fake_score, real_label)
hair_aux_loss = criterion(hair_predict, hair_tag)
eye_aux_loss = criterion(eye_predict, eye_tag)
face_aux_loss = criterion(face_predict, face_tag)
glass_aux_loss = criterion(glass_predict, glass_tag)
classifier_loss = hair_aux_loss + eye_aux_loss + face_aux_loss + glass_aux_loss
G_loss = classifier_loss * args.classification_weight + G_discrim_loss
# G_optim.zero_grad()
G_loss.backward()
G_optim.step()
########## Updating logs ##########
d_log.append(D_loss.item())
g_log.append(G_loss.item())
show_process(total_steps = iterations, step_i = step_i,
g_log = g_log, d_log = d_log, classifier_log = classifier_log)
########## Checkpointing ##########
if step_i == 1:
save_image(denorm(real_img[:16,:,:,:]), os.path.join(random_sample_dir, 'real.png'), nrow=4)
if step_i % args.sample == 0:
save_image(denorm(fake_img[:16,:,:,:]), os.path.join(random_sample_dir, 'fake_step_{}.png'.format(step_i)), nrow=4)
if step_i % args.check == 0:
save_model(model = G, optimizer = G_optim, step = step_i, log = tuple(g_log),
file_path = os.path.join(checkpoint_dir, 'G_{}.ckpt'.format(step_i)))
save_model(model = D, optimizer = D_optim, step = step_i, log = tuple(d_log),
file_path = os.path.join(checkpoint_dir, 'D_{}.ckpt'.format(step_i)))
plot_loss(g_log = g_log, d_log = d_log, file_path = os.path.join(checkpoint_dir, 'loss.png'))
plot_classifier_loss(log = classifier_log, file_path = os.path.join(checkpoint_dir, 'classifier loss.png'))
generation_by_attributes(model = G, device = args.device, step = step_i,
latent_dim = latent_dim, hair_classes = hair_class,
eye_classes = eye_class, face_classes = face_class,
glass_classes = glass_class,
sample_dir = fixed_attribute_dir)
l2_reg_lambda=0)
print("Final Train Cross Entropy Loss:",
cross_entropy_loss(Y_train, model.forward(X_train)))
print("Final Validation Cross Entropy Loss:",
cross_entropy_loss(Y_test, model.forward(X_test)))
print("Final Test Cross Entropy Loss:",
cross_entropy_loss(Y_val, model.forward(X_val)))
print("Final Train accuracy:", calculate_accuracy(X_train, Y_train, model))
print("Final Validation accuracy:", calculate_accuracy(X_val, Y_val, model))
print("Final Test accuracy:", calculate_accuracy(X_test, Y_test, model))
# Plot loss
plt.ylim([0.01, .2])
utils.plot_loss(train_loss, "Training Loss")
utils.plot_loss(val_loss, "Validation Loss")
plt.xlabel("Gradient steps")
plt.legend()
plt.savefig("softmax_train_loss.png")
plt.show()
# Plot accuracy
plt.ylim([0.8, .95])
utils.plot_loss(train_accuracy, "Training Accuracy")
utils.plot_loss(val_accuracy, "Validation Accuracy")
plt.xlabel("Gradient steps")
plt.legend()
plt.show()
lambda_values = [0, 0.1]
name = exp_name + '_' + str(date.today())
with open('models/conv_net_'+name+'.pkl', 'wb') as f:
cPickle.dump(conv_net, f, -1)
conv_net.save_params_to('models/params_'+name)
# ----- Train set ----
train_predictions = conv_net.predict_proba(X)
make_submission_file(train_predictions[:sample_size], images_id[:sample_size],
output_filepath='models/training_'+name+'.csv')
# ----- Test set ----
X_test, _, images_id_test = load_numpy_arrays(args['test_file'])
print "Test:"
print "X_test.shape:", X_test.shape
predictions = conv_net.predict_proba(X_test)
make_submission_file(predictions, images_id_test,
output_filepath='submissions/submission_'+name+'.csv')
# ----- Make plots ----
plot_loss(conv_net, "models/loss_"+name+".png", show=False)
plot_conv_weights(conv_net.layers_[1], figsize=(4, 4))
plt.savefig('models/weights_'+name+'.png')
plot_conv_activity(conv_net.layers_[1], X[0:1])
plt.savefig('models/activity_'+name+'.png')
plot_occlusion(conv_net, X[:5], y[:5])
plt.savefig('models/occlusion_'+name+'.png')
train_history, val_history = trainer.train(num_epochs)
print("Improved weights")
print("Train accuracy:", calculate_accuracy(X_train, Y_train, model))
print("Validation accuracy:", calculate_accuracy(X_val, Y_val, model))
print("Final Validation Cross Entropy Loss:",
cross_entropy_loss(Y_val, model.forward(X_val)))
print("Final Train Cross Entropy Loss:",
cross_entropy_loss(Y_train, model.forward(X_train)))
#Plotting
plt.figure(figsize=(20, 12))
plt.subplot(1, 2, 1)
plt.ylim([0., .5])
utils.plot_loss(train_history["loss"],
"Training Loss",
npoints_to_average=10)
utils.plot_loss(val_history["loss"], "Validation Loss")
plt.legend()
plt.xlabel("Number of Training Steps")
plt.ylabel("Cross Entropy Loss - Average")
# Plot accuracy
plt.subplot(1, 2, 2)
plt.ylim([0.91, 1.01])
utils.plot_loss(train_history["accuracy"], "Training Accuracy")
utils.plot_loss(val_history["accuracy"], "Validation Accuracy")
plt.xlabel("Number of Training Steps")
plt.ylabel("Accuracy")
plt.legend()
#plt.savefig("task3a_improvedweight.png")
plt.show()
