def test(self):
tf.reset_default_graph()
X = tf.placeholder("float", [None, None, None, cf.Channel])
Y = tf.placeholder("float", [None, cf.Class_num])
keep_prob = tf.placeholder("float")
## Load network model
logits = Model(x=X, phase='Test')
## Secure GPU Memory
config = tf.ConfigProto()
config.gpu_options.allow_growth = True
print('Test start !!')
table_gt_pred = np.zeros((cf.Class_num, cf.Class_num), dtype=np.int)
with tf.Session(config=config) as sess:
saver = tf.train.Saver()
saver.restore(sess, cf.Save_path)
img_paths = get_imagelist()
for img_path in img_paths:
img, img_info = load_image(img_path)
img = np.expand_dims(img, axis=0)
gt = get_gt(img_path)
pred = logits.eval(feed_dict={X: img, keep_prob: 1.0})[0]
pred_label = np.argmax(pred)
table_gt_pred[gt, pred_label] += 1
print(img_path, pred)
for cls_ind in range(cf.Class_num):
print(cf.Class_label[cls_ind], np.round(table_gt_pred[cls_ind], 3))
def main(args):
# Image preprocessing
transform = transforms.Compose([
transforms.RandomHorizontalFlip(),
transforms.ToTensor(),
transforms.Normalize((0.485, 0.456, 0.406), (0.229, 0.224, 0.225))
])
# Load vocabulary wrapper.
with open(args['vocab_path'], 'rb') as f:
vocab = pickle.load(f)
# Build data loader
data_loader = get_loader(args['image_dir'],
args['caption_path'],
vocab,
transform,
args['batch_size'],
shuffle=True,
num_workers=args['num_workers'])
# Train the model
model = Image2Seq((64, 64), vocab.word2idx)
model.sess.run(tf.global_variables_initializer())
print("Model Compiled")
for epoch in range(args['n_epoch']):
for i, (images, captions, lengths) in enumerate(data_loader):
loss = model.partial_fit(images.numpy(), captions.numpy(), lengths)
print('[%d / %d] Loss: %.4f' % (i, len(data_loader), loss))
if i % 20 == 0:
sample_image = load_image(args['sample_img']).numpy()
model.infer(sample_image, vocab.idx2word)
def make_checkpoint_image(cfg, net, save_path):
"""
Function to use a test content image, run it through the network, and save
it to track training progress.
Args:
--cfg: <dict> The training config dictionary
Returns:
None. Just writes the progress image.
"""
# Get image path for test image
im_path = cfg['test_image']
# Load and preprocess image
im = load_image(im_path, cfg)
# Turn it into a batch
im = im.unsqueeze(0)
# Pass it through the network
styled = net(im)[0]
# Post process image
styled = postprocess_image(styled.detach())
# Convert to BGR and write output
styled = cv2.cvtColor(styled, cv2.COLOR_RGB2BGR)
cv2.imwrite(save_path, styled)
return
def get_batch(step, questions, answers, images_paths, answers_vocab_len):
batch_start = (step * batch_size) % len(questions)
batch_in_questions = questions[batch_start:batch_start + batch_size]
batch_in_images = list()
batch_out = np.zeros((batch_size, answers_vocab_len))
for i in range(batch_start, batch_start + len(batch_in_questions)):
batch_in_images.append(load_image(images_paths[i]))
batch_out[i - batch_start, answers[i] - 1] = 1
tmp = batch_size - len(batch_in_questions)
if tmp > 0:
for i in range(0, tmp):
batch_out[i + len(batch_in_questions), answers[i] - 1] = 1
batch_in_images.append(load_image(images_paths[i]))
batch_in_questions = np.concatenate(
(batch_in_questions, questions[0:tmp]), axis=0)
return batch_in_questions, np.asarray(batch_in_images), batch_out
def get_batch_for_test(step, questions, answers, images_paths,
answers_vocab_len):
batch_start = (step * batch_size) % len(questions)
batch_in_questions = questions[batch_start:batch_start + batch_size]
batch_in_images = list()
batch_out = np.zeros((len(batch_in_questions), answers_vocab_len))
for i in range(batch_start, batch_start + len(batch_in_questions)):
batch_in_images.append(load_image(images_paths[i]))
batch_out[i - batch_start, answers[i] - 1] = 1
return batch_in_questions, np.asarray(batch_in_images), batch_out, len(
batch_in_questions)
def train(configuration):
"""
the main training loop
:param configuration: the config file
:return:
"""
image_path = configuration['image_path']
print('using images from {}'.format(image_path))
image_saving_path_mean = configuration['image_saving_path_mean']
print('saving result images with mean loss to {}'.format(image_saving_path_mean))
image_saving_path_mean_std = configuration['image_saving_path_mean_std']
print('saving result images with mean + std loss to {}'.format(image_saving_path_mean_std))
image_saving_path_mean_std_skew = configuration['image_saving_path_mean_std_skew']
print('saving result images with mean + std + skew loss to {}'.format(image_saving_path_mean_std_skew))
image_saving_path_mean_std_skew_kurtosis = configuration['image_saving_path_mean_std_skew_kurtosis']
print('saving result images with mean + std + skew + kurtosis loss to {}'.format(image_saving_path_mean_std_skew_kurtosis))
model_dir = configuration['model_dir']
print('vgg-19 model dir is {}'.format(model_dir))
vgg_model = get_vgg_model(configuration)
print('got vgg model')
number_style_images, style_image_file_paths = get_images(configuration)
print('got {} style images'.format(number_style_images))
all_images = []
for i in range(1, 6):
print('using style loss module: {}'.format(i))
style_loss_module = get_loss_module(i)
for j in range(number_style_images):
print('computing image {} with style loss module {}'.format(j, i))
style_image = load_image(style_image_file_paths[j])
layer_images = [style_image.squeeze(0)]
model, style_losses = get_full_style_model(configuration, vgg_model, style_image, style_loss_module)
for k in range(4):
print('computing loss at layer {}, image {}, loss module {}'.format(k, j, i))
torch.manual_seed(13)
image_noise = torch.randn(style_image.data.size()).to(device)
layer_images += [train_full_style_model(model, style_losses, image_noise, k, i * (k+1) * 20000)
.squeeze(0)]
print('saving images at the different layers')
save_layer_images(configuration, layer_images, i, j)
all_images += [layer_images]
print('saving the side-by-side comparisons')
save_all_images(configuration, all_images, number_style_images)
def train_mmd(configuration):
"""
training loop utilizing the MMD loss
:param configuration: the config file
:return:
"""
image_path = configuration['image_path']
print('using images from {}'.format(image_path))
image_saving_path_mmd = configuration['image_saving_path_mmd']
print('saving result images with mmd loss to {}'.format(image_saving_path_mmd))
model_dir = configuration['model_dir']
print('vgg-19 model dir is {}'.format(model_dir))
vgg_model = get_vgg_model(configuration)
print('got vgg model')
number_style_images, style_image_file_paths = get_images(configuration)
print('got {} style images'.format(number_style_images))
i = 6
print('using mmd loss module: {}'.format(i))
style_loss_module = get_loss_module(i)
for j in range(number_style_images):
print('computing image {} with style loss module {}'.format(j, i))
style_image = load_image(style_image_file_paths[j])
layer_images = [style_image.squeeze(0)]
model, style_losses = get_full_style_model(configuration, vgg_model, style_image, style_loss_module)
for k in range(4):
print('computing loss at layer {}, image {}, loss module {}'.format(k, j, i))
torch.manual_seed(13)
image_noise = torch.randn(style_image.data.size()).to(device)
steps = (k+2) * 50000
style_weight = 1000
layer_images += [train_full_style_model(model, style_losses, image_noise, k,
steps, style_weight, early_stopping=True).squeeze(0)]
# for debug
save_layer_images(configuration, layer_images, i, j)
print('saving images at the different layers')
save_layer_images(configuration, layer_images, i, j)
print('finished')
def read_tf_weight(self):
model = adapted_deeplab_tf.DeepLab(
num_classes=self.nSketchClasses,
upsample_mode=self.upsample_mode,
ignore_class_bg=self.ignore_class_bg)
tfconfig = tf.ConfigProto()
tfconfig.gpu_options.allow_growth = True
sess = tf.Session(config=tfconfig)
sess.run(tf.global_variables_initializer())
ckpt = tf.train.get_checkpoint_state(self.tf_model_dir)
snapshot_loader = tf.train.Saver()
print('Trained model found, loaded', ckpt.model_checkpoint_path)
snapshot_loader.restore(sess, ckpt.model_checkpoint_path)
weight_keys = [var.name[:-2] for var in tf.global_variables()]
weight_vals = sess.run(tf.global_variables())
for weight_key, weight_val in zip(weight_keys, weight_vals):
if 'factor' not in weight_key and 'Adam' not in weight_key and 'beta1_power' not in weight_key \
and 'beta2_power' not in weight_key and 'global_step' not in weight_key:
# print(weight_key, np.array(weight_val).shape)
self.tf_weight_dict[weight_key] = weight_val
print('Obtain tf_weight_dict done!')
# get output
if self.display:
infer_image, infer_image_raw = load_image(
self.image_path, self.config.mean,
True) # shape = [1, H, W, 3]
feed_dict = {model.images: infer_image, model.labels: 0}
self.pred_label_tf = sess.run(
[model.pred_label], feed_dict=feed_dict)[0] # [1, H, W, 1]
if self.ignore_class_bg:
pred_label_tf = self.pred_label_tf + 1
else:
pred_label_tf = self.pred_label_tf
pred_label_tf = np.squeeze(pred_label_tf)
pred_label_tf[infer_image_raw[:, :, 0] != 0] = 0 # [H, W]
visualize_semantic_segmentation(pred_label_tf, self.colorMap)
def train_gram(configuration):
"""
training loop utilizing the Gram matrix loss
:param configuration: the config file
:return:
"""
image_path = configuration['image_path']
print('using images from {}'.format(image_path))
image_saving_path_gram = configuration['image_saving_path_gram']
print('saving result images with gram loss to {}'.format(image_saving_path_gram))
model_dir = configuration['model_dir']
print('vgg-19 model dir is {}'.format(model_dir))
vgg_model = get_vgg_model(configuration)
print('got vgg model')
number_style_images, style_image_file_paths = get_images(configuration)
print('got {} style images'.format(number_style_images))
all_images = []
i = 5
print('using gram loss module: {}'.format(i))
style_loss_module = get_loss_module(i)
for j in range(number_style_images):
print('computing image {} with style loss module {}'.format(j, i))
style_image = load_image(style_image_file_paths[j])
layer_images = [style_image.squeeze(0)]
model, style_losses = get_full_style_model(configuration, vgg_model, style_image, style_loss_module)
for k in range(4):
print('computing loss at layer {}, image {}, loss module {}'.format(k, j, i))
torch.manual_seed(13)
image_noise = torch.randn(style_image.data.size()).to(device)
steps = i * (k+1) * 20000
layer_images += [train_full_style_model(model, style_losses, image_noise, k, steps).squeeze(0)]
print('saving images at the different layers')
save_layer_images(configuration, layer_images, i, j)
all_images += [layer_images]
print('finished')
def run(cfg):
"""Runs the neural style transfer"""
# Load image and make batch
image = load_image(cfg['content_image'], rescale=False)
image = image.unsqueeze(0)
# Set device if gpu is available
device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
# Print alert if device is cpu
if device == 'cpu':
print('No GPU detected. This may take longer to run...')
# Figure out if artist is VanGoh (VanGoh weights have slightly different shape)
vangoh = False
if cfg['artist'] == 'VanGoh':
vangoh = True
# Build network
net = ImageTransformationNet(vangoh=vangoh).to(device)
# Load trained weights
artist = torch.load(f'./trained_weights/{cfg["artist"]}.pth')
net.load_state_dict(artist['net_state_dict'])
# net.load_state_dict(artist)
# Put image through net
processed_im = net(image)[0]
# Postprocess image
output = postprocess_image(processed_im)
# Write output to output file
output = cv2.cvtColor(output, cv2.COLOR_RGB2BGR)
cv2.imwrite(cfg['output_file'], output)
print('Output image written successfully!')
return
def train(cfg):
# Set device if gpu is available
device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
# Build network
net = ImageTransformationNet().to(device)
# Setup optimizer
optimizer = optim.Adam(net.parameters())
# Load state if resuming training
if cfg['resume']:
checkpoint = torch.load(cfg['resume'])
net.load_state_dict(checkpoint['net_state_dict'])
optimizer.load_state_dict(checkpoint['opt_state_dict'])
# Get starting epoch and batch (expects weight file in form EPOCH_<>_BATCH_<>.pt)
parts = cfg['resume'].split('_')
first_epoch = int(checkpoint['epoch'])
first_batch = int(parts[-1].split('.')[0])
# Setup dataloader
train_data = tqdm(build_data_loader(cfg), initial=first_batch)
else:
# Setup dataloader
train_data = tqdm(build_data_loader(cfg))
# Set first epoch and batch
first_epoch = 1
first_batch = 0
# Fetch style image and style grams
style_im = load_image(cfg['style_image'], cfg)
style_grams = get_style_grams(style_im, cfg)
# Setup log file if specified
log_dir = Path('logs')
log_dir.mkdir(parents=True, exist_ok=True)
if cfg['log_file'] and not cfg['resume']:
today = datetime.datetime.today().strftime('%m/%d/%Y')
header = f'Feed-Forward Style Transfer Training Log - {today}'
with open(cfg['log_file'], 'w+') as file:
file.write(header + '\n\n')
# Setup log CSV if specified
if cfg['csv_log_file'] and not cfg['resume']:
utils.setup_csv(cfg)
for epoch in range(first_epoch, cfg['epochs'] + 1):
# Keep track of per epoch loss
content_loss = 0
style_loss = 0
total_var_loss = 0
train_loss = 0
num_batches = 0
# Setup first batch to start enumerate at proper place
if epoch == first_epoch:
start = first_batch
else:
start = 0
for i, batch in enumerate(train_data, start=start):
batch = batch.to(device)
# Put batch through network
batch_styled = net(batch)
# Get vgg activations for styled and unstyled batch
features = vgg_activations(batch_styled)
content_features = vgg_activations(batch)
# Get loss
c_loss, s_loss = perceptual_loss(features=features,
content_features=content_features,
style_grams=style_grams,
cfg=cfg)
tv_loss = total_variation_loss(batch_styled, cfg)
total_loss = c_loss + s_loss + tv_loss
# Backpropogate
total_loss.backward()
# Do one step of optimization
optimizer.step()
# Clear gradients before next batch
optimizer.zero_grad()
# Update summary statistics
with torch.no_grad():
content_loss += c_loss.item()
style_loss += s_loss.item()
total_var_loss += tv_loss.item()
train_loss += total_loss.item()
num_batches += 1
# Update progress bar
avg_loss = round(train_loss / num_batches, 2)
avg_c_loss = round(content_loss / num_batches, 2)
avg_s_loss = round(style_loss / num_batches, 1)
avg_tv_loss = round(total_var_loss / num_batches, 3)
train_data.set_description(
f'C - {avg_c_loss} | S - {avg_s_loss} | TV - {avg_tv_loss} | Total - {avg_loss}'
)
train_data.refresh()
# Create progress image if specified
if cfg['image_checkpoint'] and ((i + 1) % cfg['image_checkpoint']
== 0):
save_path = str(
Path(
cfg['image_checkpoint_dir'],
f'EPOCH_{str(epoch).zfill(3)}_BATCH_{str(i+1).zfill(5)}.png'
))
utils.make_checkpoint_image(cfg, net, save_path)
# Save weights if specified
if cfg['save_checkpoint'] and ((i + 1) % cfg['save_checkpoint']
== 0):
save_path = str(
Path(
cfg['save_checkpoint_dir'],
f'EPOCH_{str(epoch).zfill(3)}_BATCH_{str(i+1).zfill(5)}.pth'
))
checkpoint = {
'epoch': epoch,
'net_state_dict': net.state_dict(),
'opt_state_dict': optimizer.state_dict(),
'loss': avg_loss
}
torch.save(checkpoint, save_path)
# Write progress row to CSV
if cfg['csv_checkpoint'] and ((i + 1) % cfg['csv_checkpoint']
== 0):
row = [
epoch, i + 1, avg_c_loss, avg_s_loss, avg_tv_loss, avg_loss
]
utils.write_progress_row(cfg, row)
# Write loss at end of each epoch
if cfg['log_file']:
avg_loss = round(train_loss / num_batches, 4)
line = f'EPOCH {epoch} | Loss - {avg_loss}'
with open(cfg['log_file'], 'a') as file:
file.write(line + '\n')
# Save network if specified
if cfg['epoch_save_checkpoint'] and (
epoch % cfg['epoch_save_checkpoint'] == 0):
save_path = str(
Path(cfg['save_checkpoint_dir'],
f'EPOCH_{str(epoch).zfill(3)}.pth'))
checkpoint = {
'epoch': epoch,
'net_state_dict': net.state_dict(),
'opt_state_dict': optimizer.state_dict(),
'loss': round(train_loss / num_batches, 4)
}
torch.save(checkpoint, save_path)
from keras.utils import np_utils
from keras.models import load_model
import data_loader
from keras.models import model_from_json
im_size = 320
model = densenet.DenseNet(nb_classes=1,
img_dim=(320, 320, 1),
depth=22,
nb_dense_block=4,
growth_rate=12,
nb_filter=16,
dropout_rate=0.2,
weight_decay=1E-4)
model.load_weights('./save_models/[email protected]')
X_valid_path, Y_valid = data_loader.load_path(root_path='./valid/XR_HUMERUS',
size=im_size)
X_valid = data_loader.load_image(X_valid_path, im_size)
y1 = model.predict(X_valid, batch_size=None, verbose=0, steps=None)
j = len(y1)
for i in range(0, j):
if y1[i] > 0.5:
print(X_valid_path[i], ":\t", "Positive\t", y1[i])
else:
print(X_valid_path[i], ":\t", "Negative\t", y1[i])
__author__ = 'Mohammad'
import tensorflow as tf
import numpy as np
from data_loader import load_image
sess = tf.Session()
saver = tf.train.import_meta_graph(
'data/tensorflow-resnet-pretrained-20160509/ResNet-L152.meta')
saver.restore(sess,
'data/tensorflow-resnet-pretrained-20160509/ResNet-L152.ckpt')
graph = tf.get_default_graph()
images = graph.get_tensor_by_name("images:0")
img = load_image("data/train2014/COCO_train2014_000000000009.jpg")
img_features = graph.get_tensor_by_name("avg_pool:0")
features = sess.run(img_features, {images: img[np.newaxis, :]})
print features[0], features[0].shape
def run_MURA(batch_size, nb_epoch, depth, nb_dense_block, nb_filter,
growth_rate, dropout_rate, learning_rate, weight_decay,
plot_architecture):
""" Run MURA experiments
:param batch_size: int -- batch size
:param nb_epoch: int -- number of training epochs
:param depth: int -- network depth
:param nb_dense_block: int -- number of dense blocks
:param nb_filter: int -- initial number of conv filter
:param growth_rate: int -- number of new filters added by conv layers
:param dropout_rate: float -- dropout rate
:param learning_rate: float -- learning rate
:param weight_decay: float -- weight decay
:param plot_architecture: bool -- whether to plot network architecture
"""
###################
# Data processing #
###################
#/home/yu/Documents/tensorflow/MURA/MURA-v1.1
# the path of MURA dataset
im_size = 320 #测试修改参数 size root_path nb_epoch nb_dense_block
X_train_path, Y_train = data_loader.load_path(root_path='../train',
size=im_size)
X_valid_path, Y_valid = data_loader.load_path(root_path='../valid',
size=im_size)
X_valid = data_loader.load_image(X_valid_path, im_size) #提前加载验证集
Y_valid = np.asarray(Y_valid)
nb_classes = 1
img_dim = (im_size, im_size, 1) #加上最后一个维度,类型为tuple
###################
# Construct model #
###################
model = densenet.DenseNet(nb_classes,
img_dim,
depth,
nb_dense_block,
growth_rate,
nb_filter,
dropout_rate=dropout_rate,
weight_decay=weight_decay)
# Model output
model.summary()
# Build optimizer
opt = Adam(lr=learning_rate, beta_1=0.9, beta_2=0.999, epsilon=1e-08)
model.compile(loss='binary_crossentropy',
optimizer=opt,
metrics=["accuracy"])
if plot_architecture:
from keras.utils import plot_model
plot_model(model,
to_file='./figures/densenet_archi.png',
show_shapes=True)
####################
# Network training #
####################
print("Start Training")
list_train_loss = []
list_valid_loss = []
list_learning_rate = []
best_record = [100, 0, 100, 100] #记录最优 [验证集损失函数值,准确率，训练集数据集loss差值,acc差值]
start_time = datetime.datetime.now()
for e in range(nb_epoch):
if e == int(0.25 * nb_epoch):
K.set_value(model.optimizer.lr, np.float32(learning_rate / 10.))
if e == int(0.5 * nb_epoch):
K.set_value(model.optimizer.lr, np.float32(learning_rate / 50.))
if e == int(0.75 * nb_epoch):
K.set_value(model.optimizer.lr, np.float32(learning_rate / 100.))
split_size = batch_size
num_splits = len(X_train_path) / split_size
arr_all = np.arange(len(X_train_path)).astype(int)
random.shuffle(arr_all) #随机打乱index索引顺序
arr_splits = np.array_split(arr_all, num_splits)
l_train_loss = []
batch_train_loss = []
start = datetime.datetime.now()
for i, batch_idx in enumerate(arr_splits):
X_batch_path, Y_batch = [], []
for idx in batch_idx:
X_batch_path.append(X_train_path[idx])
Y_batch.append(Y_train[idx])
X_batch = data_loader.load_image(Path=X_batch_path, size=im_size)
Y_batch = np.asarray(Y_batch)
train_logloss, train_acc = model.train_on_batch(X_batch, Y_batch)
l_train_loss.append([train_logloss, train_acc])
batch_train_loss.append([train_logloss, train_acc])
if i % 100 == 0:
loss_1, acc_1 = np.mean(np.array(l_train_loss), 0)
loss_2, acc_2 = np.mean(np.array(batch_train_loss), 0)
batch_train_loss = [] #当前100batch的损失函数和准确率
print(
'[Epoch {}/{}] [Batch {}/{}] [Time: {}] [all_batchs--> train_epoch_logloss: {:.5f}, train_epoch_acc:{:.5f}] '
.format(e + 1, nb_epoch, i, len(arr_splits),
datetime.datetime.now() - start, loss_1, acc_1),
'[this_100_batchs-->train_batchs_logloss: {:.5f}, train_batchs_acc:{:.5f}]'
.format(loss_2, acc_2))
# 运行验证集
valid_logloss, valid_acc = model.evaluate(X_valid,
Y_valid,
verbose=0,
batch_size=64)
list_train_loss.append(np.mean(np.array(l_train_loss), 0).tolist())
list_valid_loss.append([valid_logloss, valid_acc])
list_learning_rate.append(float(K.get_value(model.optimizer.lr)))
# to convert numpy array to json serializable
print('[Epoch %s/%s] [Time: %s, Total_time: %s]' %
(e + 1, nb_epoch, datetime.datetime.now() - start,
datetime.datetime.now() - start_time),
end='')
print(
'[train_loss_and_acc:{:.5f} {:.5f}] [valid_loss_acc:{:.5f} {:.5f}]'
.format(list_train_loss[-1][0], list_train_loss[-1][1],
list_valid_loss[-1][0], list_valid_loss[-1][1]))
d_log = {}
d_log["batch_size"] = batch_size
d_log["nb_epoch"] = nb_epoch
d_log["optimizer"] = opt.get_config()
d_log["train_loss"] = list_train_loss
d_log["valid_loss"] = list_valid_loss
d_log["learning_rate"] = list_learning_rate
json_file = os.path.join('./log/experiment_log_MURA.json')
with open(json_file, 'w') as fp:
json.dump(d_log, fp, indent=4, sort_keys=True)
record = [
valid_logloss,
valid_acc,
abs(valid_logloss - list_train_loss[-1][0]),
abs(valid_acc - list_train_loss[-1][1]),
]
if ((record[0] <= best_record[0]) & (record[1] >= best_record[1])):
if e <= int(0.25 * nb_epoch) | (record[2] <= best_record[2]) & (
record[3] <= best_record[3]): #四分之一epoch之后加入差值判定
best_record = record #记录最小的 [验证集损失函数值,准确率，训练集数据loss差值,acc差值]
print('saving the best model:epoch', e + 1, best_record)
model.save('save_models/best_MURA_modle@epochs{}.h5'.format(e +
1))
model.save('save_models/MURA_modle@epochs{}.h5'.format(e + 1))
X_train_path, Y_train = data_loader.load_path(
root_path='/Users/curlyfu/Documents/MURA-6105/MURA-v1.1/train1',
size=im_size)
X_valid_path, Y_valid = data_loader.load_path(
root_path='/Users/curlyfu/Documents/MURA-6105/MURA-v1.1/valid',
size=im_size)
from sklearn.model_selection import train_test_split
# x is the feature, and y is the label.
X_train_path, X_test_path, Y_train, Y_test = train_test_split(X_train_path,
Y_train,
test_size=0.2)
print("loading train set......")
X_train = data_loader.load_image(X_train_path, im_size) # load trainset
print("loading train set finished")
print("Y_train....")
Y_train = np.asarray(Y_train)
print("Y_train finished")
print("loading valid set......")
X_valid = data_loader.load_image(X_valid_path, im_size) # loadvalidset
Y_valid = np.asarray(Y_valid)
print("loading test set......")
X_test = data_loader.load_image(X_test_path, im_size) # loadtest
Y_test = np.asarray(Y_test)
nb_classes = 1
img_dim = (im_size, im_size, 1) # plus the last dimension, type tuple
def run_MURA(batch_size, nb_epoch, depth, nb_dense_block, nb_filter,
growth_rate, dropout_rate, learning_rate, weight_decay,
plot_architecture):
"""
Run MURA experiments
:parameter batch_size: int -- batch size
:parameter nb_epoch: int -- number of training epochs
:parameter depth: int -- network depth
:parameter nb_dense_block: int -- number of dense blocks
:parameter nb_filter: int -- initial number of conv filter
:parameter growth_rate: int -- number of new filters added by conv layers
:parameter dropout_rate: float -- dropout rate
:parameter learning_rate: float -- learning rate
:parameter weight_decay: float -- weight decay
:parameter plot_architecture: bool -- whether to plot network architecture
"""
###################
# Data processing #
###################
im_size = 320 #Test modification parameters size root_path nb_epoch nb_dense_block
X_train_path, Y_train = data_loader.load_path(
root_path='./train/XR_HUMERUS', size=im_size)
X_valid_path, Y_valid = data_loader.load_path(
root_path='./valid/XR_HUMERUS', size=im_size)
X_valid = data_loader.load_image(
X_valid_path, im_size) #Load verification set ahead of time
Y_valid = np.asarray(Y_valid)
nb_classes = 1
img_dim = (im_size, im_size, 1) #Plus the last dimension, type is tuple
###################
# Construct model #
###################
model = densenet.DenseNet(nb_classes,
img_dim,
depth,
nb_dense_block,
growth_rate,
nb_filter,
dropout_rate=dropout_rate,
weight_decay=weight_decay)
# Model output
model.summary()
# Build optimizer
opt = Adam(lr=learning_rate, beta_1=0.9, beta_2=0.999, epsilon=1e-08)
model.compile(loss='binary_crossentropy',
optimizer=opt,
metrics=["accuracy"])
if plot_architecture:
from keras.utils import plot_model
plot_model(model,
to_file='./figures/densenet_archi.png',
show_shapes=True)
####################
# Network training #
####################
print("Start Training")
list_train_loss = []
list_valid_loss = []
list_learning_rate = []
best_record = [
100, 0, 100, 100
] #Recording optimal [verification set loss function value, accuracy rate, training set data set loss difference,acc difference]
start_time = datetime.datetime.now()
for e in range(nb_epoch):
if e == int(0.25 * nb_epoch):
K.set_value(model.optimizer.lr, np.float32(learning_rate / 10.))
if e == int(0.5 * nb_epoch):
K.set_value(model.optimizer.lr, np.float32(learning_rate / 50.))
if e == int(0.75 * nb_epoch):
K.set_value(model.optimizer.lr, np.float32(learning_rate / 100.))
split_size = batch_size
num_splits = len(X_train_path) / split_size
arr_all = np.arange(len(X_train_path)).astype(int)
random.shuffle(arr_all) #Randomly disrupted index order
arr_splits = np.array_split(arr_all, num_splits)
l_train_loss = []
batch_train_loss = []
start = datetime.datetime.now()
for i, batch_idx in enumerate(arr_splits):
X_batch_path, Y_batch = [], []
for idx in batch_idx:
X_batch_path.append(X_train_path[idx])
Y_batch.append(Y_train[idx])
X_batch = data_loader.load_image(Path=X_batch_path, size=im_size)
Y_batch = np.asarray(Y_batch)
train_logloss, train_acc = model.train_on_batch(X_batch, Y_batch)
l_train_loss.append([train_logloss, train_acc])
batch_train_loss.append([train_logloss, train_acc])
if i % 100 == 0:
loss_1, acc_1 = np.mean(np.array(l_train_loss), 0)
loss_2, acc_2 = np.mean(np.array(batch_train_loss), 0)
batch_train_loss = [
] #Current 100 batch loss function and accuracy
print(
'[Epoch {}/{}] [Batch {}/{}] [Time: {}] [all_batchs--> train_epoch_logloss: {:.5f}, train_epoch_acc:{:.5f}] '
.format(e + 1, nb_epoch, i, len(arr_splits),
datetime.datetime.now() - start, loss_1, acc_1),
'[this_100_batchs-->train_batchs_logloss: {:.5f}, train_batchs_acc:{:.5f}]'
.format(loss_2, acc_2))
# Run verification set
valid_logloss, valid_acc = model.evaluate(X_valid,
Y_valid,
verbose=0,
batch_size=64)
list_train_loss.append(np.mean(np.array(l_train_loss), 0).tolist())
list_valid_loss.append([valid_logloss, valid_acc])
list_learning_rate.append(float(K.get_value(model.optimizer.lr)))
# to convert numpy array to json serializable
print('[Epoch %s/%s] [Time: %s, Total_time: %s]' %
(e + 1, nb_epoch, datetime.datetime.now() - start,
datetime.datetime.now() - start_time),
end='')
print(
'[train_loss_and_acc:{:.5f} {:.5f}] [valid_loss_acc:{:.5f} {:.5f}]'
.format(list_train_loss[-1][0], list_train_loss[-1][1],
list_valid_loss[-1][0], list_valid_loss[-1][1]))
d_log = {}
d_log["batch_size"] = batch_size
d_log["nb_epoch"] = nb_epoch
d_log["optimizer"] = opt.get_config()
d_log["train_loss"] = list_train_loss
d_log["valid_loss"] = list_valid_loss
d_log["learning_rate"] = list_learning_rate
json_file = os.path.join('./log/experiment_log_MURA.json')
with open(json_file, 'w') as fp:
json.dump(d_log, fp, indent=4, sort_keys=True)
record = [
valid_logloss,
valid_acc,
abs(valid_logloss - list_train_loss[-1][0]),
abs(valid_acc - list_train_loss[-1][1]),
]
if ((record[0] <= best_record[0]) & (record[1] >= best_record[1])):
if e <= int(0.25 * nb_epoch) | (record[2] <= best_record[2]) & (
record[3] <= best_record[3]
): #Add a difference judgment after a quarter epoch
best_record = record #Record the smallest [validation set loss function value, accuracy rate, training set data loss difference, acc difference]
print('saving the best model:epoch', e + 1, best_record)
model.save('save_models/best_MURA_modle@epochs{}.h5'.format(e +
1))
model.save('save_models/MURA_modle@epochs{}.h5'.format(e + 1))
def train(configuration):
"""
this is the main training loop
:param configuration: the config
:return:
"""
image_saving_path = configuration['image_saving_path']
print('saving result images to {}'.format(image_saving_path))
model_dir = configuration['model_dir']
print('vgg-19 model dir is {}'.format(model_dir))
vgg_model = get_vgg_model(configuration)
print('got vgg model')
style_image_path = configuration['style_image_path']
number_style_images, style_image_file_paths = get_images(style_image_path)
print('got {} style images'.format(number_style_images))
print('using the style images from path: {}'.format(style_image_path))
content_image_path = configuration['content_image_path']
number_content_images, content_image_file_paths = get_images(
content_image_path)
print('got {} content images'.format(number_content_images))
print('using the content images from path: {}'.format(content_image_path))
steps = configuration['steps']
print('training for {} steps'.format(steps))
content_weight = configuration['content_weight']
style_weight = configuration['style_weight']
print('content weight: {}, style weight: {}'.format(
content_weight, style_weight))
lr = configuration['lr']
print('using a learning rate of {}'.format(lr))
for i in range(number_style_images):
print('style image {}'.format(i))
for j in range(number_content_images):
images = []
print('content image {}'.format(j))
style_image = load_image(style_image_file_paths[i])
content_image = load_image(content_image_file_paths[j])
images += [style_image.squeeze(0).cpu()]
print('got style image')
images += [content_image.squeeze(0).cpu()]
print('got content image')
for k in range(1, 6):
print('training transfer image with loss {}'.format(k))
loss_writer = LossWriter(
os.path.join(
configuration['folder_structure'].get_parent_folder(),
'./loss/loss'))
loss_writer.write_header(columns=[
'iteration', f'style_loss_{k}', f'content_loss_{k}',
f'loss_{k}'
])
torch.manual_seed(1)
image_noise = torch.randn(style_image.data.size()).to(device)
model, style_losses, content_losses = get_full_style_model(
configuration, vgg_model, style_image, content_image,
get_style_loss_module(k), get_content_loss_module())
# this is to align the loss magnitudes of Gram matrix loss and moment loss
if k == 1:
style_weight *= 100
img = train_neural_style_transfer(
model, lr, style_losses, content_losses, image_noise,
steps, style_weight, content_weight,
loss_writer).squeeze(0).cpu()
images += [img.clone()]
save_single_image(configuration, img, -k, -k)
print('got transfer image')
save_image(configuration, images, i, j)
def train_mmd(configuration):
"""
this is the MMD training loop
:param configuration: the config
:return:
"""
image_saving_path = configuration['image_saving_path']
print('saving result images to {}'.format(image_saving_path))
model_dir = configuration['model_dir']
print('vgg-19 model dir is {}'.format(model_dir))
vgg_model = get_vgg_model(configuration)
print('got vgg model')
style_image_path = configuration['style_image_path']
number_style_images, style_image_file_paths = get_images(style_image_path)
print('got {} style images'.format(number_style_images))
print('using the style images from path: {}'.format(style_image_path))
content_image_path = configuration['content_image_path']
number_content_images, content_image_file_paths = get_images(
content_image_path)
print('got {} content images'.format(number_content_images))
print('using the content images from path: {}'.format(content_image_path))
loss_writer = LossWriter(
os.path.join(configuration['folder_structure'].get_parent_folder(),
'./loss/loss'))
loss_writer.write_header(
columns=['iteration', 'style_loss', 'content_loss', 'loss'])
print(style_image_file_paths)
print(content_image_file_paths)
images = []
for i in range(number_style_images):
print('style image {}'.format(i))
for j in range(number_content_images):
style_image = load_image(style_image_file_paths[i])
content_image = load_image(content_image_file_paths[j])
images += [style_image.squeeze(0).cpu()]
print('got style image')
images += [content_image.squeeze(0).cpu()]
print('got content image')
print('training transfer image with loss {} (MMD loss)'.format(2))
torch.manual_seed(1)
image_noise = torch.randn(style_image.data.size()).to(device)
model, style_losses, content_losses = get_full_style_model(
configuration, vgg_model, style_image, content_image,
get_style_loss_module(2), get_content_loss_module())
steps = configuration['steps']
print('training for {} steps'.format(steps))
content_weight = configuration['content_weight']
style_weight = configuration['style_weight']
print('content weight: {}, style weight: {}'.format(
content_weight, style_weight))
lr = configuration['lr']
print('learning rate: {}'.format(lr))
img = train_neural_style_transfer(model, lr, style_losses,
content_losses, image_noise,
steps, style_weight,
content_weight,
loss_writer).squeeze(0).cpu()
save_image(configuration, img, j, i)
print('got transfer image')
config = tf.ConfigProto(
device_count = {'GPU': 0}
)
with tf.Session(config=config) as sess:
init_op = tf.global_variables_initializer()
model = sess.run(init_op)
if os.path.isfile(os.getcwd() + "/" + cfg.weights_dir + "/checkpoint"):
saver.restore(sess, model_file)
print("Restored model")
yolo.set_training(False)
anchors = np.reshape(np.array(cfg.anchors), [-1, 2])
images = np.array([load_image(sys.argv[1])])
img = images[0]
#normalise data between 0 and 1
imgs = np.array(images)/127.5-1
boxes = sess.run(yolo.output, feed_dict={
yolo.x: imgs,
yolo.anchors: anchors,
})
proc_boxes = yolo.convert_net_to_bb(boxes, filter_top=True).tolist()[0]
raw_img = load_raw_image(sys.argv[1])
def segment_main(**kwargs):
mode = kwargs['mode']
mu = FLAGS.mean
if FLAGS.ignore_class_bg:
nSketchClasses = FLAGS.nSketchClasses - 1
print('Ignore BG;', nSketchClasses, 'classes')
else:
nSketchClasses = FLAGS.nSketchClasses
print('Not Ignore BG;', nSketchClasses, 'classes')
data_aug = FLAGS.data_aug if mode == 'train' else False
model = adapted_deeplab_model.DeepLab(num_classes=nSketchClasses,
lrn_rate=FLAGS.learning_rate,
lrn_rate_end=FLAGS.learning_rate_end,
optimizer=FLAGS.optimizer,
upsample_mode=FLAGS.upsample_mode,
data_aug=data_aug,
image_down_scaling=FLAGS.image_down_scaling,
ignore_class_bg=FLAGS.ignore_class_bg,
mode=mode)
snapshot_saver = tf.train.Saver(max_to_keep=5)
tfconfig = tf.ConfigProto()
tfconfig.gpu_options.allow_growth = True
sess = tf.Session(config=tfconfig)
sess.run(tf.global_variables_initializer())
snapshot_dir = os.path.join(FLAGS.outputs_base_dir, FLAGS.snapshot_folder_name)
os.makedirs(snapshot_dir, exist_ok=True)
ckpt = tf.train.get_checkpoint_state(snapshot_dir)
if not ckpt:
if mode == 'train':
pretrained_model = FLAGS.resnet_pretrained_model_path
load_var = {var.op.name: var for var in tf.global_variables()
if var.op.name.startswith('ResNet')
and 'factor' not in var.op.name
and 'Adam' not in var.op.name
and 'beta1_power' not in var.op.name
and 'beta2_power' not in var.op.name
and 'fc_final_sketch46' not in var.op.name
and 'global_step' not in var.op.name # count from 0
}
snapshot_loader = tf.train.Saver(load_var)
print('Firstly training, loaded', pretrained_model)
snapshot_loader.restore(sess, pretrained_model)
else:
raise Exception("No pre-trained model for %s" % mode)
else:
load_var = {var.op.name: var for var in tf.global_variables()
if var.op.name.startswith('ResNet')
and 'global_step' not in var.op.name # count from 0
}
snapshot_loader = tf.train.Saver(load_var)
print('Trained model found, loaded', ckpt.model_checkpoint_path)
snapshot_loader.restore(sess, ckpt.model_checkpoint_path)
if mode == 'train':
log_dir = os.path.join(FLAGS.outputs_base_dir, FLAGS.log_folder_name)
os.makedirs(log_dir, exist_ok=True)
snapshot_file = os.path.join(snapshot_dir, 'iter_%d.tfmodel')
summary_op = tf.summary.merge_all()
summary_writer = tf.summary.FileWriter(log_dir, graph=sess.graph)
duration_time_n_step = 0
for n_iter in range(FLAGS.max_iteration):
start_time = time.time()
print('\n#' + str(n_iter))
## select image index
image_idx = random.randint(1, FLAGS.nTrainImgs)
## load images
image_name = 'L0_sample' + str(image_idx) + '.png' # e.g. L0_sample5564.png
# print("Load:", image_name)
image_path = os.path.join(FLAGS.data_base_dir, mode, 'DRAWING_GT', image_name)
im = load_image(image_path, mu) # shape = [1, H, W, 3]
# print("Ori shape", im.shape)
## load label
label_name = 'sample_' + str(image_idx) + '_class.mat' # e.g. sample_1_class.mat
label_path = os.path.join(FLAGS.data_base_dir, mode, 'CLASS_GT', label_name)
label = load_label(label_path) # shape = [1, H, W], [0, 46]
if FLAGS.ignore_class_bg:
label = label - 1 # [-1, 45]
label[label == -1] = 255 # [0-45, 255]
feed_dict = {model.images: im, model.labels: label}
_, learning_rate_, global_step, cost, pred, pred_label = \
sess.run([model.train_step,
model.learning_rate,
model.global_step,
model.cost,
model.pred,
model.pred_label],
feed_dict=feed_dict)
# print('pred.shape', pred.shape) # (1, H_scale, W_scale, nClasses)
print('learning_rate_', learning_rate_)
# print('global_step', global_step)
print('cost', cost)
## display left time
duration_time = time.time() - start_time
duration_time_n_step += duration_time
if n_iter % FLAGS.count_left_time_freq == 0 and n_iter != 0:
left_step = FLAGS.max_iteration - n_iter
left_sec = left_step / FLAGS.count_left_time_freq * duration_time_n_step
print("Duration_time_%d_step: %s. Left time: %s" % (
FLAGS.count_left_time_freq,
str(timedelta(seconds=duration_time_n_step)),
str(timedelta(seconds=left_sec))))
duration_time_n_step = 0
## summary
if n_iter % FLAGS.summary_write_freq == 0 and n_iter != 0:
summary_str = sess.run(summary_op, feed_dict=feed_dict)
summary_writer.add_summary(summary_str, n_iter)
summary_writer.flush()
## save model
if (n_iter + 1) % FLAGS.save_model_freq == 0 or (n_iter + 1) >= FLAGS.max_iteration:
snapshot_saver.save(sess, snapshot_file % (n_iter + 1))
print('model saved to ' + snapshot_file % (n_iter + 1))
print('Training done.')
elif mode == 'val' or mode == 'test':
def fast_hist(a, b, n):
"""
:param a: gt
:param b: pred
"""
k = (a >= 0) & (a < n)
return np.bincount(n * a[k].astype(int) + b[k], minlength=n ** 2).reshape(n, n)
use_dcrf = kwargs['use_dcrf']
eval_base_dir = os.path.join(FLAGS.outputs_base_dir, 'eval_results')
os.makedirs(eval_base_dir, exist_ok=True)
nImgs = FLAGS.nTestImgs if mode == 'test' else FLAGS.nValImgs
colorMap = scipy.io.loadmat(os.path.join(FLAGS.data_base_dir, 'colorMapC46.mat'))['colorMap']
outstr = mode + ' mode\n'
cat_max_len = 16
hist = np.zeros((FLAGS.nSketchClasses, FLAGS.nSketchClasses))
for imgIndex in range(1, nImgs + 1):
## load images
image_name = 'L0_sample' + str(imgIndex) + '.png' # e.g. L0_sample5564.png
image_path = os.path.join(FLAGS.data_base_dir, mode, 'DRAWING_GT', image_name)
test_image = load_image(image_path, mu) # shape = [1, H, W, 3]
## load gt_label
label_name = 'sample_' + str(imgIndex) + '_class.mat' # e.g. sample_1_class.mat
label_path = os.path.join(FLAGS.data_base_dir, mode, 'CLASS_GT', label_name)
gt_label = load_label(label_path) # shape = [1, H, W]
print('#' + str(imgIndex) + '/' + str(nImgs) + ': ' + image_path)
feed_dict = {model.images: test_image, model.labels: 0}
pred, pred_label_no_crf = sess.run([model.pred, model.pred_label], feed_dict=feed_dict)
if FLAGS.ignore_class_bg:
pred_label_no_crf = pred_label_no_crf + 1 # [1, 46]
# print('@ pred.shape ', pred.shape) # (1, H, W, nSketchClasses)
# print(pred_label_no_crf.shape) # shape = [1, H, W, 1]
if use_dcrf:
prob_arr = np.squeeze(pred)
prob_arr = prob_arr.transpose((2, 0, 1)) # shape = (nSketchClasses, H, W)
d_image = np.array(np.squeeze(test_image), dtype=np.uint8) # shape = (H, W, 3)
pred_label_crf = seg_densecrf(prob_arr, d_image, nSketchClasses) # shape=[H, W]
if FLAGS.ignore_class_bg:
pred_label_crf = pred_label_crf + 1 # [1, 46]
hist += fast_hist(np.squeeze(gt_label).flatten(),
pred_label_crf.flatten(),
FLAGS.nSketchClasses)
else:
hist += fast_hist(np.squeeze(gt_label).flatten(),
np.squeeze(pred_label_no_crf).flatten(),
FLAGS.nSketchClasses)
if imgIndex == nImgs:
## ignore bg pixel with value 0
if FLAGS.ignore_class_bg:
hist = hist[1:, 1:]
if use_dcrf:
print('\nRound', str(imgIndex), ', Use CRF')
outstr += '\nRound: ' + str(imgIndex) + ', Use CRF' + '\n'
else:
print('\nRound', str(imgIndex), ', Not Use CRF')
outstr += '\nRound: ' + str(imgIndex) + ', Not Use CRF' + '\n'
# overall accuracy
acc = np.diag(hist).sum() / hist.sum()
print('>>> overall accuracy', acc)
outstr += '>>> overall accuracy ' + str(acc) + '\n'
# mAcc
acc = np.diag(hist) / hist.sum(1)
mean_acc = np.nanmean(acc)
print('>>> mean accuracy', mean_acc)
outstr += '>>> mean accuracy ' + str(mean_acc) + '\n'
# mIoU
iou = np.diag(hist) / (hist.sum(1) + hist.sum(0) - np.diag(hist))
mean_iou = np.nanmean(iou)
print('>>> mean IoU', mean_iou)
outstr += '>>> mean IoU ' + str(mean_iou) + '\n'
# FWIoU
freq = hist.sum(1) / hist.sum()
fw_iou = (freq[freq > 0] * iou[freq > 0]).sum()
print('>>> freq weighted IoU', fw_iou)
print('\n')
outstr += '>>> freq weighted IoU ' + str(fw_iou) + '\n'
# IoU of each class
print('>>> IoU of each class')
outstr += '\n>>> IoU of each class' + '\n'
for classIdx in range(nSketchClasses):
if FLAGS.ignore_class_bg:
cat_name = colorMap[classIdx][0][0]
else:
if classIdx == 0:
cat_name = 'background'
else:
cat_name = colorMap[classIdx - 1][0][0]
singlestr = ' >>> '
cat_len = len(cat_name)
pad = ''
for ipad in range(cat_max_len - cat_len):
pad += ' '
singlestr += cat_name + pad + str(iou[classIdx])
print(singlestr)
outstr += singlestr + '\n'
# write validation result to txt
write_path = os.path.join(eval_base_dir, mode + '_results.txt')
fp = open(write_path, 'a')
fp.write(outstr)
fp.close()
else: # 'inference'
inference_ids = [kwargs['inference_id']]
inference_dataset = kwargs['inference_dataset']
black_bg = kwargs['black_bg']
use_dcrf = kwargs['use_dcrf']
colorMap = scipy.io.loadmat(os.path.join(FLAGS.data_base_dir, 'colorMapC46.mat'))['colorMap']
infer_result_base_dir = os.path.join(FLAGS.outputs_base_dir, 'inference_results', inference_dataset)
os.makedirs(infer_result_base_dir, exist_ok=True)
for img_count, img_id in enumerate(inference_ids):
image_name = 'L0_sample' + str(img_id) + '.png' # e.g. L0_sample5564.png
image_path = os.path.join(FLAGS.data_base_dir, inference_dataset, 'DRAWING_GT', image_name)
infer_image, infer_image_raw = load_image(image_path, mu, return_raw=True) # shape = [1, H, W, 3] / [H, W, 3]
print('\n#' + str(img_count + 1) + '/' + str(len(inference_ids)) + ': ' + image_name)
feed_dict = {model.images: infer_image, model.labels: 0}
pred, pred_label_no_crf, feat_visual \
= sess.run([model.pred, model.pred_label, model.feat_visual], feed_dict=feed_dict)
print('@ pred.shape ', pred.shape) # (1, H, W, nSketchClasses)
print('@ pred_label_no_crf.shape ', pred_label_no_crf.shape) # shape = [1, H, W, 1], contains [0, nClasses)
# print('@ feat_visual.shape ', feat_visual.shape) # shape = (1, 94, 94, 512)
if use_dcrf:
prob_arr = np.squeeze(pred)
prob_arr = prob_arr.transpose((2, 0, 1)) # shape = (nSketchClasses, H, W)
d_image = np.array(np.squeeze(infer_image), dtype=np.uint8) # shape = (H, W, 3)
pred_label_crf = seg_densecrf(prob_arr, d_image, nSketchClasses) # shape=[H, W], contains [0-46/47]
save_base_dir_crf = os.path.join(infer_result_base_dir, 'deeplab_output_crf')
os.makedirs(save_base_dir_crf, exist_ok=True)
if FLAGS.ignore_class_bg:
pred_label_crf += 1
pred_label_crf[infer_image_raw[:, :, 0] != 0] = 0 # [H, W]
save_path_crf = os.path.join(save_base_dir_crf, 'sem_result_' + str(img_id) + '.png')
visualize_semantic_segmentation(pred_label_crf, colorMap, black_bg=black_bg, save_path=save_path_crf)
else:
save_base_dir_no_crf = os.path.join(infer_result_base_dir, 'deeplab_output_no_crf')
os.makedirs(save_base_dir_no_crf, exist_ok=True)
if FLAGS.ignore_class_bg:
pred_label_no_crf += 1
pred_label_no_crf = np.squeeze(pred_label_no_crf)
pred_label_no_crf[infer_image_raw[:, :, 0] != 0] = 0 # [H, W]
save_path_no_crf = os.path.join(save_base_dir_no_crf, 'sem_result_' + str(img_id) + '.png')
visualize_semantic_segmentation(pred_label_no_crf, colorMap, black_bg=black_bg, save_path=save_path_no_crf)
def extract_samples_for_inspection(pairs,
inter_dir,
image_output_dir,
seed=123):
if not seed is None:
np.random.seed(seed)
if not os.path.exists(image_output_dir):
os.makedirs(image_output_dir, exist_ok=True)
raw_id_to_f_mapping = {get_identifier(p[0]): p for p in pairs}
fs = os.listdir(inter_dir)
# Check existence of identifier file
assert 'names.pkl' in fs
names = pickle.load(open(os.path.join(inter_dir, 'names.pkl'), 'rb'))
for i, n in names.items():
assert get_identifier(n) in raw_id_to_f_mapping
# Check phase contrast files
phase_contrast_files = [
f for f in fs if f.startswith('X_') and f.endswith('.pkl')
]
for i in range(len(phase_contrast_files)):
assert 'X_%d.pkl' % i in fs
# Sample phase contrast image
os.makedirs(os.path.join(image_output_dir, "phase_contrast"),
exist_ok=True)
random_inds = np.random.choice(list(names.keys()), (50, ), replace=False)
for ind in random_inds:
file_ind = ind // 100
identifier = get_identifier(names[ind])
try:
processed_img = pickle.load(
open(os.path.join(inter_dir, 'X_%d.pkl' % file_ind),
'rb'))[ind]
raw_img = load_image(raw_id_to_f_mapping[identifier][0])
out_path = os.path.join(image_output_dir, "phase_contrast",
"%s.png" % '_'.join(identifier))
save_multi_panel_fig([raw_img, processed_img], out_path)
except Exception:
print("Error saving sample %s" % '_'.join(identifier))
# try:
# # Check discrete segmentation annotations
# assert "classify_discrete_labels.pkl" in fs
# for i in range(len(phase_contrast_files)):
# assert 'segment_discrete_y_%d.pkl' % i in fs
# assert 'segment_discrete_w_%d.pkl' % i in fs
# classify_discrete_labels = pickle.load(open(os.path.join(inter_dir, "classify_discrete_labels.pkl"), 'rb'))
# inds_by_class = {}
# for k in classify_discrete_labels:
# if classify_discrete_labels[k][0] is None or classify_discrete_labels[k][1] == 0:
# continue
# label = classify_discrete_labels[k][0]
# if not label in inds_by_class:
# inds_by_class[label] = []
# inds_by_class[label].append(k)
# # Sample discrete fl segmentation (by class)
# for cl in inds_by_class:
# os.makedirs(os.path.join(image_output_dir, "discrete_segmentation_class_%s" % str(cl)), exist_ok=True)
# if len(inds_by_class[cl]) > 20:
# random_inds = np.random.choice(list(inds_by_class[cl]), (20,), replace=False)
# else:
# random_inds = inds_by_class[cl]
# for ind in random_inds:
# file_ind = ind // 100
# identifier = get_identifier(names[ind])
# try:
# raw_pc = load_image(raw_id_to_f_mapping[identifier][0])
# raw_fl = load_image(raw_id_to_f_mapping[identifier][1])
# processed_pc = pickle.load(open(os.path.join(inter_dir, 'X_%d.pkl' % file_ind), 'rb'))[ind]
# processed_fl_y = pickle.load(open(os.path.join(inter_dir, 'segment_discrete_y_%d.pkl' % file_ind), 'rb'))[ind]
# processed_fl_w = pickle.load(open(os.path.join(inter_dir, 'segment_discrete_w_%d.pkl' % file_ind), 'rb'))[ind]
# out_path = os.path.join(image_output_dir,
# "discrete_segmentation_class_%s" % str(cl),
# "%s.png" % '_'.join(identifier))
# save_multi_panel_fig([raw_pc,
# processed_pc,
# raw_fl,
# None,
# processed_fl_y,
# processed_fl_w], out_path)
# except Exception:
# print("Error saving fl(discrete) sample %s" % '_'.join(identifier))
# except Exception:
# print("Issue locating discrete segmentation files")
try:
# Check continuous segmentation annotations
assert "classify_continuous_labels.pkl" in fs
for i in range(len(phase_contrast_files)):
assert 'segment_continuous_y_%d.pkl' % i in fs
assert 'segment_continuous_w_%d.pkl' % i in fs
classify_continuous_labels = pickle.load(
open(os.path.join(inter_dir, "classify_continuous_labels.pkl"),
'rb'))
inds_by_class = {}
for k in classify_continuous_labels:
if classify_continuous_labels[k][
0] is None or classify_continuous_labels[k][1] == 0:
continue
label = np.argmax(classify_continuous_labels[k][0])
if not label in inds_by_class:
inds_by_class[label] = []
inds_by_class[label].append(k)
# Sample continuous fl segmentation (by class)
for cl in inds_by_class:
os.makedirs(os.path.join(
image_output_dir,
"continuous_segmentation_class_%s" % str(cl)),
exist_ok=True)
if len(inds_by_class[cl]) > 20:
random_inds = np.random.choice(list(inds_by_class[cl]), (20, ),
replace=False)
else:
random_inds = inds_by_class[cl]
for ind in random_inds:
file_ind = ind // 100
identifier = get_identifier(names[ind])
try:
raw_pc = load_image(raw_id_to_f_mapping[identifier][0])
raw_fl = load_image(raw_id_to_f_mapping[identifier][1])
processed_pc = pickle.load(
open(os.path.join(inter_dir, 'X_%d.pkl' % file_ind),
'rb'))[ind]
processed_fl_y = pickle.load(
open(
os.path.join(
inter_dir,
'segment_continuous_y_%d.pkl' % file_ind),
'rb'))[ind]
processed_fl_y = (
processed_fl_y *
np.array([0., 1. / 3, 2. / 3, 1.]).reshape(
(1, 1, 4))).sum(2)
processed_fl_w = pickle.load(
open(
os.path.join(
inter_dir,
'segment_continuous_w_%d.pkl' % file_ind),
'rb'))[ind]
out_path = os.path.join(
image_output_dir,
"continuous_segmentation_class_%s" % str(cl),
"%s.png" % '_'.join(identifier))
save_multi_panel_fig([
raw_pc, processed_pc, raw_fl, None, processed_fl_y,
processed_fl_w
], out_path)
except Exception:
print("Error saving fl(continuous) sample %s" %
'_'.join(identifier))
except Exception:
print("Issue locating continuous segmentation files")
def run_MURA(
batch_size=8, # select a batch of samples to train a time
nb_epoch=12, # times of iteration
depth=22, # network depth
nb_dense_block=4, # number of dense blocks
nb_filter=16, # initial number of conv filter
growth_rate=12, # numbers of new filters added by each layer
dropout_rate=0.2, # dropout rate
learning_rate=0.001, # learning rate
weight_decay=1E-4, # wight decay
plot_architecture=False # plot network architecture
):
###################
# Data processing #
###################
im_size = 320 # resize images
path_train = '/home/yu/Documents/tensorflow/MURA/MURA-v1.1/train/XR_ELBOW' # the absolute path
path_valid = '/home/yu/Documents/tensorflow/MURA/MURA-v1.1/valid/XR_ELBOW'
X_train_path, Y_train = data_loader.load_path(root_path=path_train,
size=im_size)
X_valid_path, Y_valid = data_loader.load_path(root_path=path_valid,
size=im_size)
X_valid = data_loader.load_image(X_valid_path,
im_size) # import path for validation
Y_valid = np.asarray(Y_valid)
nb_classes = 1
img_dim = (im_size, im_size, 1) #tuple channel last
###################
# Construct model #
###################
# model is one instance of class 'Model'
model = densenet.DenseNet(nb_classes,
img_dim,
depth,
nb_dense_block,
growth_rate,
nb_filter,
dropout_rate=dropout_rate,
weight_decay=weight_decay)
# Model output
model.summary()
# Build optimizer
opt = Adam(lr=learning_rate, beta_1=0.9, beta_2=0.999, epsilon=1e-08)
model.compile(
loss='binary_crossentropy',
optimizer=opt, # optimizer used to update gradient
metrics=["accuracy"])
if plot_architecture:
from keras.utils import plot_model
plot_model(model,
to_file='./figures/densenet_archi.png',
show_shapes=True)
####################
# Network training #
####################
print("Start Training")
list_train_loss = []
list_valid_loss = []
list_learning_rate = []
best_record = [100, 0, 100, 100] # record the best result
start_time = datetime.datetime.now()
for e in range(nb_epoch):
if e == int(0.25 * nb_epoch): # update learning_rate
K.set_value(model.optimizer.lr, np.float32(learning_rate / 10.))
if e == int(0.5 * nb_epoch):
K.set_value(model.optimizer.lr, np.float32(learning_rate / 50.))
if e == int(0.75 * nb_epoch):
K.set_value(model.optimizer.lr, np.float32(learning_rate / 100.))
split_size = batch_size
num_splits = len(
X_train_path
) / split_size # Calculate how many batches of training images
arr_all = np.arange(len(X_train_path)).astype(
int) # Return evenly spaced values within a given interval
random.shuffle(
arr_all
) # reshuffle, so the order of each training would be different
# avoid local optimal solution
# with shuffle open, it would be SGD
arr_splits = np.array_split(
arr_all,
num_splits) # Divede the training images to num_splits batches
l_train_loss = []
batch_train_loss = []
start = datetime.datetime.now()
for i, batch_idx in enumerate(
arr_splits): # i: how many batches, batch_idx: each batch
X_batch_path, Y_batch = [], [
] # X_batch_path is the path of images, Y_batch is the label
for idx in batch_idx:
X_batch_path.append(X_train_path[idx])
Y_batch.append(Y_train[idx])
X_batch = data_loader.load_image(
Path=X_batch_path, size=im_size) # load data for training
Y_batch = np.asarray(
Y_batch
) # Transform the type of Y_batch as array, that is label
train_logloss, train_acc = model.train_on_batch(
X_batch, Y_batch) # train, return loss and accuracy
l_train_loss.append([train_logloss, train_acc])
batch_train_loss.append([train_logloss, train_acc])
if i % 100 == 0: # 100 batches
loss_1, acc_1 = np.mean(np.array(l_train_loss), 0)
loss_2, acc_2 = np.mean(np.array(batch_train_loss), 0)
batch_train_loss = []
print(
'[Epoch {}/{}] [Batch {}/{}] [Time: {}] [all_batchs--> train_epoch_logloss: {:.5f}, train_epoch_acc:{:.5f}] '
.format(e + 1, nb_epoch, i, len(arr_splits),
datetime.datetime.now() - start, loss_1, acc_1),
'[this_100_batchs-->train_batchs_logloss: {:.5f}, train_batchs_acc:{:.5f}]'
.format(loss_2, acc_2))
# validate
valid_logloss, valid_acc = model.evaluate(X_valid,
Y_valid,
verbose=0,
batch_size=64)
list_train_loss.append(np.mean(np.array(l_train_loss), 0).tolist())
list_valid_loss.append([valid_logloss, valid_acc])
list_learning_rate.append(float(K.get_value(model.optimizer.lr)))
# to convert numpy array to json serializable
print('[Epoch %s/%s] [Time: %s, Total_time: %s]' %
(e + 1, nb_epoch, datetime.datetime.now() - start,
datetime.datetime.now() - start_time),
end='')
print(
'[train_loss_and_acc:{:.5f} {:.5f}] [valid_loss_acc:{:.5f} {:.5f}]'
.format(list_train_loss[-1][0], list_train_loss[-1][1],
list_valid_loss[-1][0], list_valid_loss[-1][1]))
d_log = {}
d_log["batch_size"] = batch_size
d_log["nb_epoch"] = nb_epoch
d_log["optimizer"] = opt.get_config()
d_log["train_loss"] = list_train_loss
d_log["valid_loss"] = list_valid_loss
d_log["learning_rate"] = list_learning_rate
json_file = os.path.join('./log/experiment_log_MURA.json')
with open(json_file, 'w') as fp:
json.dump(d_log, fp, indent=4, sort_keys=True)
record = [
valid_logloss,
valid_acc,
abs(valid_logloss - list_train_loss[-1][0]),
abs(valid_acc - list_train_loss[-1][1]),
]
if ((record[0] <= best_record[0]) & (record[1] >= best_record[1])):
if e <= int(0.25 * nb_epoch) | (record[2] <= best_record[2]) & (
record[3] <= best_record[3]):
best_record = record
print('saving the best model:epoch', e + 1, best_record)
model.save('save_models/best_MURA_modle@epochs{}.h5'.format(e +
1))
model.save('save_models/MURA_modle@epochs{}.h5'.format(e + 1))
