def main(argv):
# check folder
if not (os.path.exists(FLAGS.outdir)):
os.makedirs(FLAGS.outdir)
# load data set
data_set = io.load_matrix_data(FLAGS.data_list, 'float32')
# shuffle
np.random.shuffle(data_set)
num_per_tfrecord = int(FLAGS.num_per_tfrecord)
num_of_total_image = data_set.shape[0]
if (num_of_total_image % num_per_tfrecord != 0):
num_of_recordfile = num_of_total_image // num_per_tfrecord + 1
else:
num_of_recordfile = num_of_total_image // num_per_tfrecord
num_per_tfrecord_final = num_of_total_image - num_per_tfrecord * (num_of_recordfile - 1)
print('number of total TFrecordfile: {}'.format(num_of_recordfile))
# write TFrecord
for i in range(num_of_recordfile):
tfrecord_filename = os.path.join(FLAGS.outdir, 'recordfile_{}'.format(i + 1))
# options = tf.python_io.TFRecordOptions(tf.python_io.TFRecordCompressionType.GZIP) # compress
write = tf.python_io.TFRecordWriter(tfrecord_filename)
print('Writing recordfile_{}'.format(i+1))
if i == num_of_recordfile - 1:
loop_buf = num_per_tfrecord_final
else :
loop_buf = num_per_tfrecord
for image_index in range(loop_buf):
image = data_set[image_index + i*num_per_tfrecord].flatten()
example = tf.train.Example(
features=tf.train.Features(feature={
'img_raw': tf.train.Feature(float_list=tf.train.FloatList(value=image)),
'label':tf.train.Feature(int64_list=tf.train.Int64List(value=[0])),
'shape':tf.train.Feature(int64_list=tf.train.Int64List(value=FLAGS.image_size))
}))
write.write(example.SerializeToString())
write.close()
def plot_latent_space():
parser = argparse.ArgumentParser(
description='py, test_data_list, name_list, outdir')
parser.add_argument('--model', '-i1', default='', help='model')
parser.add_argument('--train_data_list',
'-i2',
default='',
help='name list')
parser.add_argument('--outdir', '-i3', default='', help='outdir')
args = parser.parse_args()
if not (os.path.exists(args.outdir)):
os.makedirs(args.outdir)
latent_dim = 64
# load test data
train_data = io.load_matrix_data(args.train_data_list, 'float32')
train_data = np.expand_dims(train_data, axis=-1)
print(train_data.shape)
image_size = []
image_size.extend([
list(train_data.shape)[1],
list(train_data.shape)[2],
list(train_data.shape)[3]
])
# set network
# network = Variational_auto_encoder(latent_dim, *image_size)
# model = network.get_vae()
# model.load_weights(args.model)
#
# mean_model = Model(inputs=model.input, outputs=model.get_layer('lambda_1').output)
#
# latent_space = mean_model.predict(train_data, batch_size=1)
encoder = load_model(args.model)
latent_space = encoder.predict(train_data, batch_size=1)
plt.figure(figsize=(8, 6))
fig = plt.scatter(latent_space[:, 0], latent_space[:, 1])
plt.title('latent distribution')
plt.xlabel('dim_1')
plt.ylabel('dim_2')
filename = open(os.path.join(args.outdir, 'latent_distribution.pickle'),
'wb')
pickle.dump(fig, filename)
plt.show()
def main():
parser = argparse.ArgumentParser(
description='py, test_data_txt, ground_truth_txt, outdir')
parser.add_argument('--ground_truth_txt', '-i1', default='')
parser.add_argument('--model', '-i2', default='./model_{}'.format(50000))
parser.add_argument('--outdir', '-i3', default='')
args = parser.parse_args()
# check folder
if not (os.path.exists(args.outdir)):
os.makedirs(args.outdir)
# tf flag
flags = tf.flags
flags.DEFINE_float("beta", 0.1, "hyperparameter beta")
flags.DEFINE_integer("num_of_generate", 1000, "number of generate data")
flags.DEFINE_integer("batch_size", 1, "batch size")
flags.DEFINE_integer("latent_dim", 2, "latent dim")
flags.DEFINE_list("image_size", [512, 512, 1], "image size")
FLAGS = flags.FLAGS
# load ground truth
ground_truth = io.load_matrix_data(args.ground_truth_txt, 'int32')
print(ground_truth.shape)
# initializer
init_op = tf.group(tf.initializers.global_variables(),
tf.initializers.local_variables())
with tf.Session(config=utils.config) as sess:
# set network
kwargs = {
'sess': sess,
'outdir': args.outdir,
'beta': FLAGS.beta,
'latent_dim': FLAGS.latent_dim,
'batch_size': FLAGS.batch_size,
'image_size': FLAGS.image_size,
'encoder': cnn_encoder,
'decoder': cnn_decoder
}
VAE = Variational_Autoencoder(**kwargs)
sess.run(init_op)
# testing
VAE.restore_model(args.model)
tbar = tqdm(range(FLAGS.num_of_generate), ascii=True)
specificity = []
for i in tbar:
sample_z = np.random.normal(0, 1.0, (1, FLAGS.latent_dim))
generate_data = VAE.generate_sample(sample_z)
generate_data = generate_data[0, :, :, 0]
# EUDT
eudt_image = sitk.GetImageFromArray(generate_data)
eudt_image.SetSpacing([1, 1])
eudt_image.SetOrigin([0, 0])
# label
label = np.where(generate_data > 0, 0, 1)
label_image = sitk.GetImageFromArray(label)
label_image.SetSpacing([1, 1])
label_image.SetOrigin([0, 0])
# calculate ji
case_max_ji = 0.
for image_index in range(ground_truth.shape[0]):
ji = utils.jaccard(label, ground_truth[image_index])
if ji > case_max_ji:
case_max_ji = ji
specificity.append([case_max_ji])
# output image
io.write_mhd_and_raw(
eudt_image,
'{}.mhd'.format(os.path.join(args.outdir, 'EUDT', str(i + 1))))
io.write_mhd_and_raw(
label_image,
'{}.mhd'.format(os.path.join(args.outdir, 'label',
str(i + 1))))
print('specificity = %f' % np.mean(specificity))
# output csv file
with open(os.path.join(args.outdir, 'specificity.csv'), 'w',
newline='') as file:
writer = csv.writer(file)
writer.writerows(specificity)
writer.writerow(['specificity:', np.mean(specificity)])
def main():
parser = argparse.ArgumentParser(description='py, train_data_txt, train_data_ture_txt, validation_data_txt, outdir')
parser.add_argument('--train_data_txt', '-i1', default='',
help='train data list')
parser.add_argument('--validation_data_txt', '-i2', default='',
help='validation data list')
parser.add_argument('--outdir', '-i3', default='', help='outdir')
args = parser.parse_args()
# check folder
if not (os.path.exists(args.outdir + '/encoder_model')):
os.makedirs(args.outdir + '/encoder_model')
if not (os.path.exists(args.outdir + '/decoder_model')):
os.makedirs(args.outdir + '/decoder_model')
# define
batch_size = 3
epoch = 200
latent_dim = 2
# load train data
train_data = io.load_matrix_data(args.train_data_txt, 'float32')
train_data = np.expand_dims(train_data, axis=4)
# load validation data
val_data = io.load_matrix_data(args.validation_data_txt, 'float32')
val_data = np.expand_dims(val_data, axis=4)
print(' number of training: {}'.format(len(train_data)))
print('size of traning: {}'.format(train_data.shape))
print(' number of validation: {}'.format(len(val_data)))
print('size of validation: {}'.format(val_data.shape))
image_size = []
image_size.extend([list(train_data.shape)[1], list(train_data.shape)[2], list(train_data.shape)[3]])
# # set network
network = Variational_auto_encoder(latent_dim, *image_size)
model = network.get_vae()
encoder = network.get_encoder()
decoder = network.get_decoder()
model.summary()
model.compile(optimizer=Adam(lr=8e-4, beta_1=0.5, beta_2=0.9), loss=[zero_loss])
# set data_set
train_steps, train_data = batch_iter(train_data, train_data, batch_size)
valid_steps, val_data = batch_iter(val_data, val_data, batch_size)
# fit network
history_total = []
for epoch_index in range(epoch):
history = model.fit_generator(train_data, steps_per_epoch=train_steps, epochs=1, validation_data=val_data,
validation_steps=valid_steps, verbose=1)
history_total.append(history)
encoder.save(os.path.join(args.outdir + '/encoder_model/', 'encoder_{}.hdf5'.format(epoch_index + 1)))
decoder.save(os.path.join(args.outdir + '/decoder_model/', 'decoder_{}.hdf5'.format(epoch_index + 1)))
plot_history(history_total, args.outdir)
def predict_spe():
parser = argparse.ArgumentParser(
description='py, test_data_list, name_list, outdir')
parser.add_argument('--model', '-i1', default='', help='model')
parser.add_argument('--truth_data_txt',
'-i2',
default='',
help='name list')
parser.add_argument('--outdir', '-i3', default='', help='outdir')
args = parser.parse_args()
if not (os.path.exists(args.outdir)):
os.mkdir(args.outdir)
latent_dim = 32
n_gen = 1
# load ground truth
ground_truth = io.load_matrix_data(args.truth_data_txt, 'int32')
print(ground_truth.shape)
# aset network
decoder = load_model(args.model)
specificity = []
for i in range(n_gen):
# # generate shape
# sample_z = np.full(latent_dim, 0)
# sample_z = np.array([sample_z])
sample_z = np.random.normal(0, 1.0, (1, latent_dim))
# print(sample_z.shape)
preds = decoder.predict(sample_z)
preds = preds[:, :, :, :, 0]
# # EUDT
eudt_image = sitk.GetImageFromArray(preds[0])
eudt_image.SetSpacing([1, 1, 1])
eudt_image.SetOrigin([0, 0, 0])
# label
label = np.where(preds[0] > 0, 0, 1)
label_image = sitk.GetImageFromArray(label)
label_image.SetSpacing([1, 1, 1])
label_image.SetOrigin([0, 0, 0])
# calculate ji
case_max_ji = 0.
for image_index in range(ground_truth.shape[0]):
ji = jaccard(label, ground_truth[image_index])
if ji > case_max_ji:
case_max_ji = ji
specificity.append([case_max_ji])
# output image
io.write_mhd_and_raw(
eudt_image,
'{}.mhd'.format(os.path.join(args.outdir, 'EUDT', str(i + 1))))
io.write_mhd_and_raw(
label_image,
'{}.mhd'.format(os.path.join(args.outdir, 'label', str(i + 1))))
# output csv file
# with open(os.path.join(args.outdir, 'specificity.csv'), 'w', newline='') as file:
# writer = csv.writer(file)
# writer.writerows(specificity)
# writer.writerow(['specificity:', np.mean(specificity)])
print('specificity = %f' % np.mean(specificity))
def main():
parser = argparse.ArgumentParser(
description=
'py, train_data_txt, train_data_ture_txt, validation_data_txt, outdir')
parser.add_argument('--train_data_txt',
'-i1',
default='',
help='train data list')
parser.add_argument('--train_ground_truth_txt',
'-i2',
default='',
help='train ground truth list')
parser.add_argument('--validation_data_txt',
'-i3',
default='',
help='validation data list')
parser.add_argument('--validation_ground_truth_txt',
'-i4',
default='',
help='validation ground truth list')
parser.add_argument('--outdir', '-i5', default='', help='outdir')
args = parser.parse_args()
# check folder
if not (os.path.exists(args.outdir)):
os.mkdir(args.outdir)
# define
batch_size = 3
epoch = 2500
# load train data
train_data = io.load_matrix_data(args.train_data_txt, 'float32')
train_data = np.expand_dims(train_data, axis=4)
# load train ground truth
train_truth = io.load_matrix_data(args.train_ground_truth_txt, 'float32')
train_truth = np.expand_dims(train_truth, axis=4)
# load validation data
val_data = io.load_matrix_data(args.validation_data_txt, 'float32')
val_data = np.expand_dims(val_data, axis=4)
# load validation ground truth
val_truth = io.load_matrix_data(args.validation_ground_truth_txt,
'float32')
val_truth = np.expand_dims(val_truth, axis=4)
print(' number of training: {}'.format(len(train_data)))
print('size of traning: {}'.format(train_data.shape))
print(' number of validation: {}'.format(len(val_data)))
print('size of validation: {}'.format(val_data.shape))
image_size = []
image_size.extend([
list(train_data.shape)[1],
list(train_data.shape)[2],
list(train_data.shape)[3]
])
# set network
network = Autoencoder(*image_size)
model = network.model()
model.summary()
model.compile(optimizer='Nadam',
loss=losses.mean_squared_error,
metrics=['mse'])
# set data_set
train_steps, train_data = batch_iter(train_data, train_truth, batch_size)
valid_steps, val_data = batch_iter(val_data, val_truth, batch_size)
# fit network
model_checkpoint = ModelCheckpoint(os.path.join(
args.outdir, 'weights.{epoch:02d}-{val_loss:.2f}.hdf5'),
verbose=1)
history = model.fit_generator(train_data,
steps_per_epoch=train_steps,
epochs=epoch,
validation_data=val_data,
validation_steps=valid_steps,
verbose=1,
callbacks=[model_checkpoint])
plot_history(history, args.outdir)
def predict_gen():
parser = argparse.ArgumentParser(
description='py, test_data_list, name_list, outdir')
parser.add_argument('--test_data_list',
'-i1',
default='',
help='test data')
parser.add_argument('--truth_data_txt',
'-i2',
default='',
help='name list')
parser.add_argument('--name_list', '-i3', default='', help='name list')
parser.add_argument('--encoder_model', '-i4', default='', help='model')
parser.add_argument('--decoder_model', '-i5', default='', help='model')
parser.add_argument('--outdir', '-i6', default='', help='outdir')
args = parser.parse_args()
if not (os.path.exists(args.outdir)):
os.makedirs(args.outdir)
latent_dim = 32
# load name_list
name_list = []
with open(args.name_list) as paths_file:
for line in paths_file:
line = line.split()
if not line: continue
name_list.append(line[:])
print('number of test data : {}'.format(len(name_list)))
# load test data
test_data = io.load_matrix_data(args.test_data_list, 'float32')
test_data = np.expand_dims(test_data, axis=4)
print(test_data.shape)
# load ground truth
ground_truth = io.load_matrix_data(args.truth_data_txt, 'int32')
print(ground_truth.shape)
# get image size
image_size = []
image_size.extend([
list(test_data.shape)[1],
list(test_data.shape)[2],
list(test_data.shape)[3]
])
print(image_size)
# # set network
# network = Variational_auto_encoder(latent_dim, *image_size)
# model = network.get_vae()
# model.load_weights(args.model)
# predict
# preds = model.predict(test_data, 1)
# network = Variational_auto_encoder(latent_dim, *image_size)
# encoder = network.get_encoder()
# encoder.load_weights(args.encoder_model)
encoder = load_model(args.encoder_model)
decoder = load_model(args.decoder_model)
encoder_result = encoder.predict(test_data, 1)
preds = decoder.predict(encoder_result, 1)
# reshape
preds = preds[:, :, :, :, 0]
print(preds.shape)
ji = []
for i in range(preds.shape[0]):
# EUDT
eudt_image = sitk.GetImageFromArray(preds[i])
eudt_image.SetSpacing([1, 1, 1])
eudt_image.SetOrigin([0, 0, 0])
# label
label = np.where(preds[i] > 0, 0, 1)
label_image = sitk.GetImageFromArray(label)
label_image.SetSpacing([1, 1, 1])
label_image.SetOrigin([0, 0, 0])
# calculate ji
ji.append([jaccard(label, ground_truth[i])])
# output image
io.write_mhd_and_raw(
eudt_image,
'{}.mhd'.format(os.path.join(args.outdir, 'EUDT', *name_list[i])))
io.write_mhd_and_raw(
label_image,
'{}.mhd'.format(os.path.join(args.outdir, 'label', *name_list[i])))
generalization = np.mean(ji)
print('generalization = %f' % generalization)
# output csv file
with open(os.path.join(args.outdir, 'generalization.csv'), 'w',
newline='') as file:
writer = csv.writer(file)
writer.writerows(ji)
writer.writerow(['generalization= ', generalization])
def main():
# tf flag
flags = tf.flags
flags.DEFINE_string("ground_truth_txt", 'F:/data_info/VAE_liver/set_5/PCA/alpha_0.1/fold_1/test_label.txt', "ground truth txt")
flags.DEFINE_string("indir", 'G:/experiment_result/liver/VAE/set_5/down/64/alpha_0.1/fold_1/VAE/axis_4/beta_6', "input dir")
flags.DEFINE_string("outdir", 'G:/experiment_result/liver/VAE/set_5/down/64/alpha_0.1/fold_1/VAE/axis_4/beta_6/random', "outdir")
flags.DEFINE_integer("model_index", 3450 ,"index of model")
flags.DEFINE_string("gpu_index", "0", "GPU-index")
flags.DEFINE_float("beta", 1.0, "hyperparameter beta")
flags.DEFINE_integer("num_of_generate", 1000, "number of generate data")
flags.DEFINE_integer("batch_size", 1, "batch size")
flags.DEFINE_integer("latent_dim", 4, "latent dim")
flags.DEFINE_list("image_size", [56, 72, 88, 1], "image size")
FLAGS = flags.FLAGS
np.random.seed(1)
# check folder
if not (os.path.exists(FLAGS.outdir)):
os.makedirs(FLAGS.outdir)
# load ground truth
ground_truth = io.load_matrix_data(FLAGS.ground_truth_txt, 'int32')
print(ground_truth.shape)
# initializer
init_op = tf.group(tf.initializers.global_variables(),
tf.initializers.local_variables())
with tf.Session(config = utils.config(index=FLAGS.gpu_index)) as sess:
# set network
kwargs = {
'sess': sess,
'outdir': FLAGS.outdir,
'beta': FLAGS.beta,
'latent_dim': FLAGS.latent_dim,
'batch_size': FLAGS.batch_size,
'image_size': FLAGS.image_size,
'encoder': encoder_resblock_bn,
'decoder': decoder_resblock_bn,
'downsampling': down_sampling,
'upsampling': up_sampling,
'is_training': False,
'is_down': False
}
VAE = Variational_Autoencoder(**kwargs)
sess.run(init_op)
# testing
VAE.restore_model(os.path.join(FLAGS.indir,'model','model_{}'.format(FLAGS.model_index)))
mean = np.loadtxt(os.path.join(FLAGS.indir, 'mean_{}.txt'.format(FLAGS.model_index)))
var = np.loadtxt(os.path.join(FLAGS.indir, 'var_{}.txt'.format(FLAGS.model_index)))
specificity = []
tbar = tqdm(range(FLAGS.num_of_generate), ascii=True)
for i in tbar:
sample_z = np.random.normal(0, 1.0, (1, FLAGS.latent_dim))
sample_z = np.asarray(mean) + np.sqrt(np.asarray(var)) * sample_z
generate_data = VAE.generate_sample(sample_z)
generate_data = generate_data[0, :, :, :, 0]
# EUDT
eudt_image = sitk.GetImageFromArray(generate_data)
eudt_image.SetSpacing([1, 1, 1])
eudt_image.SetOrigin([0, 0, 0])
# label
label = np.where(generate_data > 0.5, 0, 1)
label = label.astype(np.int8)
label_image = sitk.GetImageFromArray(label)
label_image.SetSpacing([1, 1, 1])
label_image.SetOrigin([0, 0, 0])
# # calculate ji
case_max_ji = 0.
for image_index in range(ground_truth.shape[0]):
ji = utils.jaccard(label, ground_truth[image_index])
if ji > case_max_ji:
case_max_ji = ji
specificity.append([case_max_ji])
# # output image
# io.write_mhd_and_raw(eudt_image, '{}.mhd'.format(os.path.join(FLAGS.outdir, 'EUDT', str(i+1))))
# io.write_mhd_and_raw(label_image, '{}.mhd'.format(os.path.join(FLAGS.outdir, 'label', str(i + 1))))
print('specificity = %f' % np.mean(specificity))
# # output csv file
with open(os.path.join(FLAGS.outdir, 'specificity_{}.csv'.format(FLAGS.model_index)), 'w', newline='') as file:
writer = csv.writer(file)
writer.writerows(specificity)
writer.writerow(['specificity:', np.mean(specificity)])
def main():
parser = argparse.ArgumentParser(
description='py, data_list, num_per_tfrecord, outdir')
parser.add_argument(
'--data_list',
'-i1',
default=
'F:/data_info/TFrecord/liver/set_5/down/64/alpha_0.1/fold_1/val.txt',
help='data list')
parser.add_argument('--num_per_tfrecord',
'-i2',
default=76,
help='number per tfrecord')
parser.add_argument(
'--outdir',
'-i3',
default='G:/data/tfrecord/liver/set_5/down/64/RBF/alpha_0.1/fold_1/val',
help='outdir')
args = parser.parse_args()
# check folder
if not (os.path.exists(args.outdir)):
os.makedirs(args.outdir)
data_set = io.load_matrix_data(args.data_list, 'float32')
# data_set = io.load_matrix_raw_data(args.data_list, 'float')
# shuffle
# np.random.shuffle(data_set)
print('data size: {}'.format(data_set.shape))
num_per_tfrecord = int(args.num_per_tfrecord)
num_of_total_image = data_set.shape[0]
if (num_of_total_image % num_per_tfrecord != 0):
num_of_recordfile = num_of_total_image // num_per_tfrecord + 1
else:
num_of_recordfile = num_of_total_image // num_per_tfrecord
num_per_tfrecord_final = num_of_total_image - num_per_tfrecord * (
num_of_recordfile - 1)
print('number of total TFrecordfile: {}'.format(num_of_recordfile))
# write TFrecord
for i in range(num_of_recordfile):
tfrecord_filename = os.path.join(args.outdir,
'recordfile_{}'.format(i + 1))
options = tf.python_io.TFRecordOptions(
tf.python_io.TFRecordCompressionType.GZIP)
write = tf.python_io.TFRecordWriter(tfrecord_filename, options=options)
print('Writing recordfile_{}'.format(i + 1))
if i == num_of_recordfile - 1:
loop_buf = num_per_tfrecord_final
else:
loop_buf = num_per_tfrecord
for image_index in range(loop_buf):
image = data_set[image_index + i * num_per_tfrecord].flatten()
example = tf.train.Example(features=tf.train.Features(
feature={
'img_raw':
tf.train.Feature(float_list=tf.train.FloatList(
value=image)),
}))
write.write(example.SerializeToString())
write.close()
def predict():
parser = argparse.ArgumentParser(
description='py, test_data_list, name_list, outdir')
parser.add_argument('--test_data_list',
'-i1',
default='',
help='test data')
parser.add_argument('--name_list', '-i2', default='', help='name list')
parser.add_argument('--model', '-i3', default='', help='model')
parser.add_argument('--outdir', '-i4', default='', help='outdir')
args = parser.parse_args()
if not (os.path.exists(args.outdir)):
os.mkdir(args.outdir)
# load name_list
name_list = []
with open(args.name_list) as paths_file:
for line in paths_file:
line = line.split()
if not line: continue
name_list.append(line[:])
print('number of test data : {}'.format(len(name_list)))
test_data = io.load_matrix_data(args.test_data_list, 'float32')
test_data = np.expand_dims(test_data, axis=4)
print(test_data.shape)
image_size = []
image_size.extend([
list(test_data.shape)[1],
list(test_data.shape)[2],
list(test_data.shape)[3]
])
print(image_size)
# set network
network = Autoencoder(*image_size)
model = network.model()
model.load_weights(args.model)
preds = model.predict(test_data, 1)
preds = preds[:, :, :, :, 0]
print(preds.shape)
for i in range(preds.shape[0]):
# EUDT
eudt_image = sitk.GetImageFromArray(preds[i])
eudt_image.SetSpacing([1, 1, 1])
eudt_image.SetOrigin([0, 0, 0])
# label
label = np.where(preds[i] > 0, 0, 1)
label_image = sitk.GetImageFromArray(label)
label_image.SetSpacing([1, 1, 1])
label_image.SetOrigin([0, 0, 0])
io.write_mhd_and_raw(
eudt_image,
'{}.mhd'.format(os.path.join(args.outdir, 'EUDT', *name_list[i])))
io.write_mhd_and_raw(
label_image,
'{}.mhd'.format(os.path.join(args.outdir, 'label', *name_list[i])))
