def _build(self):
if 'custom' in self.config and self.config['custom']:
return self.build_custom_model()
#if self.dataset.data_type == Dataset.TEXT_TYPE:
# lm = self.build_language_model()
# Right now there is an assumption that
if self.dataset.data_type == Dataset.IMAGE_TYPE and self.label.label_type == Label.BINARY:
return CNNModel(2, input_shape=(128, 128))
if self.dataset.data_type == Dataset.IMAGE_TYPE and self.label.label_type == Label.CLASSIFICATION:
return CNNModel(len(self.label.classes), input_shape=(128, 128))
word_list = pickle.load(open("vendor/keras_language_model/vocab.p", "rb"))
if self.dataset.data_type == Dataset.TEXT_TYPE and self.label.label_type == Label.BINARY:
cnn_model = CNNTextClassifier(2)
return cnn_model
if self.dataset.data_type == Dataset.TEXT_TYPE and self.label.label_type == Label.CLASSIFICATION:
cnn_model = CNNTextClassifier(len(self.label.classes))
return cnn_model
if self.dataset.data_type == Dataset.JSON_TYPE and self.label.label_type == Label.SEQUENCE:
from models.cnn_sequence_tagger import CNNSequenceTagger
return CNNSequenceTagger(self.label.score_classes)
if self.dataset.data_type == Dataset.OBJECT_DETECTION_TYPE and self.label.label_type == Label.OBJECT_DETECTION:
from models.lightnet_model import LightnetModel
return LightnetModel()
return StubModel()
def main():
filename = '../data/train/image_2473853.jpg'
inp = imread(filename).astype('float32')
plt.imshow(inp, cmap='gray')
plt.show()
print(inp)
print(len(inp.reshape(-1, inp.shape[0], inp.shape[1], 1)))
cnnnet = CNNModel()
conv_layer1, conv_layer2, conv_layer3, network = cnnnet.define_network(
inp.reshape(-1, inp.shape[0], inp.shape[1], 1), 'visual')
model = tflearn.DNN(network,
tensorboard_verbose=0,
checkpoint_path='nodule-classifier.tfl.ckpt')
model.load('nodule-classifier.tfl')
labels = model.predict(inp.reshape(-1, 50, 50, 1))
print("This is the labels :", labels)
if labels[0][1] == 1:
print('The input image is a nodule.')
else:
print('The input image is not a nodule.')
layer_to_be_plotted = [conv_layer1, conv_layer2, conv_layer3]
for idx, layer in enumerate(layer_to_be_plotted):
m2, yhat = get_layer_output(layer, model, inp)
plot_layers(yhat, idx, '../visual_images/conv_layer_')
weights = get_weights(m2, conv_layer1)
plot_layers(weights, 0, '../visual_images/weight_conv_layer_', 6)
plt.show()
def build(self):
if 'custom' in self.config and self.config['custom']:
return self.build_custom_model()
#if self.dataset.data_type == Dataset.TEXT_TYPE:
# lm = self.build_language_model()
# Right now there is an assumption that
if self.dataset.data_type == Dataset.IMAGE_TYPE and self.label.label_type == Label.BINARY:
return CNNModel(2, input_shape=(128, 128))
if self.dataset.data_type == Dataset.IMAGE_TYPE and self.label.label_type == Label.CLASSIFICATION:
return CNNModel(len(self.label.classes), input_shape=(128, 128))
word_list = pickle.load(open("vendor/keras_language_model/vocab.p", "rb"))
if self.dataset.data_type == Dataset.TEXT_TYPE and self.label.label_type == Label.BINARY:
cnn_model = CNNTextClassifier(2)
return cnn_model
#rnn_model = RNNModel(2, word_list)
#rnn_model.load_lm('vendor/keras_language_model')
#return rnn_model
if self.dataset.data_type == Dataset.TEXT_TYPE and self.label.label_type == Label.CLASSIFICATION:
cnn_model = CNNTextClassifier(len(self.label.classes))
return cnn_model
#rnn_model = RNNModel(len(self.label.classes), word_list)
#rnn_model.load_lm('vendor/keras_language_model')
#return rnn_model
#return LM_TextClassifier(lm, len(self.label.classes))
if self.dataset.data_type == Dataset.TEXT_TYPE and self.label.label_type == Label.SEQUENCE:
return SequenceModel(
valid_outputs=self.label.valid_tokens,
seq2seq=False,
character_mode=False,
)
if self.dataset.data_type == Dataset.OBJECT_DETECTION_TYPE and self.label.label_type == Label.OBJECT_DETECTION:
from models.lightnet_model import LightnetModel
return LightnetModel()
return StubModel()
def build_network(args):
hparams = dotdict({
"dim_output": args.num_classes,
"inner_update_lr": args.inner_update_lr,
"meta_lr": args.meta_lr,
"meta_test_num_inner_updates": args.meta_test_num_inner_updates,
"dim_hidden": args.dim_hidden,
"img_size": 28,
"channels": 1,
})
model = CNNModel(hparams)
return model
def get_predictions(x_test_images, y_test_labels):
"""
Given the test images, cnn network predicted the labels for images.
:param x_test_images: test images
:param y_test_labels: response to labels for the images
:return:
predictions: the predicted probability values of test labels for images
label_predictions: the specific class for each image
"""
cnn_net = CNNModel()
network = cnn_net.define_network(x_test_images)
model = tflearn.DNN(network,
tensorboard_verbose=0,
checkpoint_path='nodule-classifier.tfl.ckpt')
model.load('nodule-classifier.tfl')
predictions = np.vstack(model.predict(x_test_images[:, :, :, :]))
score = model.evaluate(x_test_images, y_test_labels)
print('The total classification accuracy is : ', score)
label_predictions = np.zeros_like(predictions)
label_predictions[np.arange(len(predictions)), predictions.argmax(1)] = 1
return predictions, label_predictions
def train_and_score(self, genome):
"""
Train and score a single individual.
Score (fitness) is equal to a model's accuracy on test data predictions.
"""
trainer = Trainer(CNNModel.buildForEvolution(genome), self.config,
self.data)
trainer.train() # train individual using training data
score = trainer.model.evaluate(
self.data['testX'], self.data['testY'],
verbose=0) # score individual using test data
logging.info("Score : " + str(score[1])) # 1=accuracy, 0=loss.
genome.fitness = score[1] # set the individual's fitness variable
return score
def build_model_and_trainer(
config: DotMap,
data_loader: BaseDataLoader) -> Tuple[Model, BaseTrainer]:
model_builder = CNNModel(config)
model, parallel_model = WithLoadWeights(model_builder, model_name='cnn') \
.build_model(model_name='cnn')
trainer = CNNTrainer(model=model,
parallel_model=parallel_model,
data_loader=data_loader,
config=config)
return model, trainer
def main(argv=None):
if argv is None:
argv = sys.argv[1:]
parser = argparse.ArgumentParser()
parser.add_argument('--training-data',
dest='training_data',
type=str,
required=True,
help='The path to the training data')
parser.add_argument('--model',
dest='model',
type=str,
default="",
choices=['cnn', 'lstm', 'gru'],
help='The model to train')
parser.add_argument('--num-epochs',
dest='num_epochs',
type=int,
default=50,
help='The number of epochs to train for')
parser.add_argument('--batch-size',
dest='batch_size',
type=int,
default=1,
help='The batch size to train with')
parser.add_argument('--lr',
dest='lr',
type=float,
default=0.0025,
help='The learning rate.')
parser.add_argument('--char2idx',
dest='char2idx',
type=str,
default=None,
help='char2idx')
parser.add_argument('--idx2char',
dest='idx2char',
type=str,
default=None,
help='idx2char')
parser.add_argument('--seq-length',
dest='seq_length',
type=int,
default=2048,
help='The maximum sequence length in characters.')
parser.add_argument('--output-path',
dest='output_path',
type=str,
help='The path everything to save.')
args = parser.parse_args()
# Sanity check for the output-path
if not os.path.exists(args.output_path):
os.makedirs(args.output_path)
# Read and process the data
# If args.char2idx or args.idx2char are None
if args.char2idx is None or args.idx2char is None:
# Generate those files
idx2char, char2idx = lazy_scan_vocabulary(args.training_data)
np.save(os.path.join(args.output_path, "char2idx.npy"), char2idx)
np.save(os.path.join(args.output_path, "idx2char.npy"), idx2char)
else:
pass
vocab_size = len(idx2char) + 1
print("Vocab_Size: {}".format(vocab_size))
num_threads = get_threads()
TOTAL_LABELS = 0
LABEL_COUNTS = [0., 0.]
torch.set_num_threads(num_threads)
if args.model == 'cnn':
from models.cnn_model import CNNModel
model = CNNModel(32, vocab_size, 2, 0)
elif args.model == 'lstm':
from models.rnn_model import LSTMModel
model = LSTMModel(32, 256, vocab_size, 2)
elif args.model == 'gru':
from models.rnn_model import GRUModel
model = GRUModel(32, 256, vocab_size, 2)
else:
LOGGER.error("Unknown model '" + args.model + "'")
# Break
exit(1)
def count_parameters(model):
return sum(p.numel() for p in model.parameters() if p.requires_grad)
print("# Parameters: {}".format(count_parameters(model)))
# Start training
train(model, args, char2idx, idx2char, False)
# Save trained model
torch.save(model, args.output_path + "/model")
if config['evolvingMode']:
resultsDir = ''
if config['intelligentSurvival']:
resultsDir = 'results/evolution-intelligent-survival/'
else:
resultsDir = 'results/evolution-plain/'
evolution = Evolution(config, data)
evolution.initialise_population()
for gen in range(numGenerations):
evolution.evolvePopulation(gen)
best_evolved_genome = evolution.hall_of_fame.getSolution(0)
trainer = Trainer(CNNModel.buildForEvolution(best_evolved_genome), config,
data)
generate_model_summary(trainer.model)
training_history = trainer.train()
generate_classification_report(trainer.model, data)
generate_training_stats_plot(config, training_history, resultsDir)
save_model(trainer.model, resultsDir)
score = trainer.model.evaluate(data['trainX'], data['trainY'], verbose=0)
print("Score: " + str(score[1]))
else:
resultsDir = 'results/no-evolution/'
trainer = Trainer(
CNNModel.buildNoEvolution(), config,
data) # initialise trainer with a newly initialised model
generate_model_summary(trainer.model) # summary of the model structure
training_history = trainer.train(
RB_FACTOR = 10
replay = BSZ * RB_FACTOR
if __name__ == '__main__':
batch_gen = Batchifier(data_path=DATA_PATH,
bsz=1,
bptt=BPTT,
idx=IDX,
asset_list=ASSETS,
randomize_train=randomize_train,
overlapping_train=overlapping_train)
model = CNNModel(num_hid=NUM_HID,
bptt=BPTT,
num_assets=len(asset_list),
lr=LR,
clip_norm=5.0)
buffer = ReplayBuffer(buffer_size=replay)
with tf.Session() as sess:
sess.run(model.tf_init())
losses = []
for epoch in range(1, NUM_EPOCHS + 1):
batch_losses = 0.0
for bTrainX, bTrainY in batch_gen.load_train():
if buffer.size < buffer.max_size:
buffer.add(state=bTrainX, action=bTrainY)
continue
else:
buffer.add(state=bTrainX, action=bTrainY)
# Data generation for the training, validation and the test.
data = DataProcessor("train")
data.init_random_batches(model_params["batch_size"])
val_data = DataProcessor("val")
test_data = DataProcessor("test")
# Placeholders for the model input and output.
x = tf.placeholder("float",
(None, data_params["input_size"],
data_params["input_size"], data_params["input_dims"]))
y = tf.placeholder("float", (None, 1))
# Model class and weight initialization. Also, the step initialization for the epochs.
model = CNNModel(model_params=model_params,
data_params=data_params,
data=x,
target=y)
global_step = tf.Variable(0, dtype=tf.int32, trainable=False)
init = tf.global_variables_initializer()
# Initiation of the main Session.
with tf.Session() as sess:
# Summary writer folder.
file_writer = tf.summary.FileWriter('./outs', sess.graph)
sess.run(init)
if not data.has_next():
print("All data finished")
data.init_random_batches(model_params["batch_size"])
# Enable updating through batch normalization and dropout layers.
extra_update_ops = tf.get_collection(tf.GraphKeys.UPDATE_OPS)
logger.info("Making dataset...")
data_loader_char_config = ImageSetDataLoaderConfig(
image_dict=preprocessor_char.get_image_dict(),
image_shape=INPUT_SHAPE_REVERSE,
shuffle_buffer_size=100000,
batch_size=100)
data_loader_province_config = ImageSetDataLoaderConfig(
image_dict=preprocessor_province.get_image_dict(),
image_shape=INPUT_SHAPE_REVERSE,
shuffle_buffer_size=100000,
batch_size=100)
data_loader_char = ImageSetDataLoader(data_loader_char_config)
data_loader_province = ImageSetDataLoader(data_loader_province_config)
model_char = CNNModel(model_char_config)
model_province = CNNModel(model_province_config)
trainer_char = UniversalTrainer(model_char.get_model(),
data_loader_char.get_dataset(),
trainer_config)
trainer_province = UniversalTrainer(model_province.get_model(),
data_loader_province.get_dataset(),
trainer_config)
logger.info("Training...")
trainer_char.train()
trainer_province.train()
trainer_char.save("log/test_split_char.h5")
trainer_province.save("log/test_split_province.h5")
else:
Trains a CNN model using tflearn wrapper for tensorflow
"""
import tflearn
import h5py
from models.cnn_model import CNNModel
# Load HDF5 dataset
h5f = h5py.File('../data/train.h5', 'r')
X_train_images = h5f['X']
Y_train_labels = h5f['Y']
h5f2 = h5py.File('../data/val.h5', 'r')
X_val_images = h5f2['X']
Y_val_labels = h5f2['Y']
## Model definition
convnet = CNNModel()
network = convnet.define_network(X_train_images)
model = tflearn.DNN(network, tensorboard_verbose=0, checkpoint_path='nodule-classifier.tfl.ckpt')
model.fit(X_train_images, Y_train_labels, n_epoch=50, shuffle=True, validation_set=(X_val_images, Y_val_labels),
show_metric=True, batch_size=128, snapshot_epoch=False, run_id='nodule-classifier')
model.save("nodule-classifier.tfl")
print("Network trained and saved as nodule-classifier.tfl!")
h5f.close()
h5f2.close()
def train():
# global epoch
epochs = 500
embeded_dim = 100
lr = 0.0001
batch_size = 1000
use_cuda = torch.cuda.is_available()
# use_cuda = False
loss_func = nn.CrossEntropyLoss()
word_index_dic, index_word_dic = data.word_index()
length = len(word_index_dic)
print(length)
cnn_model1 = CNNModel(int(length), int(embeded_dim), 2)
if use_cuda:
cnn_model1.cuda()
optimizer = torch.optim.Adam(cnn_model1.parameters(), lr=lr)
train_size = 100
result_list = []
targer_list = []
num = 0
F = 0
for epoch in range(epochs):
for data_list in get_data_list(train_size):
for data_item in data_list:
input_tensor = []
for item in data_item[0].lower().replace(".", "").replace(",", "").replace("<br", "") \
.replace("/>", "").replace("?", "").replace(";", "").strip().split(" "):
try:
input_tensor.append(word_index_dic[item])
except KeyError:
# print(item)
pass
try:
if len(input_tensor) > 400:
input_tensor = input_tensor[:400]
else:
while len(input_tensor) < 400:
input_tensor = input_tensor.append(0)
except:
continue
input_tensor = torch.LongTensor([input_tensor])
targer = torch.LongTensor([data_item[1]])
targer_list.append(data_item[1])
if use_cuda:
input_tensor = input_tensor.cuda()
targer = targer.cuda()
cnn_model1.zero_grad()
predict = cnn_model1(input_tensor)
# print(predict.argmax())
# print(int(predict.argmax()))
# print(targer)
result_list.append(int(predict.argmax()))
loss = loss_func(predict, targer)
# print(loss)
loss.backward()
optimizer.step()
num += 1
# if num % batch_size == 0:
# num = 0
# new_f1 = f1_score(result_list, targer_list, average='weighted')
# print(new_f1)
# # if new_f1 > F:
# # F = new_f1
# # print("F1:\t" + str(F) + "\n")
# # parmeter = cnn_model1.state_dict()
# # torch.save(parmeter, "./checkpoints_new3/CNN_model_" + str(epoch + 1) + "_f1_" + str(F) + ".pt")
# result_list = []
# targer_list = []
parmeter = cnn_model1.state_dict()
torch.save(parmeter,
"./checkpoints_new4/CNN_model_" + str(epoch + 1) + ".pt")
def test():
import os
print("开始测试")
# 定义使用的显卡
os.environ['CUDA_VISIBLE_DEVICES'] = '0'
num = 0
number = 0
use_cuda = torch.cuda.is_available()
word_index_dic, index_word_dic = data.word_index()
length = len(word_index_dic)
embeded_dim = 100
bach_size = 100
torch.cuda.set_device(0)
# 创建模型类
cnn_model1 = CNNModel(int(length), int(embeded_dim), 2)
if use_cuda:
# 将模型转换成GPU版本
cnn_model1.cuda()
# 加载模型中的参数
# print(list(os.walk("./checkpoints_new2")))
# for model_file in list(os.walk("./checkpoints_new2"))[0][2]:
# print(model_file)
# path = os.path.join("./checkpoints_new1",model_file)
# 176
# P: 0.8079631852741096
# F: 0.8076630195182558
# for index in range(150, 500):
for index in range(151, 152):
result_list = []
targer_list = []
path = "./checkpoints_new/CNN_model_" \
+ str(index) + ".pt"
print("加载模型")
print("CNN_model_" + str(index) + ".pt")
try:
cnn_model1.load_state_dict(torch.load(path))
except:
print("error")
# 开始测试
for data_list in get_data_list(bach_size):
for data_item in data_list:
# try:
input_tensor = []
# 清除一些没用的字符 包括英文的逗号等
for item in data_item[0].lower().replace(".", "").replace(",", "").replace("<br", "") \
.replace("/>", "").replace("?", "").replace(";", "").strip().split(" "):
try:
# 将单词映射成数字
key = word_index_dic[item]
input_tensor.append(key)
except KeyError:
pass
if len(input_tensor) > 400:
input_tensor = input_tensor[:400]
else:
# print(input_tensor)
# print(data_item[0])
# print(type(input_tensor))
while len(input_tensor) < 400:
input_tensor.append(0)
# 将输入转换成tensor的格式
input_tensor = torch.LongTensor([input_tensor])
# 加载计算图
with torch.no_grad():
if use_cuda:
input_tensor = input_tensor.cuda()
# 将输入放到模型中 获取输出预测值
predict = cnn_model1(input_tensor)
# argmax()函数用来获取 列表数值最大的数字的 “位置”
predict_label = int(predict.argmax())
# print("predict_label:", predict_label, "True:", data_item[1])
# 把预测结果和正确的结果记录下来
result_list.append(int(predict_label))
targer_list.append(data_item[1])
# number += 1
# except TypeError:
# num+=1
# print(data_item)
new_f1 = f1_score(result_list, targer_list, average='binary')
P = accuracy_score(
result_list,
targer_list,
)
# with open("result_file_end.txt", "a+", encoding='utf8') as f:
# f.write(str(index) + "\n")
# f.write("P:\t" + str(P) + "\n")
# f.write("F:\t" + str(new_f1) + "\n")
# f.write("________________________________________________\n")
# print("N:", index)
print("P:", P)
print("F:", new_f1)
print("==================================")
if __name__ == '__main__':
logging.basicConfig(level=logging.DEBUG)
if IS_TRAINING:
logger.info("Preprocess...")
preprocessor = ImagePreprocessor(preprocessor_config)
logger.info("Making dataset...")
data_loader_config = ImageSetDataLoaderConfig(
image_dict=preprocessor.get_image_dict(),
image_shape=INPUT_SHAPE_REVERSE,
shuffle_buffer_size=100000,
batch_size=100
)
data_loader = ImageSetDataLoader(data_loader_config)
model = CNNModel(model_config)
trainer = UniversalTrainer(model.get_model(), data_loader.get_dataset(), trainer_config)
logger.info("Training...")
trainer.train()
trainer.save("log/test.h5")
else:
model = CNNModel(model_config)
trainer = UniversalTrainer(model.get_model(), None, trainer_config)
trainer.load("log/test.h5")
valid_image_list = os.listdir(VALIDATION_IMAGE_ROOT)
logger.debug("Validation image list: %s", valid_image_list)
for image_file_name in valid_image_list:
image_file = os.path.join(VALIDATION_IMAGE_ROOT, image_file_name)
def main():
mask_extra()
lungs_ROI()
generate_images()
height = 50
width = 50
workpath = '../data/visualization/'
workpath_1 = '../testing_patient/'
file = glob(workpath + '*.npy')
target = file[0][33:-4]
print(target)
file_list = glob('../testing_patient/lungmask_*.npy')
print(file_list)
for file_l in file_list:
if file_l[33:-4] == target:
file_name = file_l[:-4]
print(file_name)
# file_name = '../testing_patient/lungmask_0003_0125'
image = cv2.imread(file_name + '.png')
lung_mask = np.load(workpath_1 + file_name + '.npy')
image_files = np.load(workpath_1 + 'images_' + file_name[28:] + '.npy')
SeletiveSearch(image)
images_path = '../temptation/'
filelist = glob(images_path + '*.png')
filelist.sort(key=lambda x: int(x[18:-4]))
print(filelist)
image_num = len(filelist)
data = np.zeros((image_num, height, width, 1))
for idx in range(image_num):
img = imread(filelist[idx]).astype('float32')
img = img.reshape(-1, img.shape[0], img.shape[1], 1)
data[idx, :, :, :] = img
print(data.shape)
# Read images and corresponding labels
csvfile = open('../testing_patient/predicted_nodules.csv', 'r')
csvReader = csv.reader(csvfile)
images_labels = list(csvReader)
csvfile_1 = open('../testing_patient/file_classes.csv', 'r')
csvReader_1 = csv.reader(csvfile_1)
images_nodules = list(csvReader_1)
real_candidates = []
candidates = []
del images_labels[0]
del images_nodules[0]
for i_l in images_labels:
i_l[1] = eval(i_l[1])
i_l[2] = eval(i_l[2])
candidates.append(i_l[2])
print(images_labels)
print(candidates)
# Get the lung nodule coordinates
for j_l in images_nodules:
if j_l[0] == file_name[19:]:
j_l[1] = eval(j_l[1])
j_l[2] = eval(j_l[2])
real_candidates.append((j_l[1], j_l[2]))
# Mapping the real regions that contain lung nodules
real_nodules = []
for candidate in candidates:
if (candidate[0], candidate[1]) in real_candidates:
real_nodules.append(candidate)
print(real_nodules)
# Feed the data into trained model.
cnnnet = CNNModel()
network = cnnnet.define_network(data, 'testtrain')
model = tflearn.DNN(network,
tensorboard_verbose=0,
checkpoint_path='nodule-classifier.tfl.ckpt')
model.load('nodule-classifier.tfl')
predictions = np.vstack(model.predict(data[:, :, :, :]))
label_predictions = np.zeros_like(predictions)
label_predictions[np.arange(len(predictions)), predictions.argmax(1)] = 1
print(len(label_predictions))
index_list = []
for ind, val in enumerate(label_predictions):
if val[1] == 1:
index_list.append(ind)
print(len(index_list))
print(index_list)
nodule_candidate = []
for i in index_list:
nodule_candidate.append(candidates[i])
print(nodule_candidate)
fig, ax = plt.subplots(2, 2, figsize=[8, 8])
ax[0, 0].imshow(image_files[0], cmap='gray')
ax[0, 1].imshow(image_files[0] * lung_mask[0], cmap='gray')
ax[1, 0].imshow(image)
ax[1, 1].imshow(image)
for x, y, w, h in candidates:
rect = mpatches.Rectangle((x, y),
w,
h,
fill=False,
edgecolor='red',
linewidth=1)
ax[1, 0].add_patch(rect)
#for x, y, w, h in real_nodules:
# rect = mpatches.Rectangle((x, y), w, h, fill=False, edgecolor='yellow', linewidth=1)
# ax[1, 1].add_patch(rect)
for x, y, w, h in nodule_candidate:
rect = mpatches.Rectangle((x - 3, y - 3),
w + 5,
h + 5,
fill=False,
edgecolor='red',
linewidth=1)
ax[1, 1].add_patch(rect)
plt.show()
shutil.rmtree('../temptation')
os.mkdir('../temptation')
def main():
height = 50
width = 50
# To get the images from the folder and to sort them as the sequence as csv files
images_path = '../images/'
filelist = glob(images_path + '*.png')
print(filelist)
filelist.sort(key=lambda x: int(x[29:-4]))
# Read images and corresponding labels
csvfile = open('../submission_files/regions_labels.csv', 'r')
csvReader = csv.reader(csvfile)
# images_labels contains the name of images and the labels generated from selective search
images_labels = list(csvReader)
del images_labels[0]
for i_l in images_labels:
i_l[1] = eval(i_l[1])
i_l[2] = eval(i_l[2])
print(len(images_labels))
# Pre-processing the data to generate all dataset.
image_num = len(filelist)
data = np.zeros((image_num, height, width, 1))
for idx in range(image_num):
img = imread(filelist[idx]).astype('float32')
img = img.reshape(-1, img.shape[0], img.shape[1], 1)
data[idx, :, :, :] = img
print(data.shape)
# Feed the data into trained model.
cnnnet = CNNModel()
network = cnnnet.define_network(data, 'testtrain')
model = tflearn.DNN(network,
tensorboard_verbose=0,
checkpoint_path='nodule-classifier.tfl.ckpt')
model.load('nodule-classifier.tfl')
predictions = np.vstack(model.predict(data[:, :, :, :]))
label_predictions = np.zeros_like(predictions)
label_predictions[np.arange(len(predictions)), predictions.argmax(1)] = 1
print(len(label_predictions))
index_list = []
for ind, val in enumerate(label_predictions):
if val[1] == 1:
index_list.append(ind)
images_name = []
for index in index_list:
#print(images_labels[index][0][:18])
images_name.append(images_labels[index][0][:18] + '.png')
images_name2 = list(set(images_name))
images_name2.sort(key=images_name.index)
print(images_name2)
print(len(images_name2))
img_list = glob('../data/test_images/*.png')
img_list.sort()
allimages = []
for l in img_list:
allimages.append(l[20:])
print(allimages)
# Writing the results into the files to show whether the patient contains nodules
f_submission1 = open('../submission_files/predicted_patient_labels.csv',
'w')
writer_1 = csv.writer(f_submission1)
writer_1.writerow(('Patient', 'Predict_Labels'))
for a_img in allimages:
if a_img in images_name2:
writer_1.writerow((a_img, 1))
else:
writer_1.writerow((a_img, 0))
# Writing the results into the submission csv file
f_submission = open('../submission_files/predicted_regions_labels.csv',
'w')
writer = csv.writer(f_submission)
writer.writerow(
('File_name', 'Labels', 'Rect', 'Actual_Nodule', 'Predict_Nodule'))
for i in range(len(images_labels)):
#index = index_list[i]
#print(images_labels[index][0][:])
writer.writerow(
(images_labels[i][0][:], images_labels[i][1], images_labels[i][2],
images_labels[i][3], label_predictions[i][1]))
actual = []
pre = []
for i in range(len(images_labels)):
actual.append(eval(images_labels[i][3]))
pre.append(predictions[i][1])
print(actual)
print(pre)
fpr, tpr, thresholds = roc_curve(actual, pre, pos_label=1)
roc_auc = auc(fpr, tpr)
print("The false positive rate is: ", fpr)
print("The true positive rate is:", tpr)
print(thresholds)
print("The auc is:", roc_auc)
plt.figure()
lw = 2
plt.plot(fpr,
tpr,
color='darkorange',
lw=lw,
label='(AUC = %0.2f)' % roc_auc)
plt.plot([0, 1], [0, 1], color='navy', lw=lw, linestyle='--')
plt.axis('equal')
plt.xlabel('False Positive Rate')
plt.ylabel('True Positive Rate')
plt.legend(loc='lower right')
plt.show()
csvfile.close()
f_submission.close()
f_submission1.close()
