def make_pos_annotation_imgs():
src_dir = variables.LUNA_16_trn_DIR2D2 + "metadata/"
dst_dir = variables.BASE_DIR_SSD + "luna16_trn_cubes_pos/"
for file_path in glob.glob(dst_dir + "*.*"):
os.remove(file_path)
for patient_index, csv_file in enumerate(
glob.glob(src_dir + "*_annos_pos.csv")):
patient_id = ntpath.basename(csv_file).replace("_annos_pos.csv", "")
df_annos = pandas.read_csv(csv_file)
if len(df_annos) == 0:
continue
imgs = operators.load_patient_imgs(patient_id,
variables.LUNA_16_trn_DIR2D2,
"*" + "_i" + ".png")
for index, row in df_annos.iterrows():
coord_x = int(row["coord_x"] * imgs.shape[2])
coord_y = int(row["coord_y"] * imgs.shape[1])
coord_z = int(row["coord_z"] * imgs.shape[0])
diam_mm = int(row["diameter"] * imgs.shape[2])
anno_index = int(row["anno_index"])
cube_img = get_cube_from_img(imgs, coord_x, coord_y, coord_z, 64)
if cube_img.sum() < 5:
print(" ***** Skipping ", coord_x, coord_y, coord_z)
continue
if cube_img.mean() < 10:
print(" ***** Suspicious ", coord_x, coord_y, coord_z)
save_cube_img(
dst_dir + patient_id + "_" + str(anno_index) + "_" +
str(diam_mm) + "_1_" + "pos.png", cube_img, 8, 8)
operators.print_tabbed([patient_index, patient_id,
len(df_annos)], [5, 64, 8])
def make_candidate_auto_imgs(candidate_types=[]):
dst_dir = variables.BASE_DIR_SSD + "generated_trndata/luna16_trn_cubes_auto/"
if not os.path.exists(dst_dir):
os.mkdir(dst_dir)
for candidate_type in candidate_types:
for file_path in glob.glob(dst_dir + "*_" + candidate_type + ".png"):
os.remove(file_path)
for candidate_type in candidate_types:
if candidate_type == "falsepos":
src_dir = "resources/luna16_falsepos_labels/"
else:
src_dir = variables.LUNA16_EXTRACTED_IMAGE_DIR + "_labels/"
for index, csv_file in enumerate(
glob.glob(src_dir + "*_candidates_" + candidate_type +
".csv")):
patient_id = ntpath.basename(csv_file).replace(
"_candidates_" + candidate_type + ".csv", "")
print(index, ",patient: ", patient_id, " type:", candidate_type)
df_annos = pandas.read_csv(csv_file)
if len(df_annos) == 0:
continue
imgs = operators.load_patient_imgs(
patient_id,
variables.LUNA16_EXTRACTED_IMAGE_DIR,
"*" + "_i" + ".png",
exclude_wildcards=[])
row_no = 0
for index, row in df_annos.iterrows():
coord_x = int(row["coord_x"] * imgs.shape[2])
coord_y = int(row["coord_y"] * imgs.shape[1])
coord_z = int(row["coord_z"] * imgs.shape[0])
anno_index = int(row["anno_index"])
cube_img = get_cube_from_img(imgs, coord_x, coord_y, coord_z,
48)
if cube_img.sum() < 10:
print("Skipping ", coord_x, coord_y, coord_z)
continue
try:
save_cube_img(
dst_dir + patient_id + "_" + str(anno_index) + "_0_" +
candidate_type + ".png", cube_img, 6, 8)
except Exception as ex:
print(ex)
row_no += 1
max_item = 240 if candidate_type == "white" else 200
if candidate_type == "luna":
max_item = 500
if row_no > max_item:
break
def filter_patient_nods_predictions(df_nod_predictions: pandas.DataFrame,
patient_id,
view_size,
luna16=True):
src_dir = variables.LUNA16_EXTRACTED_IMAGE_DIR if luna16 else variables.NDSB3_EXTRACTED_IMAGE_DIR
patient_mask = operators.load_patient_imgs(patient_id, src_dir, "*_m.png")
delete_indices = []
for index, row in df_nod_predictions.iterrows():
z_perc = row["coord_z"]
y_perc = row["coord_y"]
center_x = int(round(row["coord_x"] * patient_mask.shape[2]))
center_y = int(round(y_perc * patient_mask.shape[1]))
center_z = int(round(z_perc * patient_mask.shape[0]))
mal_score = row["diameter_mm"]
start_y = center_y - view_size / 2
start_x = center_x - view_size / 2
nod_in_mask = False
for z_index in [-1, 0, 1]:
img = patient_mask[z_index + center_z]
start_x = int(start_x)
start_y = int(start_y)
view_size = int(view_size)
img_roi = img[start_y:start_y + view_size,
start_x:start_x + view_size]
if img_roi.sum() > 255:
nod_in_mask = True
if not nod_in_mask:
print("nod not in mask: ", (center_x, center_y, center_z))
if mal_score > 0:
mal_score *= -1
df_nod_predictions.loc[index, "diameter_mm"] = mal_score
else:
if center_z < 30:
print("Z < 30: ", patient_id, " center z:", center_z,
" y_perc: ", y_perc)
if mal_score > 0:
mal_score *= -1
df_nod_predictions.loc[index, "diameter_mm"] = mal_score
if (z_perc > 0.75 or z_perc < 0.25) and y_perc > 0.85:
print("SUSPICIOUS FALSEPOSITIVE: ", patient_id, " center z:",
center_z, " y_perc: ", y_perc)
if center_z < 50 and y_perc < 0.30:
print("SUSPICIOUS FALSEPOSITIVE OUT OF RANGE: ", patient_id,
" center z:", center_z, " y_perc: ", y_perc)
df_nod_predictions.drop(df_nod_predictions.index[delete_indices],
inplace=True)
return df_nod_predictions
def make_pos_annotation_imgs_manual_ndsb3():
src_dir = "resources/ndsb3_manual_labels/"
dst_dir = variables.BASE_DIR_SSD + "generated_trndata/ndsb3_trn_cubes_manual/"
if not os.path.exists(dst_dir):
os.mkdir(dst_dir)
trn_label_df = pandas.read_csv("resources/stage1_labels.csv")
trn_label_df.set_index(["id"], inplace=True)
for file_path in glob.glob(dst_dir + "*.*"):
os.remove(file_path)
for patient_index, csv_file in enumerate(glob.glob(src_dir + "*.csv")):
patient_id = ntpath.basename(csv_file).replace(".csv", "")
if "1.3.6.1.4.1" in patient_id:
continue
cancer_label = trn_label_df.loc[patient_id]["cancer"]
df_annos = pandas.read_csv(csv_file)
if len(df_annos) == 0:
continue
imgs = operators.load_patient_imgs(patient_id,
variables.NDSB3_EXTRACTED_IMAGE_DIR,
"*" + "_i" + ".png")
anno_index = 0
for index, row in df_annos.iterrows():
pos_neg = "pos" if row["id"] == 0 else "neg"
coord_x = int(row["x"] * imgs.shape[2])
coord_y = int(row["y"] * imgs.shape[1])
coord_z = int(row["z"] * imgs.shape[0])
malscore = int(round(row["dmm"]))
anno_index += 1
cube_img = get_cube_from_img(imgs, coord_x, coord_y, coord_z, 64)
if cube_img.sum() < 5:
print(" ***** Skipping ", coord_x, coord_y, coord_z)
continue
if cube_img.mean() < 10:
print(" ***** Suspicious ", coord_x, coord_y, coord_z)
if cube_img.shape != (64, 64, 64):
print(" ***** incorrect shape !!! ", str(anno_index), " - ",
(coord_x, coord_y, coord_z))
continue
print(patient_id)
assert malscore > 0 or pos_neg == "neg"
save_cube_img(
dst_dir + "ndsb3manual_" + patient_id + "_" + str(anno_index) +
"_" + pos_neg + "_" + str(cancer_label) + "_" + str(malscore) +
"_1_pn.png", cube_img, 8, 8)
operators.print_tabbed([patient_index, patient_id,
len(df_annos)], [5, 64, 8])
def make_pos_annotation_imgs_manual():
src_dir = "resources/luna16_manual_labels/"
dst_dir = variables.BASE_DIR_SSD + "generated_trndata/luna16_trn_cubes_manual/"
if not os.path.exists(dst_dir):
os.mkdir(dst_dir)
for file_path in glob.glob(dst_dir + "*_manual.*"):
os.remove(file_path)
for patient_index, csv_file in enumerate(glob.glob(src_dir + "*.csv")):
patient_id = ntpath.basename(csv_file).replace(".csv", "")
if "1.3.6.1.4" not in patient_id:
continue
print(patient_id)
df_annos = pandas.read_csv(csv_file)
if len(df_annos) == 0:
continue
imgs = operators.load_patient_imgs(
patient_id, variables.LUNA16_EXTRACTED_IMAGE_DIR,
"*" + "_i" + ".png")
for index, row in df_annos.iterrows():
coord_x = int(row["x"] * imgs.shape[2])
coord_y = int(row["y"] * imgs.shape[1])
coord_z = int(row["z"] * imgs.shape[0])
diameter = int(row["d"] * imgs.shape[2])
node_type = int(row["id"])
malscore = int(diameter)
malscore = min(25, malscore)
malscore = max(16, malscore)
anno_index = index
cube_img = get_cube_from_img(imgs, coord_x, coord_y, coord_z, 64)
if cube_img.sum() < 5:
print(" ***** Skipping ", coord_x, coord_y, coord_z)
continue
if cube_img.mean() < 10:
print(" ***** Suspicious ", coord_x, coord_y, coord_z)
if cube_img.shape != (64, 64, 64):
print(" ***** incorrect shape !!! ", str(anno_index), " - ",
(coord_x, coord_y, coord_z))
continue
save_cube_img(
dst_dir + patient_id + "_" + str(anno_index) + "_" +
str(malscore) + "_1_" + ("pos" if node_type == 0 else "neg") +
".png", cube_img, 8, 8)
operators.print_tabbed([patient_index, patient_id,
len(df_annos)], [5, 64, 8])
def make_annotation_imgs_lidc():
src_dir = variables.LUNA16_EXTRACTED_IMAGE_DIR + "_labels/"
dst_dir = variables.BASE_DIR_SSD + "generated_trndata/luna16_trn_cubes_lidc/"
if not os.path.exists(dst_dir):
os.mkdir(dst_dir)
for file_path in glob.glob(dst_dir + "*.*"):
os.remove(file_path)
for patient_index, csv_file in enumerate(
glob.glob(src_dir + "*_annos_pos_lidc.csv")):
patient_id = ntpath.basename(csv_file).replace("_annos_pos_lidc.csv",
"")
df_annos = pandas.read_csv(csv_file)
if len(df_annos) == 0:
continue
imgs = operators.load_patient_imgs(
patient_id, variables.LUNA16_EXTRACTED_IMAGE_DIR,
"*" + "_i" + ".png")
for index, row in df_annos.iterrows():
coord_x = int(row["coord_x"] * imgs.shape[2])
coord_y = int(row["coord_y"] * imgs.shape[1])
coord_z = int(row["coord_z"] * imgs.shape[0])
malscore = int(row["malscore"])
anno_index = row["anno_index"]
anno_index = str(anno_index).replace(" ", "xspacex").replace(
".", "xpointx").replace("_", "xunderscorex")
cube_img = get_cube_from_img(imgs, coord_x, coord_y, coord_z, 64)
if cube_img.sum() < 5:
print(" ***** Skipping ", coord_x, coord_y, coord_z)
continue
if cube_img.mean() < 10:
print(" ***** Suspicious ", coord_x, coord_y, coord_z)
if cube_img.shape != (64, 64, 64):
print(" ***** incorrect shape !!! ", str(anno_index), " - ",
(coord_x, coord_y, coord_z))
continue
save_cube_img(
dst_dir + patient_id + "_" + str(anno_index) + "_" +
str(malscore * malscore) + "_1_pos.png", cube_img, 8, 8)
operators.print_tabbed([patient_index, patient_id,
len(df_annos)], [5, 64, 8])
def predict_cubes(model_path,
continue_job,
only_patient_id=None,
luna16=True,
magnification=1,
flip=False,
trn_data=True,
valid_no=-1,
ext_name="",
fold_count=2):
if luna16:
dst_dir = variables.LUNA_nod_DETECTION_DIR
else:
dst_dir = variables.NDSB3_nod_DETECTION_DIR + "2"
if not os.path.exists(dst_dir):
os.makedirs(dst_dir)
valid_ext = ""
flip_ext = ""
if flip:
flip_ext = "_flip"
dst_dir += "predictions" + str(int(
magnification * 10)) + valid_ext + flip_ext + "_" + ext_name + "/"
if not os.path.exists(dst_dir):
os.makedirs(dst_dir)
sw = operators.Stopwatch.start_new()
model = nodule_detector.get_net(input_shape=(SIZE, SIZE, SIZE, 1),
load_weight_path=model_path)
if luna16:
labels_df = pandas.read_csv(
"/media/pikachu/Seagate Backup Plus Drive/LC nod Detection/resources/luna16_annotations/annotations.csv"
)
if not luna16:
if trn_data:
labels_df = pandas.read_csv("resources/stage1_labels.csv")
labels_df.set_index(["id"], inplace=True)
else:
labels_df = pandas.read_csv(
"resources/stage2_sample_submission.csv")
labels_df.set_index(["id"], inplace=True)
patient_ids = []
for file_name in os.listdir(variables.LUNA16_EXTRACTED_IMAGE_DIR):
if not os.path.isdir(variables.LUNA16_EXTRACTED_IMAGE_DIR + file_name):
continue
patient_ids.append(file_name)
all_predictions_csv = []
for patient_index, patient_id in enumerate(reversed(patient_ids)):
if not luna16:
if patient_id not in labels_df.index:
continue
if "metadata" in patient_id:
continue
if only_patient_id is not None and only_patient_id != patient_id:
continue
if valid_no is not None and trn_data:
patient_fold = operators.get_patient_fold(patient_id)
patient_fold %= fold_count
if patient_fold != valid_no:
continue
print(patient_index, ": ", patient_id)
csv_target_path = dst_dir + patient_id + ".csv"
if continue_job and only_patient_id is None:
if os.path.exists(csv_target_path):
continue
patient_img = operators.load_patient_imgs(
patient_id, variables.LUNA16_EXTRACTED_IMAGE_DIR, "*_i.png", [])
if magnification != 1:
patient_img = operators.rescale_patient_imgs(
patient_img, (1, 1, 1), magnification)
patient_mask = operators.load_patient_imgs(
patient_id, variables.LUNA16_EXTRACTED_IMAGE_DIR, "*_m.png", [])
if magnification != 1:
patient_mask = operators.rescale_patient_imgs(patient_mask,
(1, 1, 1),
magnification,
is_mask_image=True)
step = PREDICT_STEP
CROP_SIZE = SIZE
predict_volume_shape_list = [0, 0, 0]
for dim in range(3):
dim_indent = 0
while dim_indent + CROP_SIZE < patient_img.shape[dim]:
predict_volume_shape_list[dim] += 1
dim_indent += step
predict_volume_shape = (predict_volume_shape_list[0],
predict_volume_shape_list[1],
predict_volume_shape_list[2])
predict_volume = numpy.zeros(shape=predict_volume_shape, dtype=float)
print("Predict volume shape: ", predict_volume.shape)
done_count = 0
skipped_count = 0
batch_size = 32
batch_list = []
batch_list_coords = []
patient_predictions_csv = []
cube_img = None
annotation_index = 0
for z in range(0, predict_volume_shape[0]):
for y in range(0, predict_volume_shape[1]):
for x in range(0, predict_volume_shape[2]):
cube_img = patient_img[z * step:z * step + CROP_SIZE,
y * step:y * step + CROP_SIZE,
x * step:x * step + CROP_SIZE]
cube_mask = patient_mask[z * step:z * step + CROP_SIZE,
y * step:y * step + CROP_SIZE,
x * step:x * step + CROP_SIZE]
if cube_mask.sum() < 2000:
skipped_count += 1
else:
if flip:
cube_img = cube_img[:, :, ::-1]
if CROP_SIZE != SIZE:
cube_img = operators.rescale_patient_imgs2(
cube_img, (SIZE, SIZE, SIZE))
operators.save_cube_img(
"/media/pikachu/Seagate Backup Plus Drive/LC nod Detection/workdir",
cube_img, 8, 4)
cube_mask = operators.rescale_patient_imgs2(
cube_mask, (SIZE, SIZE, SIZE))
img_prep = prepare_image_for_net3D(cube_img)
batch_list.append(img_prep)
batch_list_coords.append((z, y, x))
if len(batch_list) % batch_size == 0:
batch_data = numpy.vstack(batch_list)
p = model.predict(batch_data,
batch_size=batch_size)
for i in range(len(p[0])):
p_z = batch_list_coords[i][0]
p_y = batch_list_coords[i][1]
p_x = batch_list_coords[i][2]
nod_chance = p[0][i][0]
predict_volume[p_z, p_y, p_x] = nod_chance
if nod_chance > P_TH:
p_z = p_z * step + CROP_SIZE / 2
p_y = p_y * step + CROP_SIZE / 2
p_x = p_x * step + CROP_SIZE / 2
p_z_perc = round(
p_z / patient_img.shape[0], 4)
p_y_perc = round(
p_y / patient_img.shape[1], 4)
p_x_perc = round(
p_x / patient_img.shape[2], 4)
diameter_mm = round(p[1][i][0], 4)
diameter_perc = round(
2 * step / patient_img.shape[2], 4)
diameter_perc = round(
diameter_mm / patient_img.shape[2], 4)
nod_chance = round(nod_chance, 4)
patient_predictions_csv_line = [
annotation_index, p_x_perc, p_y_perc,
p_z_perc, diameter_perc, nod_chance,
diameter_mm
]
patient_predictions_csv.append(
patient_predictions_csv_line)
all_predictions_csv.append(
[patient_id] +
patient_predictions_csv_line)
annotation_index += 1
batch_list = []
batch_list_coords = []
done_count += 1
if done_count % 10000 == 0:
print("Done: ", done_count, " skipped:", skipped_count)
df = pandas.DataFrame(patient_predictions_csv,
columns=[
"anno_index", "coord_x", "coord_y",
"coord_z", "diameter", "nod_chance",
"diameter_mm"
])
print(df)
if not df.empty:
filter_patient_nods_predictions(df, patient_id,
CROP_SIZE * magnification)
df.to_csv(csv_target_path, index=False)
print(predict_volume.mean())
print("Done in : ", sw.get_elapsed_seconds(), " seconds")
def make_negative_trn_data_based_on_predicted_luna_nods():
src_dir = variables.LUNA_nod_DETECTION_DIR
pos_labels_dir = variables.LUNA_nod_LABELS_DIR
keep_dist = SIZE + SIZE / 2
total_false_pos = 0
for csv_index, csv_path in enumerate(glob.glob(src_dir + "*.csv")):
file_name = ntpath.basename(csv_path)
patient_id = file_name.replace(".csv", "")
df_nod_predictions = pandas.read_csv(csv_path)
pos_annos_manual = None
manual_path = variables.MANUAL_ANNOTATIONS_LABELS_DIR + patient_id + ".csv"
if os.path.exists(manual_path):
pos_annos_manual = pandas.read_csv(manual_path)
filter_patient_nods_predictions(df_nod_predictions,
patient_id,
SIZE,
luna16=True)
pos_labels = pandas.read_csv(pos_labels_dir + patient_id +
"_annos_pos_lidc.csv")
print(csv_index, ": ", patient_id, ", pos", len(pos_labels))
patient_imgs = operators.load_patient_imgs(
patient_id, variables.LUNA_16_trn_DIR2D2, "*_m.png")
for nod_pred_index, nod_pred_row in df_nod_predictions.iterrows():
if nod_pred_row["diameter_mm"] < 0:
continue
nx, ny, nz = operators.percentage_to_pixels(
nod_pred_row["coord_x"], nod_pred_row["coord_y"],
nod_pred_row["coord_z"], patient_imgs)
diam_mm = nod_pred_row["diameter_mm"]
for label_index, label_row in pos_labels.iterrows():
px, py, pz = operators.percentage_to_pixels(
label_row["coord_x"], label_row["coord_y"],
label_row["coord_z"], patient_imgs)
dist = math.sqrt(
math.pow(nx - px, 2) + math.pow(ny - py, 2) +
math.pow(nz - pz, 2))
if dist < keep_dist:
if diam_mm >= 0:
diam_mm *= -1
df_nod_predictions.loc[nod_pred_index,
"diameter_mm"] = diam_mm
break
if pos_annos_manual is not None:
for index, label_row in pos_annos_manual.iterrows():
px, py, pz = operators.percentage_to_pixels(
label_row["x"], label_row["y"], label_row["z"],
patient_imgs)
diameter = label_row["d"] * patient_imgs[0].shape[1]
dist = math.sqrt(
math.pow(px - nx, 2) + math.pow(py - ny, 2) +
math.pow(pz - nz, 2))
if dist < (diameter + 72):
if diam_mm >= 0:
diam_mm *= -1
df_nod_predictions.loc[nod_pred_index,
"diameter_mm"] = diam_mm
print("#Too close", (nx, ny, nz))
break
df_nod_predictions.to_csv(csv_path, index=False)
df_nod_predictions = df_nod_predictions[
df_nod_predictions["diameter_mm"] >= 0]
df_nod_predictions.to_csv(pos_labels_dir + patient_id +
"_candidates_falsepos.csv",
index=False)
total_false_pos += len(df_nod_predictions)
print("Total false pos:", total_false_pos)
def proc_luna_candidates_patient(src_path, patient_id):
dst_dir = variables.LUNA16_EXTRACTED_IMAGE_DIR + "/_labels/"
img_dir = dst_dir + patient_id + "/"
df_pos_annos = pandas.read_csv(dst_dir + patient_id +
"_annos_pos_lidc.csv")
if not os.path.exists(dst_dir):
os.mkdir(dst_dir)
pos_annos_manual = None
manual_path = variables.EXTRA_DATA_DIR + "luna16_manual_labels/" + patient_id + ".csv"
if os.path.exists(manual_path):
pos_annos_manual = pandas.read_csv(manual_path)
itk_img = SimpleITK.ReadImage(src_path)
img_array = SimpleITK.GetArrayFromImage(itk_img)
print("Img array: ", img_array.shape)
print("Pos annos: ", len(df_pos_annos))
num_z, height, width = img_array.shape
origin = numpy.array(itk_img.GetOrigin())
print("Origin (x,y,z): ", origin)
spacing = numpy.array(itk_img.GetSpacing())
print("Spacing (x,y,z): ", spacing)
rescale = spacing / variables.TARGET_VOXEL_MM
print("Rescale: ", rescale)
direction = numpy.array(itk_img.GetDirection())
print("Direction: ", direction)
flip_direction_x = False
flip_direction_y = False
if round(direction[0]) == -1:
origin[0] *= -1
direction[0] = 1
flip_direction_x = True
print("Swappint x origin")
if round(direction[4]) == -1:
origin[1] *= -1
direction[4] = 1
flip_direction_y = True
print("Swappint y origin")
print("Direction: ", direction)
assert abs(sum(direction) - 3) < 0.01
src_df = pandas.read_csv(
"/media/pikachu/Seagate Backup Plus Drive/LC nod Detection/resources/luna16_annotations/"
+ "candidates_V2.csv")
src_df = src_df[src_df["seriesuid"] == patient_id]
src_df = src_df[src_df["class"] == 0]
patient_imgs = operators.load_patient_imgs(
patient_id, variables.LUNA16_EXTRACTED_IMAGE_DIR, "*_i.png")
candidate_list = []
for df_index, candiate_row in src_df.iterrows():
node_x = candiate_row["coordX"]
if flip_direction_x:
node_x *= -1
node_y = candiate_row["coordY"]
if flip_direction_y:
node_y *= -1
node_z = candiate_row["coordZ"]
candidate_diameter = 6
center_float = numpy.array([node_x, node_y, node_z])
center_int = numpy.rint((center_float - origin) / spacing)
center_float_rescaled = (center_float -
origin) / variables.TARGET_VOXEL_MM
center_float_percent = center_float_rescaled / patient_imgs.swapaxes(
0, 2).shape
coord_x = center_float_rescaled[0]
coord_y = center_float_rescaled[1]
coord_z = center_float_rescaled[2]
ok = True
for index, row in df_pos_annos.iterrows():
pos_coord_x = row["coord_x"] * patient_imgs.shape[2]
pos_coord_y = row["coord_y"] * patient_imgs.shape[1]
pos_coord_z = row["coord_z"] * patient_imgs.shape[0]
diameter = row["diameter"] * patient_imgs.shape[2]
dist = math.sqrt(
math.pow(pos_coord_x - coord_x, 2) +
math.pow(pos_coord_y - coord_y, 2) +
math.pow(pos_coord_z - coord_z, 2))
if dist < (diameter + 64):
ok = False
print("CANNOT", (coord_x, coord_y, coord_z))
break
if pos_annos_manual is not None and ok:
for index, row in pos_annos_manual.iterrows():
pos_coord_x = row["x"] * patient_imgs.shape[2]
pos_coord_y = row["y"] * patient_imgs.shape[1]
pos_coord_z = row["z"] * patient_imgs.shape[0]
diameter = row["d"] * patient_imgs.shape[2]
print((pos_coord_x, pos_coord_y, pos_coord_z))
print(center_float_rescaled)
dist = math.sqrt(
math.pow(pos_coord_x - center_float_rescaled[0], 2) +
math.pow(pos_coord_y - center_float_rescaled[1], 2) +
math.pow(pos_coord_z - center_float_rescaled[2], 2))
if dist < (diameter + 72):
ok = False
print("CANNOT", center_float_rescaled)
break
if not ok:
continue
candidate_list.append([
len(candidate_list),
round(center_float_percent[0], 4),
round(center_float_percent[1], 4),
round(center_float_percent[2], 4),
round(candidate_diameter / patient_imgs.shape[0], 4), 0
])
df_candidates = pandas.DataFrame(candidate_list,
columns=[
"anno_index", "coord_x", "coord_y",
"coord_z", "diameter", "malscore"
])
df_candidates.to_csv(dst_dir + patient_id + "_candidates_luna.csv",
index=False)
def proc_excluded_annotations_patient(src_path, patient_id):
df_node = pandas.read_csv(
"/media/pikachu/Seagate Backup Plus Drive/LC nod Detection/resources/luna16_annotations/annotations_excluded.csv"
)
dst_dir = variables.LUNA16_EXTRACTED_IMAGE_DIR + "_labels/"
if not os.path.exists(dst_dir):
os.mkdir(dst_dir)
dst_dir = dst_dir + patient_id + "/"
if not os.path.exists(dst_dir):
os.mkdir(dst_dir)
pos_annos_df = pandas.read_csv(variables.LUNA16_EXTRACTED_IMAGE_DIR +
"_labels/" + patient_id + "_annos_pos.csv")
pos_annos_manual = None
manual_path = variables.EXTRA_DATA_DIR + "luna16_manual_labels/" + patient_id + ".csv"
if os.path.exists(manual_path):
pos_annos_manual = pandas.read_csv(manual_path)
dmm = pos_annos_manual["dmm"]
itk_img = SimpleITK.ReadImage(src_path)
img_array = SimpleITK.GetArrayFromImage(itk_img)
print("Img array: ", img_array.shape)
df_patient = df_node[df_node["seriesuid"] == patient_id]
print("Annos: ", len(df_patient))
num_z, height, width = img_array.shape
origin = numpy.array(itk_img.GetOrigin())
print("Origin (x,y,z): ", origin)
spacing = numpy.array(itk_img.GetSpacing())
print("Spacing (x,y,z): ", spacing)
rescale = spacing / variables.TARGET_VOXEL_MM
print("Rescale: ", rescale)
direction = numpy.array(itk_img.GetDirection())
print("Direction: ", direction)
flip_direction_x = False
flip_direction_y = False
if round(direction[0]) == -1:
origin[0] *= -1
direction[0] = 1
flip_direction_x = True
print("Swappint x origin")
if round(direction[4]) == -1:
origin[1] *= -1
direction[4] = 1
flip_direction_y = True
print("Swappint y origin")
print("Direction: ", direction)
assert abs(sum(direction) - 3) < 0.01
patient_imgs = operators.load_patient_imgs(
patient_id, variables.LUNA16_EXTRACTED_IMAGE_DIR, "*_i.png")
neg_annos = []
df_patient = df_node[df_node["seriesuid"] == patient_id]
anno_index = 0
for index, annotation in df_patient.iterrows():
node_x = annotation["coordX"]
if flip_direction_x:
node_x *= -1
node_y = annotation["coordY"]
if flip_direction_y:
node_y *= -1
node_z = annotation["coordZ"]
center_float = numpy.array([node_x, node_y, node_z])
center_int = numpy.rint((center_float - origin) / spacing)
center_float_rescaled = (center_float -
origin) / variables.TARGET_VOXEL_MM
center_float_percent = center_float_rescaled / patient_imgs.swapaxes(
0, 2).shape
diameter_pixels = 6 / variables.TARGET_VOXEL_MM
diameter_percent = diameter_pixels / float(patient_imgs.shape[1])
ok = True
for index, row in pos_annos_df.iterrows():
pos_coord_x = row["coord_x"] * patient_imgs.shape[2]
pos_coord_y = row["coord_y"] * patient_imgs.shape[1]
pos_coord_z = row["coord_z"] * patient_imgs.shape[0]
diameter = row["diameter"] * patient_imgs.shape[2]
print((pos_coord_x, pos_coord_y, pos_coord_z))
print(center_float_rescaled)
dist = math.sqrt(
math.pow(pos_coord_x - center_float_rescaled[0], 2) +
math.pow(pos_coord_y - center_float_rescaled[1], 2) +
math.pow(pos_coord_z - center_float_rescaled[2], 2))
if dist < (diameter + 64):
ok = False
print("CANNOT", center_float_rescaled)
break
if pos_annos_manual is not None and ok:
for index, row in pos_annos_manual.iterrows():
pos_coord_x = row["x"] * patient_imgs.shape[2]
pos_coord_y = row["y"] * patient_imgs.shape[1]
pos_coord_z = row["z"] * patient_imgs.shape[0]
diameter = row["d"] * patient_imgs.shape[2]
print((pos_coord_x, pos_coord_y, pos_coord_z))
print(center_float_rescaled)
dist = math.sqrt(
math.pow(pos_coord_x - center_float_rescaled[0], 2) +
math.pow(pos_coord_y - center_float_rescaled[1], 2) +
math.pow(pos_coord_z - center_float_rescaled[2], 2))
if dist < (diameter + 72):
ok = False
print("CANNOT", center_float_rescaled)
break
if not ok:
continue
neg_annos.append([
anno_index,
round(center_float_percent[0], 4),
round(center_float_percent[1], 4),
round(center_float_percent[2], 4),
round(diameter_percent, 4), 1
])
anno_index += 1
df_annos = pandas.DataFrame(neg_annos,
columns=[
"anno_index", "coord_x", "coord_y",
"coord_z", "diameter", "malscore"
])
df_annos.to_csv(variables.LUNA16_EXTRACTED_IMAGE_DIR + "_labels/" +
patient_id + "_annos_excluded.csv",
index=False)
return [patient_id, spacing[0], spacing[1], spacing[2]]
def proc_pos_annotations_patient(src_path, patient_id):
df_node = pandas.read_csv(
"/media/pikachu/Seagate Backup Plus Drive/LC nod Detection/resources/luna16_annotations/annotations.csv"
)
dst_dir = variables.LUNA16_EXTRACTED_IMAGE_DIR + "_labels/"
if not os.path.exists(dst_dir):
os.mkdir(dst_dir)
dst_dir = dst_dir + patient_id + "/"
if not os.path.exists(dst_dir):
os.mkdir(dst_dir)
itk_img = SimpleITK.ReadImage(src_path)
img_array = SimpleITK.GetArrayFromImage(itk_img)
print("Img array: ", img_array.shape)
df_patient = df_node[df_node["seriesuid"] == patient_id]
print("Annos: ", len(df_patient))
num_z, height, width = img_array.shape
origin = numpy.array(itk_img.GetOrigin())
print("Origin (x,y,z): ", origin)
spacing = numpy.array(itk_img.GetSpacing())
print("Spacing (x,y,z): ", spacing)
rescale = spacing / variables.TARGET_VOXEL_MM
print("Rescale: ", rescale)
direction = numpy.array(itk_img.GetDirection())
print("Direction: ", direction)
flip_direction_x = False
flip_direction_y = False
if round(direction[0]) == -1:
origin[0] *= -1
direction[0] = 1
flip_direction_x = True
print("Swappint x origin")
if round(direction[4]) == -1:
origin[1] *= -1
direction[4] = 1
flip_direction_y = True
assert abs(sum(direction) - 3) < 0.01
patient_imgs = operators.load_patient_imgs(
patient_id, variables.LUNA16_EXTRACTED_IMAGE_DIR, "*_i.png")
pos_annos = []
df_patient = df_node[df_node["seriesuid"] == patient_id]
anno_index = 0
for index, annotation in df_patient.iterrows():
node_x = annotation["coordX"]
if flip_direction_x:
node_x *= -1
node_y = annotation["coordY"]
if flip_direction_y:
node_y *= -1
node_z = annotation["coordZ"]
diam_mm = annotation["diameter_mm"]
center_float = numpy.array([node_x, node_y, node_z])
center_int = numpy.rint((center_float - origin) / spacing)
center_float_rescaled = (center_float -
origin) / variables.TARGET_VOXEL_MM
center_float_percent = center_float_rescaled / patient_imgs.swapaxes(
0, 2).shape
diameter_pixels = diam_mm / variables.TARGET_VOXEL_MM
diameter_percent = diameter_pixels / float(patient_imgs.shape[1])
pos_annos.append([
anno_index,
round(center_float_percent[0], 4),
round(center_float_percent[1], 4),
round(center_float_percent[2], 4),
round(diameter_percent, 4), 1
])
anno_index += 1
df_annos = pandas.DataFrame(pos_annos,
columns=[
"anno_index", "coord_x", "coord_y",
"coord_z", "diameter", "malscore"
])
df_annos.to_csv(variables.LUNA16_EXTRACTED_IMAGE_DIR + "_labels/" +
patient_id + "_annos_pos.csv",
index=False)
return [patient_id, spacing[0], spacing[1], spacing[2]]