def retrieve_data(model_base_dir, model_dir, raw_data_dir,
predict_first_dataset):
if predict_first_dataset:
dataset_ID = "1"
model_ID = "2"
else:
dataset_ID = "2"
model_ID = "1"
print("loading data set {}".format(dataset_ID))
if predict_first_dataset:
file_dataset = model_base_dir + "dataset1.h5"
else:
file_dataset = model_base_dir + "dataset2.h5"
f = h5py.File(file_dataset.strip(), "r")
sizes = f["sizes"][()]
spacings = f["spacings"][()]
origins = f["origins"][()]
directions = f["directions"][()]
img_infos = tools.deserialize_img_infos(sizes, spacings, origins,
directions)
images = f["images"][()]
if predict_first_dataset:
model_file = model_dir + "weights2.hdf5"
else:
model_file = model_dir + "weights1.hdf5"
image_filename_templates = tools.get_filenames(predict_first_dataset,
raw_data_dir)
image_filenames = []
for f in image_filename_templates:
new_name = f[:3] + "_prediction.mhd" # eg "P01_prediction.mhd"
image_filenames.append(new_name)
print("predict with model {}".format(model_ID))
return images, img_infos, image_filenames, model_file
def xy_resampler(num_classes, data_dir, isFirstDataSet):
# first retrieve the directory and all ct files corresponding to the first/second data set
subdir = data_dir + "rawdata/"
files = get_filenames(isFirstDataSet, subdir)
print("resampling in " + subdir)
# now read and resample
batch_in = []
img_infos = []
for f in files:
print("resample ct " + f)
ctimg = sitk.ReadImage(data_dir + "rawdata/" + f)
for z in range(0, ctimg.GetDepth()):
batch_in.append(
np.expand_dims(sitk.GetArrayFromImage(ctimg[:, :, z]), axis=2))
img_infos.append(extract_image_info(ctimg))
# now retrieve the directory and all gt files corresponding to the first/second data set
subdir = data_dir + "groundtruth/"
files = get_filenames(isFirstDataSet, subdir)
print("resampling in " + subdir)
# now read and resample
batch_out = []
for f in files:
print("resample gt " + f)
gtimg = sitk.ReadImage(data_dir + "groundtruth/" + f)
for z in range(0, gtimg.GetDepth()):
batch_out.append(sitk.GetArrayFromImage(gtimg[:, :, z]))
batch_in = np.stack(batch_in)
batch_out = np.stack(batch_out)
print("size {}".format(batch_in.shape))
print("size {}".format(batch_out.shape))
batch_out = utils.to_categorical(batch_out, num_classes)
return (batch_in, batch_out, img_infos)
def generate_outputfile(ra, dec, redshift, mass):
print("Collecting filenames...")
data, goods = tools.get_filenames(ra=ra, dec=dec, filter="any")
ridx = (data[:,4] == 'r')
gidx = (data[:,4] == 'g')
iidx = (data[:,4] == 'i')
yidx = (data[:,4] == 'y')
zidx = (data[:,4] == 'z')
rdata = data[ridx,:]
base = rdata[:,:-2]
rdata = rdata[:,-2:].astype(str)
gdata = data[gidx,-2:].astype(str)
idata = data[iidx,-2:].astype(str)
ydata = data[yidx,-2:].astype(str)
zdata = data[zidx,-2:].astype(str)
rdata[:,1] = [x[:-1] for x in rdata[:,1]]
gdata[:,1] = [x[:-1] for x in gdata[:,1]]
idata[:,1] = [x[:-1] for x in idata[:,1]]
zdata[:,1] = [x[:-1] for x in zdata[:,1]]
ydata[:,1] = [x[:-1] for x in ydata[:,1]]
ra = ra[goods]
dec = dec[goods]
redshift = redshift[goods]
mass = mass[goods]
ra = ra.reshape((ra.size, 1))
dec = dec.reshape((dec.size, 1))
redshift = redshift.reshape((redshift.size, 1))
mass = mass.reshape((mass.size, 1))
base = np.delete(base, [2,3,4,5], axis=1)
outdata = np.hstack((ra, dec, redshift, mass, base, rdata, gdata, idata, ydata, zdata))
outdata = outdata.astype(str)
print("Writing output-file")
out = pd.DataFrame(outdata)
out.to_csv(outfile, header=['RA', 'DEC', 'REDSHIFT', 'MASS', 'PRO', 'SKY', 'TYPE', 'RNAME', 'RSHORTNAME', 'GNAME', 'GSHORTNAME', 'INAME', 'ISHORTNAME', 'YNAME', 'YSHORTNAME', 'ZNAME', 'ZSHORTNAME'], index=False)
import numpy as np
import matplotlib.pyplot as plt
from tools import get_params, get_filenames, sort_key
Tc = 2 / np.log(1 + np.sqrt(2))
lw = 3
c = ['indianred', 'orange', 'forestgreen', 'royalblue']
fig, ax = plt.subplots(figsize=(10, 8))
for filename in get_filenames('test_scalar'):
b, L, K = get_params(filename)
results = np.loadtxt(filename, unpack=True)
temp = results[1]
mse = results[6]
i = int(np.log(L) / np.log(b) - 3)
ax.semilogy(temp / Tc, mse, label="L=" + str(L), color=c[i], linewidth=lw)
# sort legend
handles, labels = ax.get_legend_handles_labels()
labels, handles = zip(*sorted(zip(labels, handles), key=sort_key))
ax.legend(handles, labels)
# fig params
plt.ylabel('Mean Squared Error')
plt.xlim(temp[0] / Tc, temp[-1] / Tc)
plt.xlabel(r'$T/T_c$')
plt.title('Testing MSE of Outputs From Large and Small Lattices')
def test_combined_models(model_date_xy, model_date_xz, model_date_yz):
#This needs to be set according to your directory
#Needs to contain /rawdata and /groundtruth with equally named mhd-files
root_dir = "C:/users/jfauser/IPCAI2019/ModelData/"
model_xy = "XY-Unet-Init"
model_xz = "XZ-Unet-Init"
model_yz = "YZ-Unet-Init"
model_dir_xy = root_dir + model_xy + "/" + model_date_xy + "/predictions/"
model_dir_xz = root_dir + model_xz + "/" + model_date_xz + "/predictions/"
model_dir_yz = root_dir + model_yz + "/" + model_date_yz + "/predictions/"
# ------------------------ majority vote ------------------------
useMajorityVoting = True
if useMajorityVoting:
model_name = "Combined-Unet-Majority"
# time string as a changing last folder does not makes sense here
out_dir = root_dir + model_name + "/last_result/predictions/"
if not os.path.isdir(root_dir + model_name):
os.mkdir(root_dir + model_name)
if not os.path.isdir(root_dir + model_name + "/last_result/"):
os.mkdir(root_dir + model_name + "/last_result/")
if not os.path.isdir(out_dir):
os.mkdir(out_dir)
# get all results from all models
files_xy = tools.get_filenames(True, model_dir_xy)
files_xy.extend(tools.get_filenames(False, model_dir_xy))
files_xz = tools.get_filenames(True, model_dir_xz)
files_xz.extend(tools.get_filenames(False, model_dir_xz))
files_yz = tools.get_filenames(True, model_dir_yz)
files_yz.extend(tools.get_filenames(False, model_dir_yz))
# sort for ease of use
files_xy.sort()
files_xz.sort()
files_yz.sort()
for fx, fy, fz in zip(files_xy, files_xz, files_yz):
print("voting on next file: {}".format(fx[:3]))
img1, info = itk.read_itk_array(model_dir_xy + fx)
size = info[0]
img2, info = itk.read_itk_array(model_dir_xz + fy)
img3, info = itk.read_itk_array(model_dir_yz + fz)
img = np.zeros((size[2], size[1], size[0]))
# majority vote
equals = img1[:, :, :] == img2[:, :, :]
img[equals] = img1[equals]
equals = img1[:, :, :] == img3[:, :, :]
img[equals] = img1[equals]
equals = img3[:, :, :] == img2[:, :, :]
img[equals] = img2[equals]
# chorda vote
equals = img1[:, :, :] == 5
img[equals] = img1[equals]
equals = img1[:, :, :] == 5
img[equals] = img1[equals]
equals = img3[:, :, :] == 5
img[equals] = img2[equals]
img_out = itk.create_itk_image_2(img, info)
itk.write_img(img_out, out_dir + fx[:3] + ".mhd")
def main():
"""cli entry point"""
parser = argparse.ArgumentParser(usage='%(prog)s [options]')
#------ in/out ------
parser.add_argument("-i",
"--input",
dest="input",
help="input source directory")
parser.add_argument("-j",
"--jpg",
dest="jpg",
action="store_true",
help="process only jpg files")
parser.add_argument("-d",
"--description",
dest="description",
help="add a description to the the image")
parser.add_argument("-e",
"--title",
dest="title",
help="add a title to the the image")
parser.add_argument("-t",
"--tags",
dest="tags",
help="add tags to the the image")
parser.add_argument("-v",
'--version',
dest='version',
action="store_true",
help="Version information")
#--------- options ---------
args = parser.parse_args()
if not args:
parser.print_help()
sys.exit(1)
#--------- version ---------
if args.version:
print("{} {}".format(sys.argv[0], '0.1'))
sys.exit(1)
#-=------- process ---------
if args.input:
# only load 'jpg' images
print("path: <{}>".format(args.input))
if args.jpg:
afiles = tools.get_fn_jpg(args.input)
else:
afiles = tools.get_filenames(args.input)
print("AFILES <{}>".format(afiles))
print("directory: ({}) <{}>".format(len(afiles), afiles))
if len(afiles) > 1:
# set tags OR defaults
if args.tags:
params['tags'] = args.tags
else:
tags = ""
# set title OR nothing
if args.title:
params['title'] = args.title
else:
title = ""
# set description OR nothing
if args.description:
params['description'] = args.description
else:
description = ""
# authenticate ONCE, then upload
# looped list of files
flickr = login.authenticate()
process_batch(flickr, afiles, params)
flickr = None
else:
# load all files found
fpn = os.path.join(os.curdir, args.input)
fn = args.input
# parameters
params['filename'] = fpn
params['title'] = filepath2title(fpn)
params['description'] = args.description
params['tags'] = args.tags
# check filepathname
if os.path.exists(fpn):
print("fpn <{}>".format(fpn))
# process one file
flickr = login.authenticate()
process(flickr, params)
flickr = None
else:
print("error: the source file is not found")
print(" <{}>".format(fpn))
def xz_resampler(num_classes, data_dir, isFirstDataSet):
# first retrieve the directory and all ct files corresponding to the first/second data set
subdir = data_dir + "rawdata/"
files = get_filenames(isFirstDataSet, subdir)
print("num classes: {}".format(num_classes))
print("resampling in " + subdir)
# now read and resample
batch_in = []
img_infos = []
netdim_z = 128
for f in files:
print("resample ct " + f)
ctimg = sitk.ReadImage(data_dir + "rawdata/" + f)
dim_x = ctimg.GetWidth()
dim_y = ctimg.GetHeight()
dim_z = ctimg.GetDepth()
extract_labels = netdim_z > dim_z # network has more slices than original image
slices = []
for y in range(0, dim_y):
slices.append(sitk.GetArrayFromImage(ctimg[:, y, :]))
extract_labels = netdim_z > dim_z # network has more slices than original image
for slice in slices:
new_slice = np.zeros((netdim_z, dim_x, 1))
if extract_labels:
# original z-direction small than unet input? just take partial unet segmentation
idx_start = int(0.5 * (netdim_z - dim_z))
new_slice[idx_start:idx_start + dim_z, :, 0] = slice
else:
# original z-direction is larger than unet input? put in segmentation
idx_start = int(
max(0, 0.5 * dim_z - 74)
) # jugular vein is slightly lower. so do not pick the middle
new_slice[:, :, 0] = slice[idx_start:idx_start + netdim_z, :]
batch_in.append(new_slice)
img_infos.append(extract_image_info(ctimg))
batch_in = np.stack(batch_in)
# now retrieve the directory and all gt files corresponding to the first/second data set
subdir = data_dir + "groundtruth/"
files = get_filenames(isFirstDataSet, subdir)
print("resampling in " + subdir)
# now read and resample
batch_out = []
for f in files:
print("resample gt " + f)
gtimg = sitk.ReadImage(data_dir + "groundtruth/" + f)
dim_x = gtimg.GetWidth()
dim_y = gtimg.GetHeight()
dim_z = gtimg.GetDepth()
extract_labels = netdim_z > dim_z # network has more slices than original image
slices = []
for y in range(0, dim_y):
slices.append(sitk.GetArrayFromImage(gtimg[:, y, :]))
for slice in slices:
new_slice = np.zeros((netdim_z, dim_x))
if extract_labels:
# original z-direction small than unet input? just take partial unet segmentation
idx_start = int(0.5 * (netdim_z - dim_z))
new_slice[idx_start:idx_start + dim_z, :] = slice
else:
# original z-direction is larger than unet input? put in segmentation
idx_start = int(
max(0, 0.5 * dim_z - 74)
) # jugular vein is slightly lower. so do not pick the middle
new_slice[:, :] = slice[idx_start:idx_start + netdim_z, :]
batch_out.append(new_slice)
batch_out = np.stack(batch_out)
print("size {}".format(batch_in.shape))
print("size {}".format(batch_out.shape))
batch_out = utils.to_categorical(batch_out, num_classes)
return (batch_in, batch_out, img_infos)
def main():
"""cli entry point"""
usage = "usage: %prog -i -o [-s]"
parser = OptionParser(usage)
#------ in/out ------
parser.add_option("-i",
"--input",
dest="input",
help="input source directory")
parser.add_option("-o",
"--output",
dest="output",
help="output source directory"),
parser.add_option("-j",
"--jpg",
dest="jpg",
action="store_true",
help="process only jpg files")
parser.add_option("-p",
"--png",
dest="png",
action="store_true",
help="process only png files")
#------ options ------
options, args = parser.parse_args()
#------ process ------
if options.input:
# input directory must exist
if not os.path.isdir(options.input):
print("Error: processing input files has failed")
print("Warning: <{}> is ({})".format(options.input,
os.path.isdir(options.input)))
print("")
sys.exit(1)
else:
msg("source: <{}> is dir {}".format(options.input,
os.path.isdir(options.input)))
# only load 'jpg' images
if options.jpg:
afiles = get_fn_jpg(options.input)
# only load 'png' images
elif options.png:
afiles = get_fn_png(options.input)
# load all the files, skip the ones we can't work with
else:
afiles = get_filenames(options.input)
msg("files {}".format(afiles))
# where do the processed files go?
dest_fp = ""
if options.output:
if os.path.isdir(options.output):
dest_fp = options.output
else:
print("Warning: Trying to save files to an invalid directory")
print(" I suggest manually creating the directory")
print("")
sys.exit(1)
else:
print("Error: Destination path must be supplied")
print("")
sys.exit(1)
# process the files
if process(afiles, dest_fp):
msg("destination: ({}) <{}>".format(len(afiles), afiles))
else:
print("Error: processing files has failed")
print("Warning: <{}> is ({})".format(dest_fp,
os.path.isdir(dest_fp)))
print("")
sys.exit(1)
