def retrieve_data(model_base_dir, model_dir, raw_data_dir, predict_first_dataset): if predict_first_dataset: dataset_ID = "1" model_ID = "2" else: dataset_ID = "2" model_ID = "1" print("loading data set {}".format(dataset_ID)) if predict_first_dataset: file_dataset = model_base_dir + "dataset1.h5" else: file_dataset = model_base_dir + "dataset2.h5" f = h5py.File(file_dataset.strip(), "r") sizes = f["sizes"][()] spacings = f["spacings"][()] origins = f["origins"][()] directions = f["directions"][()] img_infos = tools.deserialize_img_infos(sizes, spacings, origins, directions) images = f["images"][()] if predict_first_dataset: model_file = model_dir + "weights2.hdf5" else: model_file = model_dir + "weights1.hdf5" image_filename_templates = tools.get_filenames(predict_first_dataset, raw_data_dir) image_filenames = [] for f in image_filename_templates: new_name = f[:3] + "_prediction.mhd" # eg "P01_prediction.mhd" image_filenames.append(new_name) print("predict with model {}".format(model_ID)) return images, img_infos, image_filenames, model_file
def xy_resampler(num_classes, data_dir, isFirstDataSet): # first retrieve the directory and all ct files corresponding to the first/second data set subdir = data_dir + "rawdata/" files = get_filenames(isFirstDataSet, subdir) print("resampling in " + subdir) # now read and resample batch_in = [] img_infos = [] for f in files: print("resample ct " + f) ctimg = sitk.ReadImage(data_dir + "rawdata/" + f) for z in range(0, ctimg.GetDepth()): batch_in.append( np.expand_dims(sitk.GetArrayFromImage(ctimg[:, :, z]), axis=2)) img_infos.append(extract_image_info(ctimg)) # now retrieve the directory and all gt files corresponding to the first/second data set subdir = data_dir + "groundtruth/" files = get_filenames(isFirstDataSet, subdir) print("resampling in " + subdir) # now read and resample batch_out = [] for f in files: print("resample gt " + f) gtimg = sitk.ReadImage(data_dir + "groundtruth/" + f) for z in range(0, gtimg.GetDepth()): batch_out.append(sitk.GetArrayFromImage(gtimg[:, :, z])) batch_in = np.stack(batch_in) batch_out = np.stack(batch_out) print("size {}".format(batch_in.shape)) print("size {}".format(batch_out.shape)) batch_out = utils.to_categorical(batch_out, num_classes) return (batch_in, batch_out, img_infos)
def generate_outputfile(ra, dec, redshift, mass): print("Collecting filenames...") data, goods = tools.get_filenames(ra=ra, dec=dec, filter="any") ridx = (data[:,4] == 'r') gidx = (data[:,4] == 'g') iidx = (data[:,4] == 'i') yidx = (data[:,4] == 'y') zidx = (data[:,4] == 'z') rdata = data[ridx,:] base = rdata[:,:-2] rdata = rdata[:,-2:].astype(str) gdata = data[gidx,-2:].astype(str) idata = data[iidx,-2:].astype(str) ydata = data[yidx,-2:].astype(str) zdata = data[zidx,-2:].astype(str) rdata[:,1] = [x[:-1] for x in rdata[:,1]] gdata[:,1] = [x[:-1] for x in gdata[:,1]] idata[:,1] = [x[:-1] for x in idata[:,1]] zdata[:,1] = [x[:-1] for x in zdata[:,1]] ydata[:,1] = [x[:-1] for x in ydata[:,1]] ra = ra[goods] dec = dec[goods] redshift = redshift[goods] mass = mass[goods] ra = ra.reshape((ra.size, 1)) dec = dec.reshape((dec.size, 1)) redshift = redshift.reshape((redshift.size, 1)) mass = mass.reshape((mass.size, 1)) base = np.delete(base, [2,3,4,5], axis=1) outdata = np.hstack((ra, dec, redshift, mass, base, rdata, gdata, idata, ydata, zdata)) outdata = outdata.astype(str) print("Writing output-file") out = pd.DataFrame(outdata) out.to_csv(outfile, header=['RA', 'DEC', 'REDSHIFT', 'MASS', 'PRO', 'SKY', 'TYPE', 'RNAME', 'RSHORTNAME', 'GNAME', 'GSHORTNAME', 'INAME', 'ISHORTNAME', 'YNAME', 'YSHORTNAME', 'ZNAME', 'ZSHORTNAME'], index=False)
import numpy as np import matplotlib.pyplot as plt from tools import get_params, get_filenames, sort_key Tc = 2 / np.log(1 + np.sqrt(2)) lw = 3 c = ['indianred', 'orange', 'forestgreen', 'royalblue'] fig, ax = plt.subplots(figsize=(10, 8)) for filename in get_filenames('test_scalar'): b, L, K = get_params(filename) results = np.loadtxt(filename, unpack=True) temp = results[1] mse = results[6] i = int(np.log(L) / np.log(b) - 3) ax.semilogy(temp / Tc, mse, label="L=" + str(L), color=c[i], linewidth=lw) # sort legend handles, labels = ax.get_legend_handles_labels() labels, handles = zip(*sorted(zip(labels, handles), key=sort_key)) ax.legend(handles, labels) # fig params plt.ylabel('Mean Squared Error') plt.xlim(temp[0] / Tc, temp[-1] / Tc) plt.xlabel(r'$T/T_c$') plt.title('Testing MSE of Outputs From Large and Small Lattices')
def test_combined_models(model_date_xy, model_date_xz, model_date_yz): #This needs to be set according to your directory #Needs to contain /rawdata and /groundtruth with equally named mhd-files root_dir = "C:/users/jfauser/IPCAI2019/ModelData/" model_xy = "XY-Unet-Init" model_xz = "XZ-Unet-Init" model_yz = "YZ-Unet-Init" model_dir_xy = root_dir + model_xy + "/" + model_date_xy + "/predictions/" model_dir_xz = root_dir + model_xz + "/" + model_date_xz + "/predictions/" model_dir_yz = root_dir + model_yz + "/" + model_date_yz + "/predictions/" # ------------------------ majority vote ------------------------ useMajorityVoting = True if useMajorityVoting: model_name = "Combined-Unet-Majority" # time string as a changing last folder does not makes sense here out_dir = root_dir + model_name + "/last_result/predictions/" if not os.path.isdir(root_dir + model_name): os.mkdir(root_dir + model_name) if not os.path.isdir(root_dir + model_name + "/last_result/"): os.mkdir(root_dir + model_name + "/last_result/") if not os.path.isdir(out_dir): os.mkdir(out_dir) # get all results from all models files_xy = tools.get_filenames(True, model_dir_xy) files_xy.extend(tools.get_filenames(False, model_dir_xy)) files_xz = tools.get_filenames(True, model_dir_xz) files_xz.extend(tools.get_filenames(False, model_dir_xz)) files_yz = tools.get_filenames(True, model_dir_yz) files_yz.extend(tools.get_filenames(False, model_dir_yz)) # sort for ease of use files_xy.sort() files_xz.sort() files_yz.sort() for fx, fy, fz in zip(files_xy, files_xz, files_yz): print("voting on next file: {}".format(fx[:3])) img1, info = itk.read_itk_array(model_dir_xy + fx) size = info[0] img2, info = itk.read_itk_array(model_dir_xz + fy) img3, info = itk.read_itk_array(model_dir_yz + fz) img = np.zeros((size[2], size[1], size[0])) # majority vote equals = img1[:, :, :] == img2[:, :, :] img[equals] = img1[equals] equals = img1[:, :, :] == img3[:, :, :] img[equals] = img1[equals] equals = img3[:, :, :] == img2[:, :, :] img[equals] = img2[equals] # chorda vote equals = img1[:, :, :] == 5 img[equals] = img1[equals] equals = img1[:, :, :] == 5 img[equals] = img1[equals] equals = img3[:, :, :] == 5 img[equals] = img2[equals] img_out = itk.create_itk_image_2(img, info) itk.write_img(img_out, out_dir + fx[:3] + ".mhd")
def main(): """cli entry point""" parser = argparse.ArgumentParser(usage='%(prog)s [options]') #------ in/out ------ parser.add_argument("-i", "--input", dest="input", help="input source directory") parser.add_argument("-j", "--jpg", dest="jpg", action="store_true", help="process only jpg files") parser.add_argument("-d", "--description", dest="description", help="add a description to the the image") parser.add_argument("-e", "--title", dest="title", help="add a title to the the image") parser.add_argument("-t", "--tags", dest="tags", help="add tags to the the image") parser.add_argument("-v", '--version', dest='version', action="store_true", help="Version information") #--------- options --------- args = parser.parse_args() if not args: parser.print_help() sys.exit(1) #--------- version --------- if args.version: print("{} {}".format(sys.argv[0], '0.1')) sys.exit(1) #-=------- process --------- if args.input: # only load 'jpg' images print("path: <{}>".format(args.input)) if args.jpg: afiles = tools.get_fn_jpg(args.input) else: afiles = tools.get_filenames(args.input) print("AFILES <{}>".format(afiles)) print("directory: ({}) <{}>".format(len(afiles), afiles)) if len(afiles) > 1: # set tags OR defaults if args.tags: params['tags'] = args.tags else: tags = "" # set title OR nothing if args.title: params['title'] = args.title else: title = "" # set description OR nothing if args.description: params['description'] = args.description else: description = "" # authenticate ONCE, then upload # looped list of files flickr = login.authenticate() process_batch(flickr, afiles, params) flickr = None else: # load all files found fpn = os.path.join(os.curdir, args.input) fn = args.input # parameters params['filename'] = fpn params['title'] = filepath2title(fpn) params['description'] = args.description params['tags'] = args.tags # check filepathname if os.path.exists(fpn): print("fpn <{}>".format(fpn)) # process one file flickr = login.authenticate() process(flickr, params) flickr = None else: print("error: the source file is not found") print(" <{}>".format(fpn))
def xz_resampler(num_classes, data_dir, isFirstDataSet): # first retrieve the directory and all ct files corresponding to the first/second data set subdir = data_dir + "rawdata/" files = get_filenames(isFirstDataSet, subdir) print("num classes: {}".format(num_classes)) print("resampling in " + subdir) # now read and resample batch_in = [] img_infos = [] netdim_z = 128 for f in files: print("resample ct " + f) ctimg = sitk.ReadImage(data_dir + "rawdata/" + f) dim_x = ctimg.GetWidth() dim_y = ctimg.GetHeight() dim_z = ctimg.GetDepth() extract_labels = netdim_z > dim_z # network has more slices than original image slices = [] for y in range(0, dim_y): slices.append(sitk.GetArrayFromImage(ctimg[:, y, :])) extract_labels = netdim_z > dim_z # network has more slices than original image for slice in slices: new_slice = np.zeros((netdim_z, dim_x, 1)) if extract_labels: # original z-direction small than unet input? just take partial unet segmentation idx_start = int(0.5 * (netdim_z - dim_z)) new_slice[idx_start:idx_start + dim_z, :, 0] = slice else: # original z-direction is larger than unet input? put in segmentation idx_start = int( max(0, 0.5 * dim_z - 74) ) # jugular vein is slightly lower. so do not pick the middle new_slice[:, :, 0] = slice[idx_start:idx_start + netdim_z, :] batch_in.append(new_slice) img_infos.append(extract_image_info(ctimg)) batch_in = np.stack(batch_in) # now retrieve the directory and all gt files corresponding to the first/second data set subdir = data_dir + "groundtruth/" files = get_filenames(isFirstDataSet, subdir) print("resampling in " + subdir) # now read and resample batch_out = [] for f in files: print("resample gt " + f) gtimg = sitk.ReadImage(data_dir + "groundtruth/" + f) dim_x = gtimg.GetWidth() dim_y = gtimg.GetHeight() dim_z = gtimg.GetDepth() extract_labels = netdim_z > dim_z # network has more slices than original image slices = [] for y in range(0, dim_y): slices.append(sitk.GetArrayFromImage(gtimg[:, y, :])) for slice in slices: new_slice = np.zeros((netdim_z, dim_x)) if extract_labels: # original z-direction small than unet input? just take partial unet segmentation idx_start = int(0.5 * (netdim_z - dim_z)) new_slice[idx_start:idx_start + dim_z, :] = slice else: # original z-direction is larger than unet input? put in segmentation idx_start = int( max(0, 0.5 * dim_z - 74) ) # jugular vein is slightly lower. so do not pick the middle new_slice[:, :] = slice[idx_start:idx_start + netdim_z, :] batch_out.append(new_slice) batch_out = np.stack(batch_out) print("size {}".format(batch_in.shape)) print("size {}".format(batch_out.shape)) batch_out = utils.to_categorical(batch_out, num_classes) return (batch_in, batch_out, img_infos)
def main(): """cli entry point""" usage = "usage: %prog -i -o [-s]" parser = OptionParser(usage) #------ in/out ------ parser.add_option("-i", "--input", dest="input", help="input source directory") parser.add_option("-o", "--output", dest="output", help="output source directory"), parser.add_option("-j", "--jpg", dest="jpg", action="store_true", help="process only jpg files") parser.add_option("-p", "--png", dest="png", action="store_true", help="process only png files") #------ options ------ options, args = parser.parse_args() #------ process ------ if options.input: # input directory must exist if not os.path.isdir(options.input): print("Error: processing input files has failed") print("Warning: <{}> is ({})".format(options.input, os.path.isdir(options.input))) print("") sys.exit(1) else: msg("source: <{}> is dir {}".format(options.input, os.path.isdir(options.input))) # only load 'jpg' images if options.jpg: afiles = get_fn_jpg(options.input) # only load 'png' images elif options.png: afiles = get_fn_png(options.input) # load all the files, skip the ones we can't work with else: afiles = get_filenames(options.input) msg("files {}".format(afiles)) # where do the processed files go? dest_fp = "" if options.output: if os.path.isdir(options.output): dest_fp = options.output else: print("Warning: Trying to save files to an invalid directory") print(" I suggest manually creating the directory") print("") sys.exit(1) else: print("Error: Destination path must be supplied") print("") sys.exit(1) # process the files if process(afiles, dest_fp): msg("destination: ({}) <{}>".format(len(afiles), afiles)) else: print("Error: processing files has failed") print("Warning: <{}> is ({})".format(dest_fp, os.path.isdir(dest_fp))) print("") sys.exit(1)