def _process(lock, int_from, int_to, infile, outfile, outshape, outtype, method, plan, logfilename): # Process the required subset of images: for i in range(int_from, int_to + 1): # Read input image: t0 = time() f_in = getHDF5(infile, 'r') if "/tomo" in f_in: dset = f_in['tomo'] else: dset = f_in['exchange/data'] im = tdf.read_tomo(dset,i).astype(float32) f_in.close() t1 = time() # Perform phase retrieval (first time also PyFFTW prepares a plan): if (method == 0): im = tiehom(im, plan).astype(float32) else: im = phrt(im, plan, method).astype(float32) t2 = time() # Save processed image to HDF5 file (atomic procedure - lock used): _write_data(lock, im, i, outfile, outshape, outtype, logfilename, t2 - t1, t1 - t0)
def _process (lock, int_from, int_to, infile, outfile, outshape, outtype, skipflat, plan, norm_sx, norm_dx, flat_end, half_half, half_half_line, ext_fov, ext_fov_rot_right, ext_fov_overlap, ext_fov_normalize, ext_fov_average, ringrem, dynamic_ff, EFF, filtEFF, im_dark, logfilename): # Process the required subset of images: for i in range(int_from, int_to + 1): # Read input image: t0 = time() f_in = getHDF5(infile, 'r') if "/tomo" in f_in: dset = f_in['tomo'] else: dset = f_in['exchange/data'] im = tdf.read_sino(dset,i).astype(float32) f_in.close() t1 = time() # Perform pre-processing (flat fielding, extended FOV, ring removal): if not skipflat: if dynamic_ff: # Dynamic flat fielding with downsampling = 2: im = dynamic_flat_fielding(im, i, EFF, filtEFF, 2, im_dark, norm_sx, norm_dx) else: im = flat_fielding(im, i, plan, flat_end, half_half, half_half_line, norm_sx, norm_dx) if ext_fov: im = extfov_correction(im, ext_fov_rot_right, ext_fov_overlap, ext_fov_normalize, ext_fov_average) if not skipflat and not dynamic_ff: im = ring_correction (im, ringrem, flat_end, plan['skip_flat_after'], half_half, half_half_line, ext_fov) else: im = ring_correction (im, ringrem, False, False, half_half, half_half_line, ext_fov) t2 = time() # Save processed image to HDF5 file (atomic procedure - lock used): _write_data(lock, im, i, outfile, outshape, outtype, logfilename, t2 - t1, t1 - t0)
def _process(lock, int_from, int_to, infile, outfile, outshape, outtype, method, plan, logfilename): # Process the required subset of images: for i in range(int_from, int_to + 1): # Read input image: t0 = time() f_in = getHDF5(infile, 'r') if "/tomo" in f_in: dset = f_in['tomo'] else: dset = f_in['exchange/data'] im = tdf.read_tomo(dset, i).astype(float32) f_in.close() t1 = time() # Perform phase retrieval (first time also PyFFTW prepares a plan): if (method == 0): im = tiehom(im, plan).astype(float32) elif (method == 1): im = tiehom2020(im, plan).astype(float32) else: im = phrt(im, plan, method).astype(float32) t2 = time() # Save processed image to HDF5 file (atomic procedure - lock used): _write_data(lock, im, i, outfile, outshape, outtype, logfilename, t2 - t1, t1 - t0)
def _write_data(lock, im, index, outfile, outshape, outtype, rescale_factor, logfilename, cputime, itime): lock.acquire() try: t0 = time() f_out = getHDF5(outfile, 'a') f_out_dset = f_out.require_dataset('exchange/data', outshape, outtype, chunks=tdf.get_dset_chunks( outshape[0])) im = im * rescale_factor tdf.write_tomo(f_out_dset, index, im.astype(outtype)) # Set minimum and maximum: if (amin(im[:]) < float(f_out_dset.attrs['min'])): f_out_dset.attrs['min'] = str(amin(im[:])) if (amax(im[:]) > float(f_out_dset.attrs['max'])): f_out_dset.attrs['max'] = str(amax(im[:])) f_out.close() t1 = time() # Print out execution time: log = open(logfilename, "a") log.write(linesep + "\ttomo_%s processed (CPU: %0.3f sec - I/O: %0.3f sec)." % (str(index).zfill(4), cputime, t1 - t0 + itime)) log.close() finally: lock.release()
def _write_data(lock, im, index, outfile, outshape, outtype, rescale_factor, logfilename, cputime, itime): lock.acquire() try: t0 = time() f_out = getHDF5(outfile, 'a') f_out_dset = f_out.require_dataset('exchange/data', outshape, outtype, chunks=tdf.get_dset_chunks(outshape[0])) im = im * rescale_factor tdf.write_tomo(f_out_dset,index,im.astype(outtype)) # Set minimum and maximum: if (amin(im[:]) < float(f_out_dset.attrs['min'])): f_out_dset.attrs['min'] = str(amin(im[:])) if (amax(im[:]) > float(f_out_dset.attrs['max'])): f_out_dset.attrs['max'] = str(amax(im[:])) f_out.close() t1 = time() # Print out execution time: log = open(logfilename,"a") log.write(linesep + "\ttomo_%s processed (CPU: %0.3f sec - I/O: %0.3f sec)." % (str(index).zfill(4), cputime, t1 - t0 + itime)) log.close() finally: lock.release()
def _process(lock, int_from, int_to, infile, outfile, outshape, outtype, skipflat, plan, flat_end, half_half, half_half_line, dynamic_ff, EFF, filtEFF, im_dark, rotation, interp, border, rescale_factor, logfilename): # Process the required subset of images: for i in range(int_from, int_to + 1): # Read input image: t0 = time() f_in = getHDF5(infile, 'r') if "/tomo" in f_in: dset = f_in['tomo'] else: dset = f_in['exchange/data'] im = tdf.read_tomo(dset,i).astype(float32) f_in.close() t1 = time() # Perform pre-processing (flat fielding, extended FOV, ring removal): if not skipflat: if dynamic_ff: # Dynamic flat fielding with downsampling = 2: im = dynamic_flat_fielding(im, EFF, filtEFF, 2, im_dark) else: im = flat_fielding(im, i, plan, flat_end, half_half, half_half_line, norm_sx, norm_dx) t2 = time() # Rotate: rows, cols = im.shape #if (cols > rows): # angle = - arctan(2.0 / cols) * (rows / 2.0) / 2.0 #else: # angle = - arctan(2.0 / rows) * (cols / 2.0) / 2.0 #print(angle) M = cv2.getRotationMatrix2D((cols / 2, rows / 2), rotation, 1) if interp == 'nearest': interpflag = cv2.INTER_NEAREST elif interp == 'cubic': interpflag = cv2.INTER_CUBIC elif interp == 'lanczos': interpflag = cv2.INTER_LANCZOS4 else: interpflag = cv2.INTER_LINEAR if border == 'constant': borderflag = cv2.BORDER_CONSTANT else: borderflag = cv2.BORDER_REPLICATE im = cv2.warpAffine(im, M, (cols, rows), flags = interpflag, borderMode = borderflag) # Save processed image to HDF5 file (atomic procedure - lock used): _write_data(lock, im, i, outfile, outshape, outtype, rescale_factor, logfilename, t2 - t1, t1 - t0)
def _write_data(lock, im, index, offset, abs_offset, imfilename, timestamp, projorder, tot_files, provenance_dt, outfile, dsetname, outshape, outtype, logfilename, itime, n_images): """To do... """ lock.acquire() try: # Open the HDF5 file to be populated with projections (or sinograms): t0 = time.time() f_out = getHDF5(outfile, 'a') f_out_dset = f_out.require_dataset(dsetname, outshape, outtype, chunks=tdf.get_dset_chunks(outshape[0])) # Write the projection file or sinogram file: if projorder: tdf.write_tomo(f_out_dset, index - abs_offset, im) else: tdf.write_sino(f_out_dset, index - abs_offset, im) # Set minimum and maximum (without Infs and NaNs): tmp = im[:].astype(numpy.float32) tmp = tmp[numpy.nonzero(numpy.isfinite(tmp))] if (numpy.amin(tmp[:]) < float(f_out_dset.attrs['min'])): f_out_dset.attrs['min'] = str(numpy.amin(tmp[:])) if (numpy.amax(tmp[:]) > float(f_out_dset.attrs['max'])): f_out_dset.attrs['max'] = str(numpy.amax(tmp[:])) f_out_dset.attrs['avg'] = str(float(f_out_dset.attrs['avg']) + numpy.mean(tmp[:])/(1.0*n_images) ) # Save provenance metadata: provenance_dset = f_out.require_dataset('provenance/detector_output', (tot_files,), dtype=provenance_dt) provenance_dset["filename", offset - abs_offset + index] = numpy.string_(os.path.basename(imfilename)) provenance_dset["timestamp", offset - abs_offset + index] = numpy.string_(datetime.datetime.fromtimestamp(timestamp).strftime('%Y-%m-%d %H:%M:%S.%f')[:-3]) # Close the HDF5: f_out.close() t1 = time.time() # Print out execution time: log = open(logfilename,"a") log.write(os.linesep + "\t%s processed (I: %0.3f sec - O: %0.3f sec)." % (os.path.basename(imfilename), itime, t1 - t0)) log.close() except: io_warning = True # Print out execution time: log = open(logfilename,"a") log.write(os.linesep + "\tError when writing to TDF %s. File skipped." % (os.path.basename(imfilename))) log.close() pass finally: lock.release()
def _write_data(lock, im, index, offset, abs_offset, imfilename, newfilename, timestamp, projorder, tot_files, provenance_dt, outfile, dsetname, outshape, outtype, logfilename, itime): """To do... """ lock.acquire() try: # Open the HDF5 file to be populated with projections (or sinograms): t0 = time.time() f_out = getHDF5(outfile, 'a') f_out_dset = f_out.require_dataset(dsetname, outshape, outtype, chunks=tdf.get_dset_chunks( outshape[0])) # Write the projection file or sinogram file: if projorder: tdf.write_tomo(f_out_dset, index - abs_offset, im) else: tdf.write_sino(f_out_dset, index - abs_offset, im) # Set minimum and maximum: tmp = im[:].astype(numpy.float32) tmp = tmp[numpy.nonzero(numpy.isfinite(tmp))] if (numpy.amin(tmp[:]) < float(f_out_dset.attrs['min'])): f_out_dset.attrs['min'] = str(numpy.amin(tmp[:])) if (numpy.amax(tmp[:]) > float(f_out_dset.attrs['max'])): f_out_dset.attrs['max'] = str(numpy.amax(tmp[:])) # Save provenance metadata: provenance_dset = f_out.require_dataset('provenance/detector_output', (tot_files, ), dtype=provenance_dt) provenance_dset["filename", offset - abs_offset + index] = newfilename provenance_dset["timestamp", offset - abs_offset + index] = numpy.string_( datetime.datetime.fromtimestamp(timestamp). strftime('%Y-%m-%d %H:%M:%S.%f')[:-3]) # Close the HDF5: f_out.close() t1 = time.time() # Print out execution time: log = open(logfilename, "a") log.write(os.linesep + "\t%s processed (I: %0.3f sec - O: %0.3f sec)." % (os.path.basename(imfilename), itime, t1 - t0)) log.close() finally: lock.release()
def _write_data(im, index, outfile, outshape, outtype): f_out = getHDF5(outfile, 'a') f_out_dset = f_out.require_dataset('exchange/data', outshape, outtype, chunks=tdf.get_dset_chunks(outshape[0])) tdf.write_sino(f_out_dset, index, im.astype(outtype)) # Set minimum and maximum: if (amin(im[:]) < float(f_out_dset.attrs['min'])): f_out_dset.attrs['min'] = str(amin(im[:])) if (amax(im[:]) > float(f_out_dset.attrs['max'])): f_out_dset.attrs['max'] = str(amax(im[:])) f_out.close()
def _process(lock, int_from, int_to, infile, outfile, outshape, outtype, skipflat, plan, norm_sx, norm_dx, flat_end, half_half, half_half_line, ext_fov, ext_fov_rot_right, ext_fov_overlap, ext_fov_normalize, ext_fov_average, ringrem, dynamic_ff, EFF, filtEFF, im_dark, logfilename): # Process the required subset of images: for i in range(int_from, int_to + 1): # Read input image: t0 = time() f_in = getHDF5(infile, 'r') if "/tomo" in f_in: dset = f_in['tomo'] else: dset = f_in['exchange/data'] im = tdf.read_sino(dset, i).astype(float32) f_in.close() t1 = time() # Perform pre-processing (flat fielding, extended FOV, ring removal): if not skipflat: if dynamic_ff: # Dynamic flat fielding with downsampling = 2: im = dynamic_flat_fielding(im, i, EFF, filtEFF, 2, im_dark, norm_sx, norm_dx) else: im = flat_fielding(im, i, plan, flat_end, half_half, half_half_line, norm_sx, norm_dx) if ext_fov: im = extfov_correction(im, ext_fov_rot_right, ext_fov_overlap, ext_fov_normalize, ext_fov_average) if not skipflat and not dynamic_ff: im = ring_correction(im, ringrem, flat_end, plan['skip_flat_after'], half_half, half_half_line, ext_fov) else: im = ring_correction(im, ringrem, False, False, half_half, half_half_line, ext_fov) t2 = time() # Save processed image to HDF5 file (atomic procedure - lock used): _write_data(lock, im, i, outfile, outshape, outtype, logfilename, t2 - t1, t1 - t0)
def _process(lock, int_from, int_to, infile, dset_str, TIFFFormat, projorder, outpath, outprefix, logfilename): """To do... """ try: f = getHDF5(infile, 'r') dset = f[dset_str] # Process the required subset of images: for i in range(int_from, int_to + 1): # Read input image: t0 = time.time() if projorder: im = tdf.read_tomo(dset, i) else: im = tdf.read_sino(dset, i) # Cast type (if required but it should never occur): if (((im.dtype).type is float64) or ((im.dtype).type is float16)): im = im.astype(float32, copy=False) if (TIFFFormat): fname = outpath + outprefix + '_' + str(i).zfill(4) + '.tif' imsave(fname, im) else: fname = outpath + outprefix + '_' + str(i).zfill(4) + '_' + str(im.shape[1]) + \ 'x' + str(im.shape[0]) + '_' + str(im.dtype) + '.raw' im.tofile(fname) t1 = time.time() # Print out execution time: _write_log(lock, fname, logfilename, t1 - t0) f.close() except Exception: pass
def _write_data(lock, im, index, offset, abs_offset, imfilename, timestamp, projorder, tot_files, provenance_dt, outfile, dsetname, outshape, outtype, logfilename, itime): """To do... """ lock.acquire() try: # Open the HDF5 file to be populated with projections (or sinograms): t0 = time.time() f_out = getHDF5( outfile, 'a' ) f_out_dset = f_out.require_dataset(dsetname, outshape, outtype, chunks=tdf.get_dset_chunks(outshape[0])) # Write the projection file or sinogram file: if projorder: tdf.write_tomo(f_out_dset, index - abs_offset, im) else: tdf.write_sino(f_out_dset, index - abs_offset, im) # Set minimum and maximum: if ( numpy.amin(im[:]) < float(f_out_dset.attrs['min']) ): f_out_dset.attrs['min'] = str(numpy.amin(im[:])) if ( numpy.amax(im[:]) > float(f_out_dset.attrs['max'])): f_out_dset.attrs['max'] = str(numpy.amax(im[:])) # Save provenance metadata: provenance_dset = f_out.require_dataset('provenance/detector_output', (tot_files,), dtype=provenance_dt) provenance_dset["filename", offset - abs_offset + index] = numpy.string_(os.path.basename(imfilename)) provenance_dset["timestamp", offset - abs_offset + index] = numpy.string_( datetime.datetime.fromtimestamp(timestamp).strftime('%Y-%m-%d %H:%M:%S.%f')[:-3]) # Close the HDF5: f_out.close() t1 = time.time() # Print out execution time: log = open(logfilename,"a") log.write(os.linesep + "\t%s processed (I: %0.3f sec - O: %0.3f sec)." % (os.path.basename(imfilename), itime, t1 - t0)) log.close() finally: lock.release()
def main(argv): """ Print dimensions of HDF5 file """ # # Get the parameters: # infile = argv[0] f = getHDF5(infile, 'r') if "/tomo" in f: dset = f['tomo'] else: dset = f['exchange/data'] print("Projections: " + tdf.get_nr_projs(dset)) print("Slices: " + tdf.get_nr_sinos(dset)) print("DetectorSize: " + tdf.get_det_size(dset))
def main(argv): """To do... Usage ----- Parameters --------- Example -------------------------- The following line processes the first ten TIFF files of input path "/home/in" and saves the processed files to "/home/out" with the application of the Boin and Haibel filter with smoothing via a Butterworth filter of order 4 and cutoff frequency 0.01: destripe /home/in /home/out 1 10 1 0.01 4 """ lock = Lock() rescale_factor = 10000.0 # For 16-bit floating point # Get the from and to number of files to process: int_from = int(argv[0]) int_to = int(argv[1]) # Get paths: infile = argv[2] outfile = argv[3] # Params for flat fielding with post flats/darks: flat_end = True if argv[4] == "True" else False half_half = True if argv[5] == "True" else False half_half_line = int(argv[6]) # Flat fielding method (conventional or dynamic): dynamic_ff = True if argv[7] == "True" else False # Parameters for rotation: rotation = float(argv[8]) interp = argv[9] border = argv[10] # Nr of threads and log file: nr_threads = int(argv[11]) logfilename = argv[12] # Log input parameters: log = open(logfilename,"w") log.write(linesep + "\tInput TDF file: %s" % (infile)) log.write(linesep + "\tOutput TDF file: %s" % (outfile)) log.write(linesep + "\t--------------") log.write(linesep + "\tOpening input dataset...") log.close() # Remove a previous copy of output: if exists(outfile): remove(outfile) # Open the HDF5 file: f_in = getHDF5(infile, 'r') if "/tomo" in f_in: dset = f_in['tomo'] tomoprefix = 'tomo' flatprefix = 'flat' darkprefix = 'dark' else: dset = f_in['exchange/data'] if "/provenance/detector_output" in f_in: prov_dset = f_in['provenance/detector_output'] tomoprefix = prov_dset.attrs['tomo_prefix'] flatprefix = prov_dset.attrs['flat_prefix'] darkprefix = prov_dset.attrs['dark_prefix'] num_proj = tdf.get_nr_projs(dset) num_sinos = tdf.get_nr_sinos(dset) if (num_sinos == 0): log = open(logfilename,"a") log.write(linesep + "\tNo projections found. Process will end.") log.close() exit() # Check extrema (int_to == -1 means all files): if ((int_to >= num_proj) or (int_to == -1)): int_to = num_proj - 1 # Prepare the work plan for flat and dark images: log = open(logfilename,"a") log.write(linesep + "\t--------------") log.write(linesep + "\tPreparing the work plan...") log.close() # Extract flat and darks: skipflat = False skipdark = False # Following variables make sense only for dynamic flat fielding: EFF = -1 filtEFF = -1 im_dark = -1 # Following variable makes sense only for conventional flat fielding: plan = -1 if not dynamic_ff: plan = extract_flatdark(f_in, flat_end, logfilename) if (isscalar(plan['im_flat']) and isscalar(plan['im_flat_after'])): skipflat = True else: skipflat = False else: # Dynamic flat fielding: if "/tomo" in f_in: if "/flat" in f_in: flat_dset = f_in['flat'] if "/dark" in f_in: im_dark = _medianize(f_in['dark']) else: skipdark = True else: skipflat = True # Nothing to do in this case else: if "/exchange/data_white" in f_in: flat_dset = f_in['/exchange/data_white'] if "/exchange/data_dark" in f_in: im_dark = _medianize(f_in['/exchange/data_dark']) else: skipdark = True else: skipflat = True # Nothing to do in this case # Prepare plan for dynamic flat fielding with 16 repetitions: if not skipflat: EFF, filtEFF = dff_prepare_plan(flat_dset, 16, im_dark) # Get the corrected outshape (in this case it's easy): im = tdf.read_tomo(dset,0).astype(float32) outshape = tdf.get_dset_shape(im.shape[1], im.shape[0], num_proj) # Create the output HDF5 file: f_out = getHDF5(outfile, 'w') #f_out_dset = f_out.create_dataset('exchange/data', outshape, im.dtype) f_out_dset = f_out.create_dataset('exchange/data', outshape, float16) f_out_dset.attrs['min'] = str(amin(im[:])) f_out_dset.attrs['max'] = str(amax(im[:])) f_out_dset.attrs['version'] = '1.0' f_out_dset.attrs['axes'] = "y:theta:x" f_out_dset.attrs['rescale_factor'] = str(rescale_factor) f_out.close() f_in.close() # Log infos: log = open(logfilename,"a") log.write(linesep + "\tWork plan prepared correctly.") log.write(linesep + "\t--------------") log.write(linesep + "\tPerforming pre processing...") log.close() # Run several threads for independent computation without waiting for threads # completion: for num in range(nr_threads): start = (num_proj / nr_threads) * num if (num == nr_threads - 1): end = num_proj - 1 else: end = (num_proj / nr_threads) * (num + 1) - 1 Process(target=_process, args=(lock, start, end, infile, outfile, outshape, float16, skipflat, plan, flat_end, half_half, half_half_line, dynamic_ff, EFF, filtEFF, im_dark, rotation, interp, border, rescale_factor, logfilename)).start()
def main(argv): """Extract a 2D image (projection or sinogram) from the input TDF file (DataExchange HDF5) and creates a 32-bit RAW file to disk. Parameters ---------- argv[0] : string The absolute path of the input TDF. argv[1] : int The relative position of the image within the dataset. argv[2] : string One of the following options: 'tomo', 'sino', 'flat', 'dark'. argv[3] : string The absolute path of the output 32-bit RAW image file. Filename will be modified by adding image width, image height, minimum and maximum value of the input TDF dataset. Example ------- tools_extractdata "S:\\dataset.tdf" 128 tomo "R:\\proj" """ try: # # Get input parameters: # infile = argv[0] index = int(argv[1]) imtype = argv[2] outfile = argv[3] # # Body # # Check if file exists: if not os.path.exists(infile): #log = open(logfilename,"a") #log.write(os.linesep + "\tError: input TDF file not found. Process will end.") #log.close() exit() # Open the HDF5 file: f = getHDF5( infile, 'r' ) if (imtype == 'sino'): if "/tomo" in f: dset = f['tomo'] else: dset = f['exchange/data'] im = tdf.read_sino( dset, index ) elif (imtype == 'dark'): if "/dark" in f: dset = f['dark'] else: dset = f['exchange/data_dark'] im = tdf.read_tomo( dset, index ) elif (imtype == 'flat'): if "/flat" in f: dset = f['flat'] else: dset = f['exchange/data_white'] im = tdf.read_tomo( dset, index ) else: if "/tomo" in f: dset = f['tomo'] else: dset = f['exchange/data'] im = tdf.read_tomo( dset, index ) min = float(numpy.nanmin(im[:])) max = float(numpy.nanmax(im[:])) # Get global attributes (if any): try: if ('version' in f.attrs): if (f.attrs['version'] == '1.0'): min = float(dset_tomo.attrs['min']) max = float(dset_tomo.attrs['max']) except: pass f.close() # Cast type: im = im.astype(float32) # Modify file name: outfile = outfile + '_' + str(im.shape[1]) + 'x' + str(im.shape[0]) + '_' + str(min) + '$' + str(max) # Write RAW data to disk: im.tofile(outfile) except: exit()
def main(argv): """ Converts a TDF file (HDF5 Tomo Data Format) into a sequence of TIFF (uncompressed) files. Parameters ---------- from : scalar, integer among all the projections (or sinogram) data, a subset of the volume can be specified, ranging from the parameter "from" to the parameter "to" (see next). In most cases, this parameter is 0. to : scalar, integer among all the projections (or sinogram) data, a subset of the volume can be specified, ranging from the parameter "from" (see previous parameter) to the parameter "to". If the value -1 is specified, all the projection (or sinogram) data will be considered. in_file : string path with filename of the TDF to read from (e.g. "Z:\\sample1.tdf"). out_path : string path that will contain the sequence of TIFF files (e.g. "Z:\\sample1\\tomo\\"). WARNING: the program does NOT automatically create non-existing folders and subfolders specified in the path. Moreover, if files with the same name already exist they will be automatically overwritten. file_prefix : string string to be assumed as the filename prefix of the TIFF files to create for the projection (or sinogram) data. E.g. "tomo" will create files having name "tomo_0001.tif", "tomo_0002.tif". flat_prefix : string string to be assumed as the filename prefix of the TIFF files to create for the flat (white field) data. E.g. "flat" will create files having name "flat_1.tif", "flat_2.tif". If dark or flat data have to be skipped the string "-" can be specified. dark_prefix : string string to be assumed as the filename prefix of the TIFF files to create for the dark (dark field) data. E.g. "dark" will create files having name "dark_1.tif", "dark_2.tif". If dark or flat data have to be skipped the string "-" can be specified. projection_order : boolean string specify the string "True" to create TIFF files for projections (the most common case), "False" for sinograms. TIFF_format : boolean string specify the string "True" to create TIFF files, "False" for RAW files. nr_threads : int number of multiple threads (actually processes) to consider to speed up the whole conversion process. log_file : string path with filename of a log file (e.g. "R:\\log.txt") where info about the conversion is reported. Returns ------- no return value Example ------- Example call to convert all the projections data to a sequence of tomo*.tif files: python tdf2tiff.py 0 -1 "C:\Temp\wet12T4part2.tdf" "C:\Temp\tomo" tomo flat dark True True 3 "C:\Temp\log.txt" Requirements ------- - Python 2.7 with the latest NumPy, SciPy, H5Py. - TIFFFile from C. Gohlke - tdf.py Tests ------- Tested with WinPython-64bit-2.7.6.3 (Windows) and Anaconda 2.1.0 (Linux 64-bit). """ lock = Lock() # To be used without flat fielding (just conversion): first_done = False # Get the from and to number of files to process: int_from = int(argv[0]) int_to = int(argv[1]) # -1 means "all files" # Get paths: infile = argv[2] outpath = argv[3] fileprefix = argv[4] flatprefix = argv[5] darkprefix = argv[6] if (flatprefix == "-"): skipflat = True else: skipflat = False if (darkprefix == "-"): skipdark = True else: skipdark = False if (fileprefix == "-"): skiptomo = True else: skiptomo = False projorder = argv[7] if projorder == "True": projorder = True else: projorder = False TIFFFormat = argv[8] if TIFFFormat == "True": TIFFFormat = True else: TIFFFormat = False nr_threads = int(argv[9]) logfilename = argv[10] # Check prefixes and path: if not outpath.endswith(os.path.sep): outpath += os.path.sep # Init variables: num_files = 0 num_flats = 0 num_darks = 0 # Get the files in infile: log = open(logfilename,"w") log.write(os.linesep + "\tInput TDF: %s" % (infile)) if (TIFFFormat): log.write(os.linesep + "\tOutput path where TIFF files will be created: %s" % (outpath)) else: log.write(os.linesep + "\tOutput path where RAW files will be created: %s" % (outpath)) log.write(os.linesep + "\t--------------") log.write(os.linesep + "\tFile output prefix: %s" % (fileprefix)) log.write(os.linesep + "\tFlat images output prefix: %s" % (flatprefix)) log.write(os.linesep + "\tDark images output prefix: %s" % (darkprefix)) log.write(os.linesep + "\t--------------") if (not (skiptomo)): if (int_to != -1): log.write(os.linesep + "\tThe subset [%d,%d] of the data will be considered." % (int_from, int_to)) if (projorder): log.write(os.linesep + "\tProjection order assumed.") else: log.write(os.linesep + "\tSinogram order assumed.") log.write(os.linesep + "\t--------------") log.close() if not os.path.exists(infile): log = open(logfilename,"a") log.write(os.linesep + "\tError: input TDF file not found. Process will end.") log.close() exit() # Open the HDF5 file: f = getHDF5(infile, 'r') oldTDF = False try: dset = f['tomo'] oldTDF = True except Exception: pass if not oldTDF: #try: dset = f['exchange/data'] #except Exception: # log = open(logfilename,"a") # log.write(os.linesep + "\tError: invalid TDF format. Process will end.") # log.close() # exit() if projorder: num_files = tdf.get_nr_projs(dset) else: num_files = tdf.get_nr_sinos(dset) f.close() # Get attributes: try: f = getHDF5(infile, 'r') if ('version' in f.attrs) and (f.attrs['version'] == 'TDF 1.0'): log = open(logfilename,"a") log.write(os.linesep + "\tTDF version 1.0 found.") log.write(os.linesep + "\t--------------") log.close() f.close() except: log = open(logfilename,"a") log.write(os.linesep + "\tWarning: TDF version unknown. Some features will not be available.") log.write(os.linesep + "\t--------------") log.close() # Check extrema (int_to == -1 means all files): if ((int_to >= num_files) or (int_to == -1)): int_to = num_files - 1 # Spawn the process for the conversion of flat images: if not skipflat: f = getHDF5(infile, 'r') if oldTDF: dset_str = 'flat' else: dset_str = 'exchange/data_white' num_flats = tdf.get_nr_projs(f[dset_str]) f.close() if (num_flats > 0): Process(target=_process, args=(lock, 0, num_flats - 1, infile, dset_str, TIFFFormat, True, outpath, flatprefix, logfilename)).start() #_process(lock, 0, num_flats - 1, infile, dset_str, TIFFFormat, projorder, #outpath, flatprefix, logfilename) # Spawn the process for the conversion of dark images: if not skipdark: f = getHDF5(infile, 'r') if oldTDF: dset_str = 'dark' else: dset_str = 'exchange/data_dark' num_darks = tdf.get_nr_projs(f[dset_str]) f.close() if (num_darks > 0): Process(target=_process, args=(lock, 0, num_darks - 1, infile, dset_str, TIFFFormat, True, outpath, darkprefix, logfilename)).start() #_process(lock, 0, num_darks - 1, infile, dset_str, TIFFFormat, projorder, #outpath, darkprefix, logfilename) # Spawn the processes for the conversion of projection or sinogram images: if not skiptomo: if oldTDF: dset_str = 'tomo' else: dset_str = 'exchange/data' # Start the process for the conversion of the projections (or sinograms) in a # multi-threaded way: for num in range(nr_threads): start = ((int_to - int_from + 1) / nr_threads) * num + int_from if (num == nr_threads - 1): end = int_to else: end = ((int_to - int_from + 1) / nr_threads) * (num + 1) + int_from - 1 Process(target=_process, args=(lock, start, end, infile, dset_str, TIFFFormat, projorder, outpath, fileprefix, logfilename)).start()
def main(argv): """Extract a 2D image (projection or sinogram) from the input TDF file (DataExchange HDF5) and creates a 32-bit RAW file to disk. Parameters ---------- argv[0] : string The absolute path of the input TDF. argv[1] : int The relative position of the image within the dataset. argv[2] : string One of the following options: 'tomo', 'sino', 'flat', 'dark'. argv[3] : string The absolute path of the output 32-bit RAW image file. Filename will be modified by adding image width, image height, minimum and maximum value of the input TDF dataset. Example ------- tools_extractdata "S:\\dataset.tdf" 128 tomo "R:\\proj" """ try: # # Get input parameters: # infile = argv[0] index = int(argv[1]) imtype = argv[2] outfile = argv[3] # # Body # # Check if file exists: if not os.path.exists(infile): #log = open(logfilename,"a") #log.write(os.linesep + "\tError: input TDF file not found. Process will end.") #log.close() exit() # Open the HDF5 file: f = getHDF5(infile, 'r') if (imtype == 'sino'): if "/tomo" in f: dset = f['tomo'] else: dset = f['exchange/data'] im = tdf.read_sino(dset, index) elif (imtype == 'dark'): if "/dark" in f: dset = f['dark'] else: dset = f['exchange/data_dark'] im = tdf.read_tomo(dset, index) elif (imtype == 'flat'): if "/flat" in f: dset = f['flat'] else: dset = f['exchange/data_white'] im = tdf.read_tomo(dset, index) else: if "/tomo" in f: dset = f['tomo'] else: dset = f['exchange/data'] im = tdf.read_tomo(dset, index) # Remove Infs e NaNs tmp = im[:].astype(numpy.float32) tmp = tmp[numpy.nonzero(numpy.isfinite(tmp))] # Sort the gray levels: tmp = numpy.sort(tmp) # Return as minimum the value the skip 0.30% of "black" tail and 0.05% of "white" tail: low_idx = int(tmp.shape[0] * 0.0030) high_idx = int(tmp.shape[0] * 0.9995) min = tmp[low_idx] max = tmp[high_idx] # Modify file name: outfile = outfile + '_' + str(im.shape[1]) + 'x' + str( im.shape[0]) + '_' + str(min) + '$' + str(max) # Cast type: im = im.astype(float32) # Write RAW data to disk: im.tofile(outfile) except: exit()
def process(sino_idx, num_sinos, infile, outpath, preprocessing_required, corr_plan, norm_sx, norm_dx, flat_end, half_half, half_half_line, ext_fov, ext_fov_rot_right, ext_fov_overlap, ext_fov_normalize, ext_fov_average, ringrem, phaseretrieval_required, phrtmethod, phrt_param1, phrt_param2, energy, distance, pixsize, phrtpad, approx_win, angles, offset, logtransform, param1, circle, scale, pad, method, zerone_mode, dset_min, dset_max, decim_factor, downsc_factor, corr_offset, postprocess_required, convert_opt, crop_opt, nr_threads, angles_from, angles_to, logfilename, lock, slice_prefix): """To do... """ slice_nr = sino_idx # Perform reconstruction (on-the-fly preprocessing and phase retrieval, if required): if (phaseretrieval_required): # In this case a bunch of sinograms is loaded into memory: # # Load the temporary data structure reading the input TDF file. # To know the right dimension the first sinogram is pre-processed. # # Open the TDF file and get the dataset: f_in = getHDF5(infile, 'r') if "/tomo" in f_in: dset = f_in['tomo'] else: dset = f_in['exchange/data'] # Downscaling and decimation factors considered when determining the approximation window: zrange = arange(sino_idx - approx_win*downsc_factor/2, sino_idx + approx_win*downsc_factor/2, downsc_factor) zrange = zrange[ (zrange >= 0) ] zrange = zrange[ (zrange < num_sinos) ] approx_win = zrange.shape[0] # Approximation window cannot be odd: if (approx_win % 2 == 1): approx_win = approx_win-1 zrange = zrange[0:approx_win] # Read one sinogram to get the proper dimensions: test_im = tdf.read_sino(dset, zrange[0]*downsc_factor).astype(float32) # Apply decimation and downscaling (if required): test_im = test_im[::decim_factor, ::downsc_factor] # Perform the pre-processing of the first sinogram to get the right dimension: if (preprocessing_required): test_im = flat_fielding (test_im, zrange[0], corr_plan, flat_end, half_half, half_half_line/decim_factor, norm_sx, norm_dx).astype(float32) if ext_fov: test_im = extfov_correction (test_im, ext_fov_rot_right, ext_fov_overlap/downsc_factor, ext_fov_normalize, ext_fov_average).astype(float32) test_im = ring_correction (test_im, ringrem, flat_end, corr_plan['skip_flat_after'], half_half, half_half_line/decim_factor, ext_fov).astype(float32) # Now we can allocate memory for the bunch of slices: tmp_im = empty((approx_win, test_im.shape[0], test_im.shape[1]), dtype=float32) tmp_im[0,:,:] = test_im # Reading all the the sinos from TDF file and close: for ct in range(1, approx_win): test_im = tdf.read_sino(dset, zrange[ct]*downsc_factor).astype(float32) test_im = test_im[::decim_factor, ::downsc_factor] # Perform the pre-processing for each sinogram of the bunch: if (preprocessing_required): test_im = flat_fielding (test_im, zrange[ct], corr_plan, flat_end, half_half, half_half_line/decim_factor, norm_sx, norm_dx).astype(float32) if ext_fov: test_im = extfov_correction (test_im, ext_fov_rot_right, ext_fov_overlap/downsc_factor, ext_fov_normalize, ext_fov_average).astype(float32) test_im = ring_correction (test_im, ringrem, flat_end, corr_plan['skip_flat_after'], half_half, half_half_line/decim_factor, ext_fov).astype(float32) tmp_im[ct,:,:] = test_im f_in.close() # Now everything has to refer to a downscaled dataset: sino_idx = ((zrange == sino_idx).nonzero()) # # Perform phase retrieval: # # Prepare the plan: if (phrtmethod == 0): # Paganin's: phrtplan = tiehom_plan (tmp_im[:,0,:], phrt_param1, phrt_param2, energy, distance, pixsize*downsc_factor, padding=phrtpad) else: phrtplan = phrt_plan (tmp_im[:,0,:], energy, distance, pixsize*downsc_factor, phrt_param2, phrt_param1, phrtmethod, padding=phrtpad) # Process each projection (whose height depends on the size of the bunch): for ct in range(0, tmp_im.shape[1]): if (phrtmethod == 0): tmp_im[:,ct,:] = tiehom(tmp_im[:,ct,:], phrtplan).astype(float32) else: tmp_im[:,ct,:] = phrt(tmp_im[:,ct,:], phrtplan, phrtmethod).astype(float32) # Extract the requested sinogram: im = tmp_im[sino_idx[0],:,:].squeeze() else: # Read only one sinogram: f_in = getHDF5(infile, 'r') if "/tomo" in f_in: dset = f_in['tomo'] else: dset = f_in['exchange/data'] im = tdf.read_sino(dset,sino_idx * downsc_factor).astype(float32) f_in.close() # Downscale and decimate the sinogram: im = im[::decim_factor,::downsc_factor] #sino_idx = sino_idx/downsc_factor # Perform the preprocessing of the sinogram (if required): if (preprocessing_required): im = flat_fielding (im, sino_idx, corr_plan, flat_end, half_half, half_half_line/decim_factor, norm_sx, norm_dx).astype(float32) if ext_fov: test_im = extfov_correction (test_im, ext_fov_rot_right, ext_fov_overlap/downsc_factor, ext_fov_normalize, ext_fov_average).astype(float32) im = ring_correction (im, ringrem, flat_end, corr_plan['skip_flat_after'], half_half, half_half_line/decim_factor, ext_fov) # Log infos: log = open(logfilename,"a") log.write(linesep + "\tPerforming reconstruction with multiple centers of rotation...") log.write(linesep + "\t--------------") log.close() # Split the computation into multiple processes: for num in range(nr_threads): start = ( (angles_to - angles_from + 1) / nr_threads)*num + angles_from if (num == nr_threads - 1): end = angles_to else: end = ( (angles_to - angles_from + 1) / nr_threads)*(num + 1) + angles_from - 1 Process(target=reconstruct, args=(im, angles, offset/downsc_factor, logtransform, param1, circle, scale, pad, method, zerone_mode, dset_min, dset_max, corr_offset, postprocess_required, convert_opt, crop_opt, start, end, outpath, slice_nr, downsc_factor, decim_factor, logfilename, lock, slice_prefix)).start()
def main(argv): """ Converts a sequence of TIFF files into a TDF file (HDF5 Tomo Data Format). Parameters ---------- from : scalar, integer among all the projections (or sinogram) files, a subset of files can be specified, ranging from the parameter "from" to the parameter "to" (see next). In most cases, this parameter is 0. to : scalar, integer among all the projections (or sinogram) files, a subset of files can be specified, ranging from the parameter "from" (see previous parameter) to the parameter "to". If the value -1 is specified, all the projection files will be considered. in_path : string path containing the sequence of TIFF files to consider (e.g. "Z:\\sample1\\tomo\\"). out_file : string path with filename of the TDF to create (e.g. "Z:\\sample1.tdf"). WARNING: the program does NOT automatically create non-existing folders and subfolders specified in the path. Moreover, if a file with the same name already exists it will be automatically deleted and overwritten. crop_top : scalar, integer during the conversion, images can be cropped if required. This parameter specifies the number of pixels to crop from the top of the image. Leave 0 for no cropping. crop_bottom : scalar, integer during the conversion, images can be cropped if required. This parameter specifies the number of pixels to crop from the bottom of the image. Leave 0 for no cropping. crop_left : scalar, integer during the conversion, images can be cropped if required. This parameter specifies the number of pixels to crop from the left of the image. Leave 0 for no cropping. crop_right : scalar, integer during the conversion, images can be cropped if required. This parameter specifies the number of pixels to crop from the right of the image. Leave 0 for no cropping. file_prefix : string string to be assumed as the filename prefix of the TIFF files to consider for the projection (or sinogram) files. E.g. "tomo" will consider files having name "tomo_0001.tif", "tomo_0002.tif". flat_prefix : string string to be assumed as the filename prefix of the TIFF files to consider for the flat (white field) files. E.g. "flat" will consider files having name "flat_1.tif", "flat_2.tif". If dark or flat files have to be skipped the string "-" can be specified. dark_prefix : string string to be assumed as the filename prefix of the TIFF files to consider for the dark (dark field) files. E.g. "dark" will consider files having name "dark_1.tif", "dark_2.tif". If dark or flat files have to be skipped the string "-" can be specified. projection_order : boolean string specify the string "True" if the TIFF files represent projections (the most common case), "False" for sinograms. privilege_sino : boolean string specify the string "True" if the TDF will privilege a fast read/write of sinograms (the most common case), "False" for fast read/write of projections. compression : scalar, integer an integer value in the range of [1,9] to be used as GZIP compression factor in the HDF5 file, where 1 is the minimum compression (and maximum speed) and 9 is the maximum (and slow) compression. The value 0 can be specified with the meaning of no compression. nr_threads : int number of multiple threads (actually processes) to consider to speed up the whole conversion process. log_file : string path with filename of a log file (e.g. "R:\\log.txt") where info about the conversion is reported. Returns ------- no return value Example ------- Example call to convert all the tomo*.tif* projections to a TDF with no cropping and minimum compression: python tiff2tdf.py 0 -1 "Z:\\rawdata\\c_1\\tomo\\" "Z:\\work\\c1_compr9.tdf" 0 0 0 0 tomo flat dark True True 1 "S:\\conversion.txt" Requirements ------- - Python 2.7 with the latest NumPy, SciPy, H5Py. - TIFFFile from C. Gohlke's website http://www.lfd.uci.edu/~gohlke/ (consider also to install TIFFFile.c for performances). - tdf.py Tests ------- Tested with WinPython-64bit-2.7.6.3 (Windows) and Anaconda 2.1.0 (Linux 64-bit). """ lock = Lock() # Get the from and to number of files to process: int_from = int(argv[0]) int_to = int(argv[1]) # -1 means "all files" # Get paths: inpath = argv[2] outfile = argv[3] crop_top = int(argv[4]) # 0 for all means "no cropping" crop_bottom = int(argv[5]) crop_left = int(argv[6]) crop_right = int(argv[7]) tomoprefix = argv[8] flatprefix = argv[9] # - means "do not consider flat or darks" darkprefix = argv[10] # - means "do not consider flat or darks" if (flatprefix == "-") or (darkprefix == "-"): skipflat = True else: skipflat = False projorder = True if argv[11] == "True" else False privilege_sino = True if argv[12] == "True" else False # Get compression factor: compr_opts = int(argv[13]) compressionFlag = True; if (compr_opts <= 0): compressionFlag = False; elif (compr_opts > 9): compr_opts = 9 nr_threads = int(argv[14]) logfilename = argv[15] # Check prefixes and path: if not inpath.endswith(os.path.sep): inpath += os.path.sep # Get the files in inpath: log = open(logfilename,"w") log.write(os.linesep + "\tInput path: %s" % (inpath)) log.write(os.linesep + "\tOutput TDF file: %s" % (outfile)) log.write(os.linesep + "\t--------------") log.write(os.linesep + "\tProjection file prefix: %s" % (tomoprefix)) log.write(os.linesep + "\tDark file prefix: %s" % (darkprefix)) log.write(os.linesep + "\tFlat file prefix: %s" % (flatprefix)) log.write(os.linesep + "\t--------------") log.write(os.linesep + "\tCropping:") log.write(os.linesep + "\t\tTop: %d pixels" % (crop_top)) log.write(os.linesep + "\t\tBottom: %d pixels" % (crop_bottom)) log.write(os.linesep + "\t\tLeft: %d pixels" % (crop_left)) log.write(os.linesep + "\t\tRight: %d pixels" % (crop_right)) if (int_to != -1): log.write(os.linesep + "\tThe subset [%d,%d] of the input files will be considered." % (int_from, int_to)) if (projorder): log.write(os.linesep + "\tProjection order assumed.") else: log.write(os.linesep + "\tSinogram order assumed.") if (privilege_sino): log.write(os.linesep + "\tFast I/O for sinograms privileged.") else: log.write(os.linesep + "\tFast I/O for projections privileged.") if (compressionFlag): log.write(os.linesep + "\tTDF compression factor: %d" % (compr_opts)) else: log.write(os.linesep + "\tTDF compression: none.") if (skipflat): log.write(os.linesep + "\tWarning: flat/dark images (if any) will not be considered.") log.write(os.linesep + "\t--------------") log.close() # Remove a previous copy of output: if os.path.exists(outfile): log = open(logfilename,"a") log.write(os.linesep + "\tWarning: an output file with the same name was overwritten.") os.remove(outfile) log.close() log = open(logfilename,"a") log.write(os.linesep + "\tBrowsing input files...") log.close() # Pythonic way to get file list: if os.path.exists(inpath): tomo_files = sorted(glob(inpath + tomoprefix + '*.tif*')) num_files = len(tomo_files) else: log = open(logfilename,"a") log.write(os.linesep + "\tError: input path does not exist. Process will end.") log.close() exit() if (num_files == 0): log = open(logfilename,"a") log.write(os.linesep + "\tError: no projection files found. Check input path and file prefixes.") log.close() exit() log = open(logfilename,"a") log.write(os.linesep + "\tInput files browsed correctly.") log.close() # Check extrema (int_to == -1 means all files): if ( (int_to >= num_files) or (int_to == -1) ): int_from = 0 int_to = num_files - 1 # In case of subset specified: num_files = int_to - int_from + 1 # Prepare output HDF5 output (should this be atomic?): im = imread(tomo_files[0]) # Crop: im = im[crop_top:im.shape[0]-crop_bottom,crop_left:im.shape[1]-crop_right] log = open(logfilename,"a") log.write(os.linesep + "\tPreparing the working plan...") log.close() #dsetshape = (num_files,) + im.shape if projorder: #dsetshape = tdf.get_dset_shape(privilege_sino, im.shape[1], im.shape[0], num_files) datashape = tdf.get_dset_shape(im.shape[1], im.shape[0], num_files) else: #dsetshape = tdf.get_dset_shape(privilege_sino, im.shape[1], num_files, im.shape[0]) datashape = tdf.get_dset_shape(im.shape[1], num_files, im.shape[0]) if not os.path.isfile(outfile): f = getHDF5( outfile, 'w' ) f.attrs['version'] = '1.0' f.attrs['implements'] = "exchange:provenance" echange_group = f.create_group( 'exchange' ) if (compressionFlag): dset = f.create_dataset('exchange/data', datashape, im.dtype, chunks=tdf.get_dset_chunks(im.shape[1]), compression="gzip", compression_opts=compr_opts, shuffle=True, fletcher32=True) else: dset = f.create_dataset('exchange/data', datashape, im.dtype, chunks=tdf.get_dset_chunks(im.shape[1])) if privilege_sino: dset.attrs['axes'] = "y:theta:x" else: dset.attrs['axes'] = "theta:y:x" dset.attrs['min'] = str(numpy.amin(im[:])) dset.attrs['max'] = str(numpy.amax(im[:])) # Get the total number of files to consider: tot_files = num_files if not skipflat: num_flats = len(sorted(glob(inpath + flatprefix + '*.tif*'))) num_darks = len(sorted(glob(inpath + darkprefix + '*.tif*'))) tot_files = tot_files + num_flats + num_darks # Create provenance dataset: provenance_dt = numpy.dtype([("filename", numpy.dtype("S255")), ("timestamp", numpy.dtype("S255"))]) metadata_group = f.create_group( 'provenance' ) provenance_dset = metadata_group.create_dataset('detector_output', (tot_files,), dtype=provenance_dt) provenance_dset.attrs['tomo_prefix'] = tomoprefix; provenance_dset.attrs['dark_prefix'] = darkprefix; provenance_dset.attrs['flat_prefix'] = flatprefix; provenance_dset.attrs['first_index'] = int(tomo_files[0][-8:-4]); # Handle the metadata: if (os.path.isfile(inpath + 'logfile.xml')): with open (inpath + 'logfile.xml', "r") as file: xml_command = file.read() tdf.parse_metadata(f, xml_command) f.close() # Print out about plan preparation: log = open(logfilename,"a") log.write(os.linesep + "\tWorking plan prepared succesfully.") log.close() # Get the files in inpath: if not skipflat: # # Flat part # flat_files = sorted(glob(inpath + flatprefix + '*.tif*')) num_flats = len(flat_files) if ( num_flats > 0): # Create acquisition group: im = imread(flat_files[0]) im = im[crop_top:im.shape[0]-crop_bottom,crop_left:im.shape[1]-crop_right] #flatshape = tdf.get_dset_shape(privilege_sino, im.shape[1], im.shape[0], num_flats) flatshape = tdf.get_dset_shape(im.shape[1], im.shape[0], num_flats) f = getHDF5( outfile, 'a' ) if (compressionFlag): dset = f.create_dataset('exchange/data_white', flatshape, im.dtype, chunks=tdf.get_dset_chunks(im.shape[1]), compression="gzip", compression_opts=compr_opts, shuffle=True, fletcher32=True) else: dset = f.create_dataset('exchange/data_white', flatshape, im.dtype, chunks=tdf.get_dset_chunks(im.shape[1])) dset.attrs['min'] = str(numpy.amin(im[:])) dset.attrs['max'] = str(numpy.amax(im[:])) if privilege_sino: dset.attrs['axes'] = "y:theta:x" else: dset.attrs['axes'] = "theta:y:x" f.close() #process(lock, 0, num_flats - 1, 0, flat_files, True, outfile, 'exchange/data_white', dsetshape, im.dtype, # crop_top, crop_bottom, crop_left, crop_right, tot_files, provenance_dt, logfilename ) else: log = open(logfilename,"a") log.write(os.linesep + "\tWarning: flat files not found.") log.close() # # Dark part # dark_files = sorted(glob(inpath + darkprefix + '*.tif*')) num_darks = len(dark_files) if ( num_darks > 0): im = imread(dark_files[0]) im = im[crop_top:im.shape[0]-crop_bottom,crop_left:im.shape[1]-crop_right] #darkshape = tdf.get_dset_shape(privilege_sino, im.shape[1], im.shape[0], num_flats) darkshape = tdf.get_dset_shape(im.shape[1], im.shape[0], num_darks) f = getHDF5( outfile, 'a' ) if (compressionFlag): dset = f.create_dataset('exchange/data_dark', darkshape, im.dtype, chunks=tdf.get_dset_chunks(im.shape[1]), compression="gzip", compression_opts=compr_opts, shuffle=True, fletcher32=True) else: dset = f.create_dataset('exchange/data_dark', darkshape, im.dtype, chunks=tdf.get_dset_chunks(im.shape[1])) dset.attrs['min'] = str(numpy.amin(im)) dset.attrs['max'] = str(numpy.amax(im)) if privilege_sino: dset.attrs['axes'] = "y:theta:x" else: dset.attrs['axes'] = "theta:y:x" f.close() #process(lock, 0, num_darks - 1, num_flats, dark_files, True, outfile, 'exchange/data_dark', dsetshape, im.dtype, # crop_top, crop_bottom, crop_left, crop_right, tot_files, provenance_dt, logfilename ) else: log = open(logfilename,"a") log.write(os.linesep + "\tWarning: dark files not found.") log.close() # Process the required subset of images: if not skipflat: flatdark_offset = num_flats + num_darks else: flatdark_offset = 0 # Spawn the process for the conversion of flat images: Process(target=_process, args=(lock, 0, num_flats - 1, 0, 0, flat_files, True, outfile, 'exchange/data_white', flatshape, im.dtype, crop_top, crop_bottom, crop_left, crop_right, tot_files, provenance_dt, logfilename )).start() # Spawn the process for the conversion of dark images: Process(target=_process, args=(lock, 0, num_darks - 1, num_flats, 0, dark_files, True, outfile, 'exchange/data_dark', darkshape, im.dtype, crop_top, crop_bottom, crop_left, crop_right, tot_files, provenance_dt, logfilename )).start() # Start the process for the conversion of the projections (or sinograms) in a multi-threaded way: for num in range(nr_threads): start = ( (int_to - int_from + 1) / nr_threads)*num + int_from if (num == nr_threads - 1): end = int_to else: end = ( (int_to - int_from + 1) / nr_threads)*(num + 1) + int_from - 1 Process(target=_process, args=(lock, start, end, flatdark_offset, int_from, tomo_files, projorder, outfile, 'exchange/data', datashape, im.dtype, crop_top, crop_bottom, crop_left, crop_right, tot_files, provenance_dt, logfilename )).start()
def main(argv): """To do... """ # Get the zero-order index of the sinogram to pre-process: idx = int(argv[0]) # Get paths: infile = argv[1] outfile = argv[2] # Normalization parameters: norm_sx = int(argv[3]) norm_dx = int(argv[4]) # Params for flat fielding with post flats/darks: flat_end = True if argv[5] == "True" else False half_half = True if argv[6] == "True" else False half_half_line = int(argv[7]) # Params for extended FOV: ext_fov = True if argv[8] == "True" else False ext_fov_rot_right = argv[9] if ext_fov_rot_right == "True": ext_fov_rot_right = True if (ext_fov): norm_sx = 0 else: ext_fov_rot_right = False if (ext_fov): norm_dx = 0 ext_fov_overlap = int(argv[10]) ext_fov_normalize = True if argv[11] == "True" else False ext_fov_average = True if argv[12] == "True" else False # Method and parameters coded into a string: ringrem = argv[13] # Flat fielding method (conventional or dynamic): dynamic_ff = True if argv[14] == "True" else False # Tmp path and log file: tmppath = argv[15] if not tmppath.endswith(sep): tmppath += sep logfilename = argv[16] # Open the HDF5 file: f_in = getHDF5(infile, 'r') try: if "/tomo" in f_in: dset = f_in['tomo'] else: dset = f_in['exchange/data'] except: log = open(logfilename, "a") log.write(linesep + "\tError reading input dataset. Process will end.") log.close() exit() num_proj = tdf.get_nr_projs(dset) num_sinos = tdf.get_nr_sinos(dset) # Check if the HDF5 makes sense: if (num_sinos == 0): log = open(logfilename, "a") log.write(linesep + "\tNo projections found. Process will end.") log.close() exit() # Get flat and darks from cache or from file: skipflat = False skipdark = False if not dynamic_ff: try: corrplan = cache2plan(infile, tmppath) except Exception as e: #print "Error(s) when reading from cache" corrplan = extract_flatdark(f_in, flat_end, logfilename) if (isscalar(corrplan['im_flat']) and isscalar(corrplan['im_flat_after'])): skipflat = True else: plan2cache(corrplan, infile, tmppath) else: # Dynamic flat fielding: if "/tomo" in f_in: if "/flat" in f_in: flat_dset = f_in['flat'] if "/dark" in f_in: im_dark = _medianize(f_in['dark']) else: skipdark = True else: skipflat = True # Nothing to do in this case else: if "/exchange/data_white" in f_in: flat_dset = f_in['/exchange/data_white'] if "/exchange/data_dark" in f_in: im_dark = _medianize(f_in['/exchange/data_dark']) else: skipdark = True else: skipflat = True # Nothing to do in this case # Prepare plan for dynamic flat fielding with 16 repetitions: if not skipflat: EFF, filtEFF = dff_prepare_plan(flat_dset, 16, im_dark) # Read input image: im = tdf.read_sino(dset, idx).astype(float32) f_in.close() # Perform pre-processing (flat fielding, extended FOV, ring removal): if not skipflat: if dynamic_ff: # Dynamic flat fielding with downsampling = 2: im = dynamic_flat_fielding(im, idx, EFF, filtEFF, 2, im_dark, norm_sx, norm_dx) else: im = flat_fielding(im, idx, corrplan, flat_end, half_half, half_half_line, norm_sx, norm_dx) if ext_fov: im = extfov_correction(im, ext_fov_rot_right, ext_fov_overlap, ext_fov_normalize, ext_fov_average) if not skipflat and not dynamic_ff: im = ring_correction(im, ringrem, flat_end, corrplan['skip_flat_after'], half_half, half_half_line, ext_fov) else: im = ring_correction(im, ringrem, False, False, half_half, half_half_line, ext_fov) # Write down reconstructed preview file (file name modified with metadata): im = im.astype(float32) outfile = outfile + '_' + str(im.shape[1]) + 'x' + str( im.shape[0]) + '_' + str(nanmin(im)) + '$' + str(nanmax(im)) im.tofile(outfile)
def main(argv): """ Converts a set of HIS files into a TDF file (HDF5 Tomo Data Format). Parameters ---------- from : scalar, integer among all the projections (or sinogram) files, a subset of files can be specified, ranging from the parameter "from" to the parameter "to" (see next). In most cases, this parameter is 0. to : scalar, integer among all the projections (or sinogram) files, a subset of files can be specified, ranging from the parameter "from" (see previous parameter) to the parameter "to". If the value -1 is specified, all the projection files will be considered. data_in_path : string path of the HIS file of the projections (e.g. "Z:\\sample1.his"). dark_in_path : string path of the HIS file of the flat (e.g. "Z:\\sample1_dark.his"). flat_in_path : string path of the HIS file of the flat (e.g. "Z:\\sample1_flat.his"). postdark_in_path : string path of the HIS file of the flat (e.g. "Z:\\sample1_postdark.his"). postflat_in_path : string path of the HIS file of the flat (e.g. "Z:\\sample1_postflat.his"). out_file : string path with filename of the TDF to create (e.g. "Z:\\sample1.tdf"). WARNING: the program does NOT automatically create non-existing folders and subfolders specified in the path. Moreover, if a file with the same name already exists it will be automatically deleted and overwritten. crop_top : scalar, integer during the conversion, images can be cropped if required. This parameter specifies the number of pixels to crop from the top of the image. Leave 0 for no cropping. crop_bottom : scalar, integer during the conversion, images can be cropped if required. This parameter specifies the number of pixels to crop from the bottom of the image. Leave 0 for no cropping. crop_left : scalar, integer during the conversion, images can be cropped if required. This parameter specifies the number of pixels to crop from the left of the image. Leave 0 for no cropping. crop_right : scalar, integer during the conversion, images can be cropped if required. This parameter specifies the number of pixels to crop from the right of the image. Leave 0 for no cropping. privilege_sino : boolean string specify the string "True" if the TDF will privilege a fast read/write of sinograms (the most common case), "False" for fast read/write of projections. compression : scalar, integer an integer value in the range of [1,9] to be used as GZIP compression factor in the HDF5 file, where 1 is the minimum compression (and maximum speed) and 9 is the maximum (and slow) compression. The value 0 can be specified with the meaning of no compression. log_file : string path with filename of a log file (e.g. "R:\\log.txt") where info about the conversion is reported. Returns ------- no return value Example ------- Example call to convert all the tomo*.tif* projections to a TDF with no cropping and minimum compression: python his2tdf.py 0 -1 "tomo.his" "dark.his" "flat.his" "postdark.his" "postflat.his" "dataset.tdf" 0 0 0 0 True True 1 "S:\\conversion.txt" Requirements ------- - Python 2.7 with the latest NumPy, SciPy, H5Py. - tdf.py Tests ------- Tested with WinPython-64bit-2.7.6.3 (Windows) and Anaconda 2.1.0 (Linux 64-bit). """ # Get the from and to number of files to process: int_from = int(argv[0]) int_to = int(argv[1]) # -1 means "all files" # Get paths: tomo_file = argv[2] dark_file = argv[3] flat_file = argv[4] darkpost_file = argv[5] flatpost_file = argv[6] outfile = argv[7] crop_top = int(argv[8]) # 0 for all means "no cropping" crop_bottom = int(argv[9]) crop_left = int(argv[10]) crop_right = int(argv[11]) projorder = argv[12] if projorder == "True": projorder = True else: projorder = False privilege_sino = argv[13] if privilege_sino == "True": privilege_sino = True else: privilege_sino = False # Get compression factor: compr_opts = int(argv[14]) compressionFlag = True; if (compr_opts <= 0): compressionFlag = False; elif (compr_opts > 9): compr_opts = 9 logfilename = argv[15] # Get the files in inpath: log = open(logfilename,"w") log.write(os.linesep + "\tInput HIS files:") log.write(os.linesep + "\t\tProjections: %s" % (tomo_file)) log.write(os.linesep + "\t\tDark: %s" % (dark_file)) log.write(os.linesep + "\t\tFlat: %s" % (flat_file)) log.write(os.linesep + "\t\tPost dark: %s" % (darkpost_file)) log.write(os.linesep + "\t\tPost flat: %s" % (flatpost_file)) log.write(os.linesep + "\tOutput TDF file: %s" % (outfile)) log.write(os.linesep + "\t--------------") log.write(os.linesep + "\tCropping:") log.write(os.linesep + "\t\tTop: %d pixels" % (crop_top)) log.write(os.linesep + "\t\tBottom: %d pixels" % (crop_bottom)) log.write(os.linesep + "\t\tLeft: %d pixels" % (crop_left)) log.write(os.linesep + "\t\tRight: %d pixels" % (crop_right)) if (int_to != -1): log.write(os.linesep + "\tThe subset [%d,%d] of the input files will be considered." % (int_from, int_to)) if (projorder): log.write(os.linesep + "\tProjection order assumed.") else: log.write(os.linesep + "\tSinogram order assumed.") if (privilege_sino): log.write(os.linesep + "\tFast I/O for sinograms privileged.") else: log.write(os.linesep + "\tFast I/O for projections privileged.") if (compressionFlag): log.write(os.linesep + "\tTDF compression factor: %d" % (compr_opts)) else: log.write(os.linesep + "\tTDF compression: none.") log.write(os.linesep + "\t--------------") log.close() # Remove a previous copy of output: if os.path.exists(outfile): log = open(logfilename,"a") log.write(os.linesep + "\tWarning: an output file with the same name was overwritten.") os.remove(outfile) log.close() # Check input file: if not os.path.exists(tomo_file): log = open(logfilename,"a") log.write(os.linesep + "\tError: input HIS file for projections does not exist. Process will end.") log.close() exit() # First time get the plan: log = open(logfilename,"a") log.write(os.linesep + "\tPreparing the working plan...") log.close() # Get info from projection file: dim1, dim2, dimz, dtype = _getHISdim ( tomo_file ) if ( ((int_to - int_from + 1) > 0) and ((int_to - int_from + 1) < dimz) ): dimz = int_to - int_from + 1 #dsetshape = (num_files,) + im.shape if projorder: #dsetshape = tdf.get_dset_shape(privilege_sino, im.shape[1], im.shape[0], num_files) dsetshape = tdf.get_dset_shape(dim1 - crop_left - crop_right, dim2 - crop_top - crop_bottom, dimz) else: #dsetshape = tdf.get_dset_shape(privilege_sino, im.shape[1], num_files, im.shape[0]) dsetshape = tdf.get_dset_shape(dim1 - crop_left - crop_right, dim2 - crop_top - crop_bottom, dimz) f = getHDF5( outfile, 'w' ) print dsetshape f.attrs['version'] = '1.0' f.attrs['implements'] = "exchange:provenance" echange_group = f.create_group( 'exchange' ) if (compressionFlag): dset = f.create_dataset('exchange/data', dsetshape, dtype, chunks=tdf.get_dset_chunks(dim1 - crop_left - crop_right), compression="gzip", compression_opts=compr_opts, shuffle=True, fletcher32=True) else: dset = f.create_dataset('exchange/data', dsetshape, dtype, chunks=tdf.get_dset_chunks(dim1 - crop_left - crop_right)) if privilege_sino: dset.attrs['axes'] = "y:theta:x" else: dset.attrs['axes'] = "theta:y:x" dset.attrs['min'] = str(numpy.iinfo(dtype).max) dset.attrs['max'] = str(numpy.iinfo(dtype).min) # Get the total number of files to consider: num_darks = 0 num_flats = 0 num_postdarks = 0 num_postflats = 0 if os.path.exists(dark_file): dim1, dim2, num_darks, dtype = _getHISdim ( dark_file ) if os.path.exists(flat_file): dim1, dim2, num_flats, dtype = _getHISdim ( flat_file ) if os.path.exists(darkpost_file): dim1, dim2, num_postdarks, dtype = _getHISdim ( darkpost_file ) if os.path.exists(flatpost_file): dim1, dim2, num_postflats, dtype = _getHISdim ( flatpost_file ) tot_files = dimz + num_darks + num_flats + num_postdarks + num_postflats # Create provenance dataset: provenance_dt = numpy.dtype([("filename", numpy.dtype("S255")), ("timestamp", numpy.dtype("S255"))]) metadata_group = f.create_group( 'provenance' ) provenance_dset = metadata_group.create_dataset('detector_output', (tot_files,), dtype=provenance_dt) provenance_dset.attrs['tomo_prefix'] = 'tomo'; provenance_dset.attrs['dark_prefix'] = 'dark'; provenance_dset.attrs['flat_prefix'] = 'flat'; provenance_dset.attrs['first_index'] = 1; # Handle the metadata: if (os.path.isfile(os.path.dirname(tomo_file) + os.sep + 'logfile.xml')): with open (os.path.dirname(tomo_file) + os.sep + 'logfile.xml', "r") as file: xml_command = file.read() tdf.parse_metadata(f, xml_command) # Print out about plan preparation: first_done = True log = open(logfilename,"a") log.write(os.linesep + "\tWorking plan prepared succesfully.") log.close() # Get the data from HIS: if (num_darks > 0) or (num_postdarks > 0): #dsetshape = (num_files,) + im.shape if projorder: #dsetshape = tdf.get_dset_shape(privilege_sino, im.shape[1], im.shape[0], num_files) dsetshape = tdf.get_dset_shape(dim1 - crop_left - crop_right, dim2 - crop_top - crop_bottom, num_darks + num_postdarks) else: #dsetshape = tdf.get_dset_shape(privilege_sino, im.shape[1], num_files, im.shape[0]) dsetshape = tdf.get_dset_shape(dim1 - crop_left - crop_right, dim2 - crop_top - crop_bottom, num_darks + num_postdarks) if (compressionFlag): darkdset = f.create_dataset('exchange/data_dark', dsetshape, dtype, chunks=tdf.get_dset_chunks(dim1 - crop_left - crop_right), compression="gzip", compression_opts=compr_opts, shuffle=True, fletcher32=True) else: darkdset = f.create_dataset('exchange/data_dark', dsetshape, dtype, chunks=tdf.get_dset_chunks(dim1 - crop_left - crop_right)) if privilege_sino: darkdset.attrs['axes'] = "y:theta:x" else: darkdset.attrs['axes'] = "theta:y:x" darkdset.attrs['min'] = str(numpy.iinfo(dtype).max) darkdset.attrs['max'] = str(numpy.iinfo(dtype).min) else: log = open(logfilename,"a") log.write(os.linesep + "\tWarning: dark images (if any) not considered.") log.close() if (num_flats > 0) or (num_postflats > 0): #dsetshape = (num_files,) + im.shape if projorder: #dsetshape = tdf.get_dset_shape(privilege_sino, im.shape[1], im.shape[0], num_files) dsetshape = tdf.get_dset_shape(dim1 - crop_left - crop_right, dim2 - crop_top - crop_bottom, num_flats + num_postflats) else: #dsetshape = tdf.get_dset_shape(privilege_sino, im.shape[1], num_files, im.shape[0]) dsetshape = tdf.get_dset_shape(dim1 - crop_left - crop_right, dim2 - crop_top - crop_bottom, num_flats + num_postflats) if (compressionFlag): flatdset = f.create_dataset('exchange/data_white', dsetshape, dtype, chunks=tdf.get_dset_chunks(dim1 - crop_left - crop_right), compression="gzip", compression_opts=compr_opts, shuffle=True, fletcher32=True) else: flatdset = f.create_dataset('exchange/data_white', dsetshape, dtype, chunks=tdf.get_dset_chunks(dim1 - crop_left - crop_right)) if privilege_sino: flatdset.attrs['axes'] = "y:theta:x" else: flatdset.attrs['axes'] = "theta:y:x" flatdset.attrs['min'] = str(numpy.iinfo(dtype).max) flatdset.attrs['max'] = str(numpy.iinfo(dtype).min) else: log = open(logfilename,"a") log.write(os.linesep + "\tWarning: flat images (if any) not considered.") log.close() # Process the HIS: provenance_offset = 0 if num_flats > 0: provenance_offset = _processHIS( flat_file, flatdset, 0, provenance_dset, provenance_offset, 0, 'flat', crop_top, crop_bottom, crop_left, crop_right, logfilename ) if num_postflats > 0: provenance_offset = _processHIS( flatpost_file, flatdset, num_flats, provenance_dset, provenance_offset, 7, 'flat', crop_top, crop_bottom, crop_left, crop_right, logfilename ) if num_darks > 0: provenance_offset = _processHIS( dark_file, darkdset, 0, provenance_dset, provenance_offset, 0, 'dark', crop_top, crop_bottom, crop_left, crop_right, logfilename ) if num_postdarks > 0: provenance_offset = _processHIS( darkpost_file, darkdset, num_darks, provenance_dset, provenance_offset, 7, 'dark', crop_top, crop_bottom, crop_left, crop_right, logfilename ) provenance_offset = _processHIS( tomo_file, dset, 0, provenance_dset, provenance_offset, 0, 'tomo', crop_top, crop_bottom, crop_left, crop_right, logfilename, int_from, int_to ) # Close TDF: f.close()
def process(sino_idx, num_sinos, infile, outfile, preprocessing_required, corr_plan, norm_sx, norm_dx, flat_end, half_half, half_half_line, ext_fov, ext_fov_rot_right, ext_fov_overlap, ringrem, phaseretrieval_required, beta, delta, energy, distance, pixsize, phrtpad, approx_win, angles, offset, logtransform, param1, circle, scale, pad, method, zerone_mode, dset_min, dset_max, decim_factor, downsc_factor, corr_offset, postprocess_required, convert_opt, crop_opt, nr_threads, logfilename): """To do... """ # Perform reconstruction (on-the-fly preprocessing and phase retrieval, if required): if (phaseretrieval_required): # In this case a bunch of sinograms is loaded into memory: # # Load the temporary data structure reading the input TDF file. # To know the right dimension the first sinogram is pre-processed. # # Open the TDF file and get the dataset: f_in = getHDF5(infile, 'r') if "/tomo" in f_in: dset = f_in['tomo'] else: dset = f_in['exchange/data'] # Downscaling and decimation factors considered when determining the approximation window: zrange = arange(sino_idx - approx_win*downsc_factor/2, sino_idx + approx_win*downsc_factor/2, downsc_factor) zrange = zrange[ (zrange >= 0) ] zrange = zrange[ (zrange < num_sinos) ] approx_win = zrange.shape[0] # Approximation window cannot be odd: if (approx_win % 2 == 1): approx_win = approx_win-1 zrange = zrange[0:approx_win] # Read one sinogram to get the proper dimensions: test_im = tdf.read_sino(dset, zrange[0]).astype(float32) test_im = test_im[::decim_factor, ::downsc_factor] # Perform the pre-processing of the first sinogram to get the right dimension: if (preprocessing_required): test_im = flat_fielding (test_im, zrange[0]/downsc_factor, corr_plan, flat_end, half_half, half_half_line/decim_factor, norm_sx, norm_dx).astype(float32) test_im = extfov_correction (test_im, ext_fov, ext_fov_rot_right, ext_fov_overlap/downsc_factor).astype(float32) test_im = ring_correction (test_im, ringrem, flat_end, corr_plan['skip_flat_after'], half_half, half_half_line/decim_factor, ext_fov).astype(float32) # Now we can allocate memory for the bunch of slices: tmp_im = empty((approx_win, test_im.shape[0], test_im.shape[1]), dtype=float32) tmp_im[0,:,:] = test_im # Reading all the the sinos from TDF file and close: for ct in range(1, approx_win): test_im = tdf.read_sino(dset, zrange[ct]).astype(float32) test_im = test_im[::decim_factor, ::downsc_factor] # Perform the pre-processing for each sinogram of the bunch: if (preprocessing_required): test_im = flat_fielding (test_im, zrange[ct]/downsc_factor, corr_plan, flat_end, half_half, half_half_line/decim_factor, norm_sx, norm_dx).astype(float32) test_im = extfov_correction (test_im, ext_fov, ext_fov_rot_right, ext_fov_overlap/downsc_factor).astype(float32) test_im = ring_correction (test_im, ringrem, flat_end, corr_plan['skip_flat_after'], half_half, half_half_line/decim_factor, ext_fov).astype(float32) tmp_im[ct,:,:] = test_im f_in.close() # Now everything has to refer to a downscaled dataset: sino_idx = ((zrange == sino_idx).nonzero()) # # Perform phase retrieval: # # Prepare the plan: phrtplan = prepare_plan (tmp_im[:,0,:], beta, delta, energy, distance, pixsize*downsc_factor, padding=phrtpad) # Process each projection (whose height depends on the size of the bunch): for ct in range(0, tmp_im.shape[1]): tmp_im[:,ct,:] = phase_retrieval(tmp_im[:,ct,:], phrtplan).astype(float32) # Extract the requested sinogram: im = tmp_im[sino_idx[0],:,:].squeeze() else: # Read only one sinogram: f_in = getHDF5(infile, 'r') if "/tomo" in f_in: dset = f_in['tomo'] else: dset = f_in['exchange/data'] im = tdf.read_sino(dset,sino_idx).astype(float32) f_in.close() # Downscale and decimate the sinogram: im = im[::decim_factor,::downsc_factor] sino_idx = sino_idx/downsc_factor # Perform the preprocessing of the sinogram (if required): if (preprocessing_required): im = flat_fielding (im, sino_idx, corr_plan, flat_end, half_half, half_half_line/decim_factor, norm_sx, norm_dx).astype(float32) im = extfov_correction (im, ext_fov, ext_fov_rot_right, ext_fov_overlap) #imsave('C:\\Temp\\StupidFolder\\sinoprimaringrem.tif', im.astype(float32)) im = ring_correction (im, ringrem, flat_end, corr_plan['skip_flat_after'], half_half, half_half_line/decim_factor, ext_fov) #imsave('C:\\Temp\\StupidFolder\\sinodoporingrem.tif', im.astype(float32)) # Actual reconstruction: im = reconstruct(im, angles, offset/downsc_factor, logtransform, param1, circle, scale, pad, method, zerone_mode, dset_min, dset_max, corr_offset).astype(float32) # Apply post-processing (if required): if postprocess_required: im = postprocess(im, convert_opt, crop_opt) else: # Create the circle mask for fancy output: if (circle == True): siz = im.shape[1] if siz % 2: rang = arange(-siz / 2 + 1, siz / 2 + 1) else: rang = arange(-siz / 2,siz / 2) x,y = meshgrid(rang,rang) z = x ** 2 + y ** 2 a = (z < (siz / 2 - int(round(abs(offset)/downsc_factor)) ) ** 2) im = im * a # Write down reconstructed preview file (file name modified with metadata): im = im.astype(float32) outfile = outfile + '_' + str(im.shape[1]) + 'x' + str(im.shape[0]) + '_' + str( amin(im)) + '$' + str( amax(im) ) im.tofile(outfile)
def main(argv): """To do... Usage ----- Parameters --------- Example -------------------------- The following line processes the first ten TIFF files of input path "/home/in" and saves the processed files to "/home/out" with the application of the Boin and Haibel filter with smoothing via a Butterworth filter of order 4 and cutoff frequency 0.01: reconstruct 0 4 C:\Temp\Dullin_Aug_2012\sino_noflat C:\Temp\Dullin_Aug_2012\sino_noflat\output 9.0 10.0 0.0 0.0 0.0 true sino slice C:\Temp\Dullin_Aug_2012\sino_noflat\tomo_conv flat dark """ skip_flat = False skip_flat_after = True # Get the from and to number of files to process: sino_idx = int(argv[0]) # Get paths: infile = argv[1] outfile = argv[2] # Essential reconstruction parameters:: angles = float(argv[3]) offset = float(argv[4]) param1 = argv[5] scale = int(float(argv[6])) overpad = True if argv[7] == "True" else False logtrsf = True if argv[8] == "True" else False circle = True if argv[9] == "True" else False # Parameters for on-the-fly pre-processing: preprocessing_required = True if argv[10] == "True" else False flat_end = True if argv[11] == "True" else False half_half = True if argv[12] == "True" else False half_half_line = int(argv[13]) ext_fov = True if argv[14] == "True" else False norm_sx = int(argv[17]) norm_dx = int(argv[18]) ext_fov_rot_right = argv[15] if ext_fov_rot_right == "True": ext_fov_rot_right = True if (ext_fov): norm_sx = 0 else: ext_fov_rot_right = False if (ext_fov): norm_dx = 0 ext_fov_overlap = int(argv[16]) skip_ringrem = True if argv[19] == "True" else False ringrem = argv[20] # Extra reconstruction parameters: zerone_mode = True if argv[21] == "True" else False corr_offset = float(argv[22]) reconmethod = argv[23] decim_factor = int(argv[24]) downsc_factor = int(argv[25]) # Parameters for postprocessing: postprocess_required = True if argv[26] == "True" else False convert_opt = argv[27] crop_opt = argv[28] # Parameters for on-the-fly phase retrieval: phaseretrieval_required = True if argv[29] == "True" else False beta = double(argv[30]) # param1( e.g. regParam, or beta) delta = double(argv[31]) # param2( e.g. thresh or delta) energy = double(argv[32]) distance = double(argv[33]) pixsize = double(argv[34]) / 1000.0 # pixsixe from micron to mm: phrtpad = True if argv[35] == "True" else False approx_win = int(argv[36]) preprocessingplan_fromcache = True if argv[37] == "True" else False nr_threads = int(argv[38]) tmppath = argv[39] if not tmppath.endswith(sep): tmppath += sep logfilename = argv[40] # Open the HDF5 file: f_in = getHDF5(infile, 'r') if "/tomo" in f_in: dset = f_in['tomo'] else: dset = f_in['exchange/data'] if "/provenance/detector_output" in f_in: prov_dset = f_in['provenance/detector_output'] dset_min = -1 dset_max = -1 if (zerone_mode): if ('min' in dset.attrs): dset_min = float(dset.attrs['min']) else: zerone_mode = False if ('max' in dset.attrs): dset_max = float(dset.attrs['max']) else: zerone_mode = False num_sinos = tdf.get_nr_sinos(dset) # Pay attention to the downscale factor if (num_sinos == 0): exit() # Check extrema: if (sino_idx >= num_sinos): sino_idx = num_sinos - 1 # Get correction plan and phase retrieval plan (if required): corrplan = 0 if (preprocessing_required): # Load flat fielding plan either from cache (if required) or from TDF file and cache it for faster re-use: if (preprocessingplan_fromcache): try: corrplan = cache2plan(infile, tmppath) except Exception as e: #print "Error(s) when reading from cache" corrplan = extract_flatdark(f_in, flat_end, logfilename) plan2cache(corrplan, infile, tmppath) else: corrplan = extract_flatdark(f_in, flat_end, logfilename) plan2cache(corrplan, infile, tmppath) # Dowscale flat and dark images if necessary: if isinstance(corrplan['im_flat'], ndarray): corrplan['im_flat'] = corrplan['im_flat'][::downsc_factor,::downsc_factor] if isinstance(corrplan['im_dark'], ndarray): corrplan['im_dark'] = corrplan['im_dark'][::downsc_factor,::downsc_factor] if isinstance(corrplan['im_flat_after'], ndarray): corrplan['im_flat_after'] = corrplan['im_flat_after'][::downsc_factor,::downsc_factor] if isinstance(corrplan['im_dark_after'], ndarray): corrplan['im_dark_after'] = corrplan['im_dark_after'][::downsc_factor,::downsc_factor] f_in.close() # Run computation: process( sino_idx, num_sinos, infile, outfile, preprocessing_required, corrplan, norm_sx, norm_dx, flat_end, half_half, half_half_line, ext_fov, ext_fov_rot_right, ext_fov_overlap, ringrem, phaseretrieval_required, beta, delta, energy, distance, pixsize, phrtpad, approx_win, angles, offset, logtrsf, param1, circle, scale, overpad, reconmethod, zerone_mode, dset_min, dset_max, decim_factor, downsc_factor, corr_offset, postprocess_required, convert_opt, crop_opt, nr_threads, logfilename )
def main(argv): """ Converts a set of HIS files into a TDF file (HDF5 Tomo Data Format). Parameters ---------- from : scalar, integer among all the projections (or sinogram) files, a subset of files can be specified, ranging from the parameter "from" to the parameter "to" (see next). In most cases, this parameter is 0. to : scalar, integer among all the projections (or sinogram) files, a subset of files can be specified, ranging from the parameter "from" (see previous parameter) to the parameter "to". If the value -1 is specified, all the projection files will be considered. data_in_path : string path of the HIS file of the projections (e.g. "Z:\\sample1.his"). dark_in_path : string path of the HIS file of the flat (e.g. "Z:\\sample1_dark.his"). flat_in_path : string path of the HIS file of the flat (e.g. "Z:\\sample1_flat.his"). postdark_in_path : string path of the HIS file of the flat (e.g. "Z:\\sample1_postdark.his"). postflat_in_path : string path of the HIS file of the flat (e.g. "Z:\\sample1_postflat.his"). out_file : string path with filename of the TDF to create (e.g. "Z:\\sample1.tdf"). WARNING: the program does NOT automatically create non-existing folders and subfolders specified in the path. Moreover, if a file with the same name already exists it will be automatically deleted and overwritten. crop_top : scalar, integer during the conversion, images can be cropped if required. This parameter specifies the number of pixels to crop from the top of the image. Leave 0 for no cropping. crop_bottom : scalar, integer during the conversion, images can be cropped if required. This parameter specifies the number of pixels to crop from the bottom of the image. Leave 0 for no cropping. crop_left : scalar, integer during the conversion, images can be cropped if required. This parameter specifies the number of pixels to crop from the left of the image. Leave 0 for no cropping. crop_right : scalar, integer during the conversion, images can be cropped if required. This parameter specifies the number of pixels to crop from the right of the image. Leave 0 for no cropping. privilege_sino : boolean string specify the string "True" if the TDF will privilege a fast read/write of sinograms (the most common case), "False" for fast read/write of projections. compression : scalar, integer an integer value in the range of [1,9] to be used as GZIP compression factor in the HDF5 file, where 1 is the minimum compression (and maximum speed) and 9 is the maximum (and slow) compression. The value 0 can be specified with the meaning of no compression. log_file : string path with filename of a log file (e.g. "R:\\log.txt") where info about the conversion is reported. Returns ------- no return value Example ------- Example call to convert all the tomo*.tif* projections to a TDF with no cropping and minimum compression: python his2tdf.py 0 -1 "tomo.his" "dark.his" "flat.his" "postdark.his" "postflat.his" "dataset.tdf" 0 0 0 0 True True 1 "S:\\conversion.txt" Requirements ------- - Python 2.7 with the latest NumPy, SciPy, H5Py. - tdf.py Tests ------- Tested with WinPython-64bit-2.7.6.3 (Windows) and Anaconda 2.1.0 (Linux 64-bit). """ # Get the from and to number of files to process: int_from = int(argv[0]) int_to = int(argv[1]) # -1 means "all files" # Get paths: tomo_file = argv[2] flat_file = argv[3] outfile = argv[4] crop_top = int(argv[5]) # 0 for all means "no cropping" crop_bottom = int(argv[6]) crop_left = int(argv[7]) crop_right = int(argv[8]) projorder = argv[9] if projorder == "True": projorder = True else: projorder = False privilege_sino = argv[10] if privilege_sino == "True": privilege_sino = True else: privilege_sino = False # Get compression factor: compr_opts = int(argv[11]) compressionFlag = True if (compr_opts <= 0): compressionFlag = False elif (compr_opts > 9): compr_opts = 9 logfilename = argv[12] # Get the files in inpath: log = open(logfilename,"w") log.write(os.linesep + "\tInput PiXirad files:") log.write(os.linesep + "\t\tProjections: %s" % (tomo_file)) log.write(os.linesep + "\t\tFlat: %s" % (flat_file)) log.write(os.linesep + "\tOutput TDF file: %s" % (outfile)) log.write(os.linesep + "\t--------------") log.write(os.linesep + "\tCropping:") log.write(os.linesep + "\t\tTop: %d pixels" % (crop_top)) log.write(os.linesep + "\t\tBottom: %d pixels" % (crop_bottom)) log.write(os.linesep + "\t\tLeft: %d pixels" % (crop_left)) log.write(os.linesep + "\t\tRight: %d pixels" % (crop_right)) if (int_to != -1): log.write(os.linesep + "\tThe subset [%d,%d] of the input files will be considered." % (int_from, int_to)) if (projorder): log.write(os.linesep + "\tProjection order assumed.") else: log.write(os.linesep + "\tSinogram order assumed.") if (privilege_sino): log.write(os.linesep + "\tFast I/O for sinograms privileged.") else: log.write(os.linesep + "\tFast I/O for projections privileged.") if (compressionFlag): log.write(os.linesep + "\tTDF compression factor: %d" % (compr_opts)) else: log.write(os.linesep + "\tTDF compression: none.") log.write(os.linesep + "\t--------------") log.close() # Remove a previous copy of output: if os.path.exists(outfile): log = open(logfilename,"a") log.write(os.linesep + "\tWarning: an output file with the same name was overwritten.") os.remove(outfile) log.close() # Check input file: if not os.path.exists(tomo_file): log = open(logfilename,"a") log.write(os.linesep + "\tError: input PiXirad file for projections does not exist. Process will end.") log.close() exit() # First time get the plan: log = open(logfilename,"a") log.write(os.linesep + "\tPreparing the work plan...") log.close() # Get info from projection file: data = _read_pixirad_data(tomo_file) dim1 = data.shape[0] dim2 = data.shape[1] dimz = data.shape[2] dtype = data.dtype if (((int_to - int_from + 1) > 0) and ((int_to - int_from + 1) < dimz)): dimz = int_to - int_from + 1 #dsetshape = (num_files,) + im.shape if projorder: #dsetshape = tdf.get_dset_shape(privilege_sino, im.shape[1], im.shape[0], #num_files) dsetshape = tdf.get_dset_shape(dim1 - crop_left - crop_right, dim2 - crop_top - crop_bottom, dimz) else: #dsetshape = tdf.get_dset_shape(privilege_sino, im.shape[1], num_files, #im.shape[0]) dsetshape = tdf.get_dset_shape(dim1 - crop_left - crop_right, dim2 - crop_top - crop_bottom, dimz) f = getHDF5(outfile, 'w') f.attrs['version'] = '1.0' f.attrs['implements'] = "exchange:provenance" echange_group = f.create_group('exchange') if (compressionFlag): dset = f.create_dataset('exchange/data', dsetshape, dtype, chunks=tdf.get_dset_chunks(dim1 - crop_left - crop_right), \ compression="gzip", compression_opts=compr_opts, shuffle=True, fletcher32=True) else: dset = f.create_dataset('exchange/data', dsetshape, dtype) if privilege_sino: dset.attrs['axes'] = "y:theta:x" else: dset.attrs['axes'] = "theta:y:x" dset.attrs['min'] = str(numpy.iinfo(dtype).max) dset.attrs['max'] = str(numpy.iinfo(dtype).min) # Get the total number of files to consider: num_flats = 0 if os.path.exists(flat_file): data = _read_pixirad_data(flat_file) dim1 = data.shape[0] dim2 = data.shape[1] num_flats = data.shape[2] dtype = data.dtype tot_files = dimz + num_flats # Create provenance dataset: provenance_dt = numpy.dtype([("filename", numpy.dtype("S255")), ("timestamp", numpy.dtype("S255"))]) metadata_group = f.create_group('provenance') provenance_dset = metadata_group.create_dataset('detector_output', (tot_files,), dtype=provenance_dt) provenance_dset.attrs['tomo_prefix'] = 'tomo' provenance_dset.attrs['flat_prefix'] = 'flat' provenance_dset.attrs['first_index'] = 1 # Handle the metadata: if (os.path.isfile(os.path.dirname(tomo_file) + os.sep + 'logfile.xml')): with open(os.path.dirname(tomo_file) + os.sep + 'logfile.xml', "r") as file: xml_command = file.read() tdf.parse_metadata(f, xml_command) # Print out about plan preparation: first_done = True log = open(logfilename,"a") log.write(os.linesep + "\tWork plan prepared succesfully.") log.close() # Get the data from Pixirad: if (num_flats > 0) or (num_postflats > 0): #dsetshape = (num_files,) + im.shape if projorder: #dsetshape = tdf.get_dset_shape(privilege_sino, im.shape[1], im.shape[0], #num_files) dsetshape = tdf.get_dset_shape(dim1 - crop_left - crop_right, dim2 - crop_top - crop_bottom, num_flats) else: #dsetshape = tdf.get_dset_shape(privilege_sino, im.shape[1], num_files, #im.shape[0]) dsetshape = tdf.get_dset_shape(dim1 - crop_left - crop_right, dim2 - crop_top - crop_bottom, num_flats) if (compressionFlag): flatdset = f.create_dataset('exchange/data_white', dsetshape, dtype, chunks=tdf.get_dset_chunks(dim1 - crop_left - crop_right), compression="gzip", compression_opts=compr_opts, shuffle=True, fletcher32=True) else: flatdset = f.create_dataset('exchange/data_white', dsetshape, dtype) if privilege_sino: flatdset.attrs['axes'] = "y:theta:x" else: flatdset.attrs['axes'] = "theta:y:x" flatdset.attrs['min'] = str(numpy.iinfo(dtype).max) flatdset.attrs['max'] = str(numpy.iinfo(dtype).min) else: log = open(logfilename,"a") log.write(os.linesep + "\tWarning: flat images (if any) not considered.") log.close() # Process the Pixirad data: provenance_offset = 0 # (Stupid idea for time offset): if num_flats > 0: provenance_offset = _process_dset(flat_file, flatdset, 0, provenance_dset, provenance_offset, 0, 'flat', crop_top, crop_bottom, crop_left, crop_right, logfilename) provenance_offset = _process_dset(tomo_file, dset, 0, provenance_dset, provenance_offset, 0, 'tomo', crop_top, crop_bottom, crop_left, crop_right, logfilename, int_from, int_to) # Close TDF: f.close()
def main(argv): """To do... """ # Get the zero-order index of the sinogram to pre-process: idx = int(argv[0]) # Get paths: infile = argv[1] outfile = argv[2] # Normalization parameters: norm_sx = int(argv[3]) norm_dx = int(argv[4]) # Params for flat fielding with post flats/darks: flat_end = True if argv[5] == "True" else False half_half = True if argv[6] == "True" else False half_half_line = int(argv[7]) # Flat fielding method (conventional or dynamic): dynamic_ff = True if argv[8] == "True" else False # Parameters for rotation: rotation = float(argv[9]) interp = argv[10] border = argv[11] # Tmp path and log file: tmppath = argv[12] if not tmppath.endswith(sep): tmppath += sep logfilename = argv[13] # Open the HDF5 file: f_in = getHDF5(infile, 'r') try: if "/tomo" in f_in: dset = f_in['tomo'] else: dset = f_in['exchange/data'] except: log = open(logfilename, "a") log.write(linesep + "\tError reading input dataset. Process will end.") log.close() exit() num_proj = tdf.get_nr_projs(dset) num_sinos = tdf.get_nr_sinos(dset) # Check if the HDF5 makes sense: if (num_sinos == 0): log = open(logfilename, "a") log.write(linesep + "\tNo projections found. Process will end.") log.close() exit() # Get flat and darks from cache or from file: skipflat = False skipdark = False if not dynamic_ff: try: corrplan = cache2plan(infile, tmppath) except Exception as e: #print "Error(s) when reading from cache" corrplan = extract_flatdark(f_in, flat_end, logfilename) if (isscalar(corrplan['im_flat']) and isscalar(corrplan['im_flat_after'])): skipflat = True else: plan2cache(corrplan, infile, tmppath) else: # Dynamic flat fielding: if "/tomo" in f_in: if "/flat" in f_in: flat_dset = f_in['flat'] if "/dark" in f_in: im_dark = _medianize(f_in['dark']) else: skipdark = True else: skipflat = True # Nothing to do in this case else: if "/exchange/data_white" in f_in: flat_dset = f_in['/exchange/data_white'] if "/exchange/data_dark" in f_in: im_dark = _medianize(f_in['/exchange/data_dark']) else: skipdark = True else: skipflat = True # Nothing to do in this case # Prepare plan for dynamic flat fielding with 16 repetitions: if not skipflat: EFF, filtEFF = dff_prepare_plan(flat_dset, 16, im_dark) # Read input image: im = tdf.read_tomo(dset, idx).astype(float32) f_in.close() # Perform pre-processing (flat fielding, extended FOV, ring removal): if not skipflat: if dynamic_ff: # Dynamic flat fielding with downsampling = 2: im = dynamic_flat_fielding(im, EFF, filtEFF, 2, im_dark) else: im = flat_fielding(im, idx, corrplan, flat_end, half_half, half_half_line, norm_sx, norm_dx) # Rotate: rows, cols = im.shape M = cv2.getRotationMatrix2D((cols / 2, rows / 2), rotation, 1) if interp == 'nearest': interpflag = cv2.INTER_NEAREST elif interp == 'cubic': interpflag = cv2.INTER_CUBIC elif interp == 'lanczos': interpflag = cv2.INTER_LANCZOS4 else: interpflag = cv2.INTER_LINEAR if border == 'constant': borderflag = cv2.BORDER_CONSTANT else: borderflag = cv2.BORDER_REPLICATE im = cv2.warpAffine(im, M, (cols, rows), flags=interpflag, borderMode=borderflag) # Write down reconstructed preview file (file name modified with metadata): im = im.astype(float32) outfile2 = outfile + '_' + str(im.shape[1]) + 'x' + str( im.shape[0]) + '_' + str(nanmin(im)) + '$' + str( nanmax(im)) + '_after.raw' im.tofile(outfile2)
def _process(lock, int_from, int_to, num_sinos, infile_1, infile_2, infile_3, outfile_abs, outfile_ref, outfile_sca, r1, r2, r3, d1, d2, d3, dd1, dd2, dd3, shiftVert_1, shiftHoriz_1, shiftVert_2, shiftHoriz_2, shiftVert_3, shiftHoriz_3, outshape, outtype, skipflat_1, skipflat_2, skipflat_3, plan_1, plan_2, plan_3, norm_sx, norm_dx, flat_end, half_half, half_half_line, ext_fov, ext_fov_rot_right, ext_fov_overlap, ext_fov_normalize, ext_fov_average, ringrem, dynamic_ff, EFF_1, EFF_2, EFF_3, filtEFF_1, filtEFF_2, filtEFF_3, im_dark_1, im_dark_2, im_dark_3, logfilename): # Process the required subset of images: for i in range(int_from, int_to + 1): # Read input image for top, left and right: t0 = time() f_in = getHDF5(infile_1, 'r') if "/tomo" in f_in: dset = f_in['tomo'] flat_avg1 = float(f_in['flat'].attrs['avg']) dark_avg1 = float(f_in['dark'].attrs['avg']) else: dset = f_in['exchange/data'] flat_avg1 = float(f_in['exchange/data_white'].attrs['avg']) dark_avg1 = float(f_in['exchange/data_dark'].attrs['avg']) # Processing in the sinogram domain so a vertical shift of the # projection requires loading a different sinogram: idx1 = min(max(0, i - shiftVert_1), num_sinos - 1) im_1 = tdf.read_sino(dset, idx1).astype(float32) f_in.close() f_in = getHDF5(infile_2, 'r') if "/tomo" in f_in: dset = f_in['tomo'] flat_avg2 = float(f_in['flat'].attrs['avg']) dark_avg2 = float(f_in['dark'].attrs['avg']) else: dset = f_in['exchange/data'] flat_avg2 = float(f_in['exchange/data_white'].attrs['avg']) dark_avg2 = float(f_in['exchange/data_dark'].attrs['avg']) # Processing in the sinogram domain so a vertical shift of the # projection requires loading a different sinogram: idx2 = min(max(0, i - shiftVert_2), num_sinos - 1) im_2 = tdf.read_sino(dset, idx2).astype(float32) f_in.close() f_in = getHDF5(infile_3, 'r') if "/tomo" in f_in: dset = f_in['tomo'] flat_avg3 = float(f_in['flat'].attrs['avg']) dark_avg3 = float(f_in['dark'].attrs['avg']) else: dset = f_in['exchange/data'] flat_avg3 = float(f_in['exchange/data_white'].attrs['avg']) dark_avg3 = float(f_in['exchange/data_dark'].attrs['avg']) # Processing in the sinogram domain so a vertical shift of the projection # requires loading a different sinogram: idx3 = min(max(0, i - shiftVert_3), num_sinos - 1) im_3 = tdf.read_sino(dset, idx3).astype(float32) f_in.close() t1 = time() # Perform pre-processing (flat fielding, extended FOV, ring removal): if not skipflat_1: if dynamic_ff: # Dynamic flat fielding with downsampling = 2: im_1 = dynamic_flat_fielding(im_1, idx1, EFF_1, filtEFF_1, 2, im_dark_1, norm_sx, norm_dx) else: im_1 = flat_fielding(im_1, idx1, plan_1, flat_end, half_half, half_half_line, norm_sx, norm_dx) if ext_fov: im_1 = extfov_correction(im_1, ext_fov_rot_right, ext_fov_overlap, ext_fov_normalize, ext_fov_average) if not skipflat_1 and not dynamic_ff: im_1 = ring_correction(im_1, ringrem, flat_end, plan_1['skip_flat_after'], half_half, half_half_line, ext_fov) else: im_1 = ring_correction(im_1, ringrem, False, False, half_half, half_half_line, ext_fov) # Perform pre-processing (flat fielding, extended FOV, ring removal): if not skipflat_2: if dynamic_ff: # Dynamic flat fielding with downsampling = 2: im_2 = dynamic_flat_fielding(im_2, idx2, EFF_2, filtEFF_2, 2, im_dark_2, norm_sx, norm_dx) else: im_2 = flat_fielding(im_2, idx2, plan_2, flat_end, half_half, half_half_line, norm_sx, norm_dx) if ext_fov: im_2 = extfov_correction(im_2, ext_fov_rot_right, ext_fov_overlap, ext_fov_normalize, ext_fov_average) if not skipflat_2 and not dynamic_ff: im_2 = ring_correction(im_2, ringrem, flat_end, plan_2['skip_flat_after'], half_half, half_half_line, ext_fov) else: im_2 = ring_correction(im_2, ringrem, False, False, half_half, half_half_line, ext_fov) # Perform pre-processing (flat fielding, extended FOV, ring removal): if not skipflat_3: if dynamic_ff: # Dynamic flat fielding with downsampling = 2: im_3 = dynamic_flat_fielding(im_3, idx3, EFF_3, filtEFF_3, 2, im_dark_3, norm_sx, norm_dx) else: im_3 = flat_fielding(im_3, idx3, plan_3, flat_end, half_half, half_half_line, norm_sx, norm_dx) if ext_fov: im_3 = extfov_correction(im_3, ext_fov_rot_right, ext_fov_overlap, ext_fov_normalize, ext_fov_average) if not skipflat_3 and not dynamic_ff: im_3 = ring_correction(im_3, ringrem, flat_end, plan_3['skip_flat_after'], half_half, half_half_line, ext_fov) else: im_3 = ring_correction(im_3, ringrem, False, False, half_half, half_half_line, ext_fov) t2 = time() # Processing in the sinogram domain so a vertical shift of the # projection requires loading a different sinogram: # Only horizontal shift can be considered at a sinogram level: if (shiftHoriz_1 != 0): im_1 = _shift_horiz(im_1, shiftHoriz_1) if (shiftHoriz_2 != 0): im_2 = _shift_horiz(im_2, shiftHoriz_2) if (shiftHoriz_3 != 0): im_3 = _shift_horiz(im_3, shiftHoriz_3) # Re-normalize with average of the flat-field images: max_val = amax( array([ mean(flat_avg1 - dark_avg1), mean(flat_avg2 - dark_avg2), mean(flat_avg3 - dark_avg3) ])) im_1 = im_1 * (flat_avg1 - dark_avg1) / max_val im_2 = im_2 * (flat_avg2 - dark_avg2) / max_val im_3 = im_3 * (flat_avg3 - dark_avg3) / max_val # Apply GDEI: (im_abs, im_ref, im_sca) = gdei(im_1, im_2, im_3, r1, r2, r3, d1, d2, d3, dd1, dd2, dd3) # Save processed image to HDF5 file (atomic procedure - lock used): lock.acquire() try: t3 = time() _write_data(im_abs, i, outfile_abs, outshape, outtype) _write_data(im_ref, i, outfile_ref, outshape, outtype) _write_data(im_sca, i, outfile_sca, outshape, outtype) t4 = time() # Print out execution time: log = open(logfilename, "a") log.write( linesep + "\tsino_%s processed (CPU: %0.3f sec - I/O: %0.3f sec)." % (str(i).zfill(4), t2 - t1, (t1 - t0) + (t4 - t3))) log.close() finally: lock.release()
def main(argv): """To do... Usage ----- Parameters --------- Example -------------------------- """ # Get the from and to number of files to process: sino_idx = int(argv[0]) # Get paths: infile = argv[1] outfile = argv[2] # Essential reconstruction parameters: angles = float(argv[3]) offset = float(argv[4]) recpar = argv[5] scale = int(float(argv[6])) overpad = True if argv[7] == "True" else False logtrsf = True if argv[8] == "True" else False circle = True if argv[9] == "True" else False # Parameters for on-the-fly pre-processing: preprocessing_required = True if argv[10] == "True" else False flat_end = True if argv[11] == "True" else False half_half = True if argv[12] == "True" else False half_half_line = int(argv[13]) ext_fov = True if argv[14] == "True" else False norm_sx = int(argv[19]) norm_dx = int(argv[20]) ext_fov_rot_right = argv[15] if ext_fov_rot_right == "True": ext_fov_rot_right = True if (ext_fov): norm_sx = 0 else: ext_fov_rot_right = False if (ext_fov): norm_dx = 0 ext_fov_overlap = int(argv[16]) ext_fov_normalize = True if argv[17] == "True" else False ext_fov_average = True if argv[18] == "True" else False skip_ringrem = True if argv[21] == "True" else False ringrem = argv[22] # Extra reconstruction parameters: zerone_mode = True if argv[23] == "True" else False corr_offset = float(argv[24]) reconmethod = argv[25] # Force overpadding in case of GRIDREC for unknown reasons: if reconmethod == "GRIDREC": overpad = True decim_factor = int(argv[26]) downsc_factor = int(argv[27]) # Parameters for postprocessing: postprocess_required = True if argv[28] == "True" else False polarfilt_opt = argv[29] convert_opt = argv[30] crop_opt = argv[31] # Parameters for on-the-fly phase retrieval: phaseretrieval_required = True if argv[32] == "True" else False phrtmethod = int(argv[33]) phrt_param1 = double(argv[34]) # param1( e.g. regParam, or beta) phrt_param2 = double(argv[35]) # param2( e.g. thresh or delta) energy = double(argv[36]) distance = double(argv[37]) pixsize = double(argv[38]) / 1000.0 # pixsixe from micron to mm: phrtpad = True if argv[39] == "True" else False approx_win = int(argv[40]) angles_projfrom = int(argv[41]) angles_projto = int(argv[42]) rolling = True if argv[43] == "True" else False roll_shift = int(int(argv[44]) / decim_factor) preprocessingplan_fromcache = True if argv[45] == "True" else False dynamic_ff = True if argv[46] == "True" else False nr_threads = int(argv[47]) tmppath = argv[48] if not tmppath.endswith(sep): tmppath += sep logfilename = argv[49] # Open the HDF5 file: f_in = getHDF5(infile, 'r') if "/tomo" in f_in: dset = f_in['tomo'] else: dset = f_in['exchange/data'] if "/provenance/detector_output" in f_in: prov_dset = f_in['provenance/detector_output'] dset_min = -1 dset_max = -1 if (zerone_mode): if ('min' in dset.attrs): dset_min = float(dset.attrs['min']) else: zerone_mode = False if ('max' in dset.attrs): dset_max = float(dset.attrs['max']) else: zerone_mode = False num_sinos = tdf.get_nr_sinos(dset) # Pay attention to the downscale factor if (num_sinos == 0): exit() # Check extrema: if (sino_idx >= num_sinos / downsc_factor): sino_idx = num_sinos / downsc_factor - 1 # Get correction plan and phase retrieval plan (if required): skipflat = False corrplan = 0 im_dark = 0 EFF = 0 filtEFF = 0 if (preprocessing_required): if not dynamic_ff: # Load flat fielding plan either from cache (if required) or from TDF file # and cache it for faster re-use: if (preprocessingplan_fromcache): try: corrplan = cache2plan(infile, tmppath) except Exception as e: #print "Error(s) when reading from cache" corrplan = extract_flatdark(f_in, flat_end, logfilename) if (isscalar(corrplan['im_flat']) and isscalar(corrplan['im_flat_after'])): skipflat = True else: plan2cache(corrplan, infile, tmppath) else: corrplan = extract_flatdark(f_in, flat_end, logfilename) if (isscalar(corrplan['im_flat']) and isscalar(corrplan['im_flat_after'])): skipflat = True else: plan2cache(corrplan, infile, tmppath) # Dowscale flat and dark images if necessary: if isinstance(corrplan['im_flat'], ndarray): corrplan['im_flat'] = corrplan[ 'im_flat'][::downsc_factor, ::downsc_factor] if isinstance(corrplan['im_dark'], ndarray): corrplan['im_dark'] = corrplan[ 'im_dark'][::downsc_factor, ::downsc_factor] if isinstance(corrplan['im_flat_after'], ndarray): corrplan['im_flat_after'] = corrplan[ 'im_flat_after'][::downsc_factor, ::downsc_factor] if isinstance(corrplan['im_dark_after'], ndarray): corrplan['im_dark_after'] = corrplan[ 'im_dark_after'][::downsc_factor, ::downsc_factor] else: # Dynamic flat fielding: if "/tomo" in f_in: if "/flat" in f_in: flat_dset = f_in['flat'] if "/dark" in f_in: im_dark = _medianize(f_in['dark']) else: skipdark = True else: skipflat = True # Nothing to do in this case else: if "/exchange/data_white" in f_in: flat_dset = f_in['/exchange/data_white'] if "/exchange/data_dark" in f_in: im_dark = _medianize(f_in['/exchange/data_dark']) else: skipdark = True else: skipflat = True # Nothing to do in this case # Prepare plan for dynamic flat fielding with 16 repetitions: if not skipflat: EFF, filtEFF = dff_prepare_plan(flat_dset, 16, im_dark) # Downscale images if necessary: im_dark = im_dark[::downsc_factor, ::downsc_factor] EFF = EFF[::downsc_factor, ::downsc_factor, :] filtEFF = filtEFF[::downsc_factor, ::downsc_factor, :] f_in.close() # Run computation: process(sino_idx, num_sinos, infile, outfile, preprocessing_required, corrplan, skipflat, norm_sx, norm_dx, flat_end, half_half, half_half_line, ext_fov, ext_fov_rot_right, ext_fov_overlap, ext_fov_normalize, ext_fov_average, ringrem, phaseretrieval_required, phrtmethod, phrt_param1, phrt_param2, energy, distance, pixsize, phrtpad, approx_win, angles, angles_projfrom, angles_projto, offset, logtrsf, recpar, circle, scale, overpad, reconmethod, rolling, roll_shift, zerone_mode, dset_min, dset_max, decim_factor, downsc_factor, corr_offset, postprocess_required, polarfilt_opt, convert_opt, crop_opt, dynamic_ff, EFF, filtEFF, im_dark, nr_threads, logfilename, tmppath)
def main(argv): """To do... """ lock = Lock() skip_flat = True first_done = False pyfftw_cache_disable() pyfftw_cache_enable() pyfftw_set_keepalive_time(1800) # Get the from and to number of files to process: idx = int(argv[0]) # Get full paths of input TDF and output TDF: infile = argv[1] outfile = argv[2] # Get the phase retrieval parameters: beta = double(argv[3]) # param1( e.g. regParam, or beta) delta = double(argv[4]) # param2( e.g. thresh or delta) energy = double(argv[5]) distance = double(argv[6]) pixsize = double(argv[7]) / 1000.0 # pixsixe from micron to mm: pad = True if argv[8] == "True" else False # Tmp path and log file: tmppath = argv[9] if not tmppath.endswith(sep): tmppath += sep logfilename = argv[10] # Open the HDF5 file: f_in = getHDF5(infile, 'r') if "/tomo" in f_in: dset = f_in['tomo'] else: dset = f_in['exchange/data'] num_proj = tdf.get_nr_projs(dset) num_sinos = tdf.get_nr_sinos(dset) # Check if the HDF5 makes sense: if (num_proj == 0): log = open(logfilename,"a") log.write(linesep + "\tNo projections found. Process will end.") log.close() exit() # Get flats and darks from cache or from file: try: corrplan = cache2plan(infile, tmppath) except Exception as e: #print "Error(s) when reading from cache" corrplan = extract_flatdark(f_in, True, logfilename) remove(logfilename) plan2cache(corrplan, infile, tmppath) # Read projection: im = tdf.read_tomo(dset,idx).astype(float32) f_in.close() # Apply simple flat fielding (if applicable): if (isinstance(corrplan['im_flat_after'], ndarray) and isinstance(corrplan['im_flat'], ndarray) and isinstance(corrplan['im_dark'], ndarray) and isinstance(corrplan['im_dark_after'], ndarray)) : if (idx < num_proj/2): im = (im - corrplan['im_dark']) / (abs(corrplan['im_flat'] - corrplan['im_dark']) + finfo(float32).eps) else: im = (im - corrplan['im_dark_after']) / (abs(corrplan['im_flat_after'] - corrplan['im_dark_after']) + finfo(float32).eps) # Prepare plan: im = im.astype(float32) plan = prepare_plan (im, beta, delta, energy, distance, pixsize, padding=pad) # Perform phase retrieval (first time also PyFFTW prepares a plan): im = phase_retrieval(im, plan) # Write down reconstructed preview file (file name modified with metadata): im = im.astype(float32) outfile = outfile + '_' + str(im.shape[1]) + 'x' + str(im.shape[0]) + '_' + str( nanmin(im)) + '$' + str( nanmax(im) ) im.tofile(outfile)
def main(argv): """Try to guess the amount of overlap in the case of extended FOV CT. Parameters ---------- infile : array_like HDF5 input dataset outfile : string Full path where the identified overlap will be written as output scale : int If sub-pixel precision is interesting, use e.g. 2.0 to get an overlap of .5 value. Use 1.0 if sub-pixel precision is not required tmppath : int Temporary path where look for cached flat/dark files """ # Get path: infile = argv[0] # The HDF5 file on the SSD outfile = argv[1] # The txt file with the proposed center scale = float(argv[2]) tmppath = argv[3] if not tmppath.endswith(sep): tmppath += sep # Create a silly temporary log: tmplog = tmppath + basename(infile) + str(time.time()) # Open the HDF5 file: f_in = getHDF5( infile, 'r' ) if "/tomo" in f_in: dset = f_in['tomo'] else: dset = f_in['exchange/data'] num_proj = tdf.get_nr_projs(dset) # Get first and 180 deg projections: im1 = tdf.read_tomo(dset,0).astype(float32) im2 = tdf.read_tomo(dset,num_proj/2).astype(float32) # Get flats and darks from cache or from file: try: corrplan = cache2plan(infile, tmppath) except Exception as e: #print "Error(s) when reading from cache" corrplan = extract_flatdark(f_in, True, tmplog) remove(tmplog) plan2cache(corrplan, infile, tmppath) # Apply simple flat fielding (if applicable): if (isinstance(corrplan['im_flat_after'], ndarray) and isinstance(corrplan['im_flat'], ndarray) and isinstance(corrplan['im_dark'], ndarray) and isinstance(corrplan['im_dark_after'], ndarray)) : im1 = ((abs(im1 - corrplan['im_dark'])) / (abs(corrplan['im_flat'] - corrplan['im_dark']) + finfo(float32).eps)).astype(float32) im2 = ((abs(im2 - corrplan['im_dark_after'])) / (abs(corrplan['im_flat_after'] - corrplan['im_dark_after']) + finfo(float32).eps)).astype(float32) # Scale projections (if required) to get subpixel estimation: if ( abs(scale - 1.0) > finfo(float32).eps ): im1 = imresize(im1, (int(round(scale*im1.shape[0])), int(round(scale*im1.shape[1]))), interp='bicubic', mode='F'); im2 = imresize(im2, (int(round(scale*im2.shape[0])), int(round(scale*im2.shape[1]))), interp='bicubic', mode='F'); # Find the center (flipping left-right im2): DISTINGUISH BETWEEN AIR ON THE RIGHT AND ON THE LEFT?????? cen = findcenter.usecorrelation(im1, im2[ :,::-1]) cen = (cen / scale)*2.0 # Print center to output file: text_file = open(outfile, "w") text_file.write(str(int(abs(cen)))) text_file.close() # Close input HDF5: f_in.close()
def main(argv): """To do... Usage ----- Parameters --------- Example -------------------------- """ # Get the from and to number of files to process: sino_idx = int(argv[0]) # Get paths: infile = argv[1] outfile = argv[2] # Essential reconstruction parameters: angles = float(argv[3]) offset = float(argv[4]) recpar = argv[5] scale = int(float(argv[6])) overpad = True if argv[7] == "True" else False logtrsf = True if argv[8] == "True" else False circle = True if argv[9] == "True" else False # Parameters for on-the-fly pre-processing: preprocessing_required = True if argv[10] == "True" else False flat_end = True if argv[11] == "True" else False half_half = True if argv[12] == "True" else False half_half_line = int(argv[13]) ext_fov = True if argv[14] == "True" else False norm_sx = int(argv[19]) norm_dx = int(argv[20]) ext_fov_rot_right = argv[15] if ext_fov_rot_right == "True": ext_fov_rot_right = True if (ext_fov): norm_sx = 0 else: ext_fov_rot_right = False if (ext_fov): norm_dx = 0 ext_fov_overlap = int(argv[16]) ext_fov_normalize = True if argv[17] == "True" else False ext_fov_average = True if argv[18] == "True" else False skip_ringrem = True if argv[21] == "True" else False ringrem = argv[22] # Extra reconstruction parameters: zerone_mode = True if argv[23] == "True" else False corr_offset = float(argv[24]) reconmethod = argv[25] # Force overpadding in case of GRIDREC for unknown reasons: if reconmethod == "GRIDREC": overpad = True decim_factor = int(argv[26]) downsc_factor = int(argv[27]) # Parameters for postprocessing: postprocess_required = True if argv[28] == "True" else False polarfilt_opt = argv[29] convert_opt = argv[30] crop_opt = argv[31] # Parameters for on-the-fly phase retrieval: phaseretrieval_required = True if argv[32] == "True" else False phrtmethod = int(argv[33]) phrt_param1 = double(argv[34]) # param1( e.g. regParam, or beta) phrt_param2 = double(argv[35]) # param2( e.g. thresh or delta) energy = double(argv[36]) distance = double(argv[37]) pixsize = double(argv[38]) / 1000.0 # pixsixe from micron to mm: phrtpad = True if argv[39] == "True" else False approx_win = int(argv[40]) angles_projfrom = int(argv[41]) angles_projto = int(argv[42]) rolling = True if argv[43] == "True" else False roll_shift = int(int(argv[44]) / decim_factor) preprocessingplan_fromcache = True if argv[45] == "True" else False dynamic_ff = True if argv[46] == "True" else False nr_threads = int(argv[47]) tmppath = argv[48] if not tmppath.endswith(sep): tmppath += sep logfilename = argv[49] # Open the HDF5 file: f_in = getHDF5(infile, 'r') if "/tomo" in f_in: dset = f_in['tomo'] else: dset = f_in['exchange/data'] if "/provenance/detector_output" in f_in: prov_dset = f_in['provenance/detector_output'] dset_min = -1 dset_max = -1 if (zerone_mode): if ('min' in dset.attrs): dset_min = float(dset.attrs['min']) else: zerone_mode = False if ('max' in dset.attrs): dset_max = float(dset.attrs['max']) else: zerone_mode = False num_sinos = tdf.get_nr_sinos(dset) # Pay attention to the downscale factor if (num_sinos == 0): exit() # Check extrema: if (sino_idx >= num_sinos / downsc_factor): sino_idx = num_sinos / downsc_factor - 1 # Get correction plan and phase retrieval plan (if required): skipflat = False corrplan = 0 im_dark = 0 EFF = 0 filtEFF = 0 if (preprocessing_required): if not dynamic_ff: # Load flat fielding plan either from cache (if required) or from TDF file # and cache it for faster re-use: if (preprocessingplan_fromcache): try: corrplan = cache2plan(infile, tmppath) except Exception as e: #print "Error(s) when reading from cache" corrplan = extract_flatdark(f_in, flat_end, logfilename) if (isscalar(corrplan['im_flat']) and isscalar(corrplan['im_flat_after'])): skipflat = True else: plan2cache(corrplan, infile, tmppath) else: corrplan = extract_flatdark(f_in, flat_end, logfilename) if (isscalar(corrplan['im_flat']) and isscalar(corrplan['im_flat_after'])): skipflat = True else: plan2cache(corrplan, infile, tmppath) # Dowscale flat and dark images if necessary: if isinstance(corrplan['im_flat'], ndarray): corrplan['im_flat'] = corrplan['im_flat'][::downsc_factor,::downsc_factor] if isinstance(corrplan['im_dark'], ndarray): corrplan['im_dark'] = corrplan['im_dark'][::downsc_factor,::downsc_factor] if isinstance(corrplan['im_flat_after'], ndarray): corrplan['im_flat_after'] = corrplan['im_flat_after'][::downsc_factor,::downsc_factor] if isinstance(corrplan['im_dark_after'], ndarray): corrplan['im_dark_after'] = corrplan['im_dark_after'][::downsc_factor,::downsc_factor] else: # Dynamic flat fielding: if "/tomo" in f_in: if "/flat" in f_in: flat_dset = f_in['flat'] if "/dark" in f_in: im_dark = _medianize(f_in['dark']) else: skipdark = True else: skipflat = True # Nothing to do in this case else: if "/exchange/data_white" in f_in: flat_dset = f_in['/exchange/data_white'] if "/exchange/data_dark" in f_in: im_dark = _medianize(f_in['/exchange/data_dark']) else: skipdark = True else: skipflat = True # Nothing to do in this case # Prepare plan for dynamic flat fielding with 16 repetitions: if not skipflat: EFF, filtEFF = dff_prepare_plan(flat_dset, 16, im_dark) # Downscale images if necessary: im_dark = im_dark[::downsc_factor,::downsc_factor] EFF = EFF[::downsc_factor,::downsc_factor,:] filtEFF = filtEFF[::downsc_factor,::downsc_factor,:] f_in.close() # Run computation: process(sino_idx, num_sinos, infile, outfile, preprocessing_required, corrplan, skipflat, norm_sx, norm_dx, flat_end, half_half, half_half_line, ext_fov, ext_fov_rot_right, ext_fov_overlap, ext_fov_normalize, ext_fov_average, ringrem, phaseretrieval_required, phrtmethod, phrt_param1, phrt_param2, energy, distance, pixsize, phrtpad, approx_win, angles, angles_projfrom, angles_projto, offset, logtrsf, recpar, circle, scale, overpad, reconmethod, rolling, roll_shift, zerone_mode, dset_min, dset_max, decim_factor, downsc_factor, corr_offset, postprocess_required, polarfilt_opt, convert_opt, crop_opt, dynamic_ff, EFF, filtEFF, im_dark, nr_threads, logfilename, tmppath)
def main(argv): """To do... """ lock = Lock() skip_flat = False skip_flat_after = True # Get the from and to number of files to process: sino_idx = int(argv[0]) # Get paths: infile = argv[1] outpath = argv[2] # Essential reconstruction parameters:: angles = float(argv[3]) offset = float(argv[4]) param1 = argv[5] scale = int(float(argv[6])) overpad = True if argv[7] == "True" else False logtrsf = True if argv[8] == "True" else False circle = True if argv[9] == "True" else False # Parameters for on-the-fly pre-processing: preprocessing_required = True if argv[10] == "True" else False flat_end = True if argv[11] == "True" else False half_half = True if argv[12] == "True" else False half_half_line = int(argv[13]) ext_fov = True if argv[14] == "True" else False norm_sx = int(argv[19]) norm_dx = int(argv[20]) ext_fov_rot_right = argv[15] if ext_fov_rot_right == "True": ext_fov_rot_right = True if (ext_fov): norm_sx = 0 else: ext_fov_rot_right = False if (ext_fov): norm_dx = 0 ext_fov_overlap = int(argv[16]) ext_fov_normalize = True if argv[17] == "True" else False ext_fov_average = True if argv[18] == "True" else False skip_ringrem = True if argv[21] == "True" else False ringrem = argv[22] # Extra reconstruction parameters: zerone_mode = True if argv[23] == "True" else False corr_offset = float(argv[24]) reconmethod = argv[25] decim_factor = int(argv[26]) downsc_factor = int(argv[27]) # Parameters for postprocessing: postprocess_required = True if argv[28] == "True" else False convert_opt = argv[29] crop_opt = argv[30] # Parameters for on-the-fly phase retrieval: phaseretrieval_required = True if argv[31] == "True" else False phrtmethod = int(argv[32]) phrt_param1 = double(argv[33]) # param1( e.g. regParam, or beta) phrt_param2 = double(argv[34]) # param2( e.g. thresh or delta) energy = double(argv[35]) distance = double(argv[36]) pixsize = double(argv[37]) / 1000.0 # pixsixe from micron to mm: phrtpad = True if argv[38] == "True" else False approx_win = int(argv[39]) preprocessingplan_fromcache = True if argv[40] == "True" else False tmppath = argv[41] if not tmppath.endswith(sep): tmppath += sep nr_threads = int(argv[42]) angles_from = float(argv[43]) angles_to = float(argv[44]) slice_prefix = argv[45] logfilename = argv[46] if not exists(outpath): makedirs(outpath) if not outpath.endswith(sep): outpath += sep # Log info: log = open(logfilename,"w") log.write(linesep + "\tInput dataset: %s" % (infile)) log.write(linesep + "\tOutput path: %s" % (outpath)) log.write(linesep + "\t--------------") log.write(linesep + "\tLoading flat and dark images...") log.close() # Open the HDF5 file: f_in = getHDF5(infile, 'r') if "/tomo" in f_in: dset = f_in['tomo'] else: dset = f_in['exchange/data'] if "/provenance/detector_output" in f_in: prov_dset = f_in['provenance/detector_output'] dset_min = -1 dset_max = -1 if (zerone_mode): if ('min' in dset.attrs): dset_min = float(dset.attrs['min']) else: zerone_mode = False if ('max' in dset.attrs): dset_max = float(dset.attrs['max']) else: zerone_mode = False num_sinos = tdf.get_nr_sinos(dset) # Pay attention to the downscale factor if (num_sinos == 0): exit() # Check extrema: if (sino_idx >= num_sinos / downsc_factor): sino_idx = num_sinos / downsc_factor - 1 # Get correction plan and phase retrieval plan (if required): corrplan = 0 if (preprocessing_required): # Load flat fielding plan either from cache (if required) or from TDF file and cache it for faster re-use: if (preprocessingplan_fromcache): try: corrplan = cache2plan(infile, tmppath) except Exception as e: #print "Error(s) when reading from cache" corrplan = extract_flatdark(f_in, flat_end, logfilename) plan2cache(corrplan, infile, tmppath) else: corrplan = extract_flatdark(f_in, flat_end, logfilename) plan2cache(corrplan, infile, tmppath) # Dowscale flat and dark images if necessary: if isinstance(corrplan['im_flat'], ndarray): corrplan['im_flat'] = corrplan['im_flat'][::downsc_factor,::downsc_factor] if isinstance(corrplan['im_dark'], ndarray): corrplan['im_dark'] = corrplan['im_dark'][::downsc_factor,::downsc_factor] if isinstance(corrplan['im_flat_after'], ndarray): corrplan['im_flat_after'] = corrplan['im_flat_after'][::downsc_factor,::downsc_factor] if isinstance(corrplan['im_dark_after'], ndarray): corrplan['im_dark_after'] = corrplan['im_dark_after'][::downsc_factor,::downsc_factor] f_in.close() # Log infos: log = open(logfilename,"a") log.write(linesep + "\tPerforming preprocessing...") log.close() # Run computation: process( sino_idx, num_sinos, infile, outpath, preprocessing_required, corrplan, norm_sx, norm_dx, flat_end, half_half, half_half_line, ext_fov, ext_fov_rot_right, ext_fov_overlap, ext_fov_normalize, ext_fov_average, ringrem, phaseretrieval_required, phrtmethod, phrt_param1, phrt_param2, energy, distance, pixsize, phrtpad, approx_win, angles, offset, logtrsf, param1, circle, scale, overpad, reconmethod, zerone_mode, dset_min, dset_max, decim_factor, downsc_factor, corr_offset, postprocess_required, convert_opt, crop_opt, nr_threads, angles_from, angles_to, logfilename, lock, slice_prefix )
def main(argv): """To do... """ lock = Lock() skip_flat = True first_done = False pyfftw_cache_disable() pyfftw_cache_enable() pyfftw_set_keepalive_time(1800) # Get the from and to number of files to process: int_from = int(argv[0]) int_to = int(argv[1]) # Get full paths of input TDF and output TDF: infile = argv[2] outfile = argv[3] # Get the phase retrieval parameters: method = int(argv[4]) param1 = double(argv[5]) # e.g. regParam, or beta param2 = double(argv[6]) # e.g. thresh or delta energy = double(argv[7]) distance = double(argv[8]) pixsize = double(argv[9]) / 1000.0 # pixsixe from micron to mm: pad = True if argv[10] == "True" else False # Number of threads (actually processes) to use and logfile: nr_threads = int(argv[11]) logfilename = argv[12] # Log infos: log = open(logfilename,"w") log.write(linesep + "\tInput TDF file: %s" % (infile)) log.write(linesep + "\tOutput TDF file: %s" % (outfile)) log.write(linesep + "\t--------------") if (method == 0): log.write(linesep + "\tMethod: TIE-Hom (Paganin et al., 2002)") log.write(linesep + "\t--------------") log.write(linesep + "\tDelta/Beta: %0.1f" % ((param2/param1)) ) #else: # log.write(linesep + "\tMethod: Projected CTF (Moosmann et al., 2011)") # log.write(linesep + "\t--------------") # log.write(linesep + "\tDelta/Beta: %0.1f" % ((param2/param1)) ) log.write(linesep + "\tEnergy: %0.1f keV" % (energy)) log.write(linesep + "\tDistance: %0.1f mm" % (distance)) log.write(linesep + "\tPixel size: %0.3f micron" % (pixsize*1000)) log.write(linesep + "\t--------------") log.write(linesep + "\tBrowsing input files...") log.close() # Remove a previous copy of output: if exists(outfile): remove(outfile) # Open the HDF5 file: f_in = getHDF5(infile, 'r') if "/tomo" in f_in: dset = f_in['tomo'] else: dset = f_in['exchange/data'] num_proj = tdf.get_nr_projs(dset) num_sinos = tdf.get_nr_sinos(dset) if (num_proj == 0): log = open(logfilename,"a") log.write(linesep + "\tNo projections found. Process will end.") log.close() exit() log = open(logfilename,"a") log.write(linesep + "\tInput files browsed correctly.") log.close() # Check extrema (int_to == -1 means all files): if ( (int_to >= num_proj) or (int_to == -1) ): int_to = num_proj - 1 if ( (int_from < 0) ): int_from = 0 # Prepare the plan: log = open(logfilename,"a") log.write(linesep + "\tPreparing the work plan...") log.close() im = tdf.read_tomo(dset,0).astype(float32) outshape = tdf.get_dset_shape(im.shape[1], im.shape[0], num_proj) f_out = getHDF5(outfile, 'w') f_out_dset = f_out.create_dataset('exchange/data', outshape, float32) f_out_dset.attrs['min'] = str(amin(im[:])) f_out_dset.attrs['max'] = str(amax(im[:])) f_out_dset.attrs['version'] = '1.0' f_out_dset.attrs['axes'] = "y:theta:x" f_in.close() f_out.close() if (method == 0): # Paganin's: plan = tiehom_plan (im, param1, param2, energy, distance, pixsize, pad) else: plan = phrt_plan (im, energy, distance, pixsize, param2, param1, method, pad) # Run several threads for independent computation without waiting for threads completion: for num in range(nr_threads): start = (num_proj / nr_threads)*num if (num == nr_threads - 1): end = num_proj - 1 else: end = (num_proj / nr_threads)*(num + 1) - 1 Process(target=_process, args=(lock, start, end, infile, outfile, outshape, float32, method, plan, logfilename)).start()
def main(argv): """To do... """ # Get the zero-order index of the sinogram to pre-process: idx = int(argv[0]) # Get paths: infile = argv[1] outfile = argv[2] # Normalization parameters: norm_sx = int(argv[3]) norm_dx = int(argv[4]) # Params for flat fielding with post flats/darks: flat_end = True if argv[5] == "True" else False half_half = True if argv[6] == "True" else False half_half_line = int(argv[7]) # Params for extended FOV: ext_fov = True if argv[8] == "True" else False ext_fov_rot_right = argv[9] if ext_fov_rot_right == "True": ext_fov_rot_right = True if (ext_fov): norm_sx = 0 else: ext_fov_rot_right = False if (ext_fov): norm_dx = 0 ext_fov_overlap = int(argv[10]) ext_fov_normalize = True if argv[11] == "True" else False ext_fov_average = True if argv[12] == "True" else False # Method and parameters coded into a string: ringrem = argv[13] # Flat fielding method (conventional or dynamic): dynamic_ff = True if argv[14] == "True" else False # Tmp path and log file: tmppath = argv[15] if not tmppath.endswith(sep): tmppath += sep logfilename = argv[16] # Open the HDF5 file: f_in = getHDF5(infile, 'r') try: if "/tomo" in f_in: dset = f_in['tomo'] else: dset = f_in['exchange/data'] except: log = open(logfilename,"a") log.write(linesep + "\tError reading input dataset. Process will end.") log.close() exit() num_proj = tdf.get_nr_projs(dset) num_sinos = tdf.get_nr_sinos(dset) # Check if the HDF5 makes sense: if (num_sinos == 0): log = open(logfilename,"a") log.write(linesep + "\tNo projections found. Process will end.") log.close() exit() # Get flat and darks from cache or from file: skipflat = False skipdark = False if not dynamic_ff: try: corrplan = cache2plan(infile, tmppath) except Exception as e: #print "Error(s) when reading from cache" corrplan = extract_flatdark(f_in, flat_end, logfilename) if (isscalar(corrplan['im_flat']) and isscalar(corrplan['im_flat_after']) ): skipflat = True else: plan2cache(corrplan, infile, tmppath) else: # Dynamic flat fielding: if "/tomo" in f_in: if "/flat" in f_in: flat_dset = f_in['flat'] if "/dark" in f_in: im_dark = _medianize(f_in['dark']) else: skipdark = True else: skipflat = True # Nothing to do in this case else: if "/exchange/data_white" in f_in: flat_dset = f_in['/exchange/data_white'] if "/exchange/data_dark" in f_in: im_dark = _medianize(f_in['/exchange/data_dark']) else: skipdark = True else: skipflat = True # Nothing to do in this case # Prepare plan for dynamic flat fielding with 16 repetitions: if not skipflat: EFF, filtEFF = dff_prepare_plan(flat_dset, 16, im_dark) # Read input image: im = tdf.read_sino(dset,idx).astype(float32) f_in.close() # Perform pre-processing (flat fielding, extended FOV, ring removal): if not skipflat: if dynamic_ff: # Dynamic flat fielding with downsampling = 2: im = dynamic_flat_fielding(im, idx, EFF, filtEFF, 2, im_dark, norm_sx, norm_dx) else: im = flat_fielding(im, idx, corrplan, flat_end, half_half, half_half_line, norm_sx, norm_dx) if ext_fov: im = extfov_correction(im, ext_fov_rot_right, ext_fov_overlap, ext_fov_normalize, ext_fov_average) if not skipflat and not dynamic_ff: im = ring_correction (im, ringrem, flat_end, corrplan['skip_flat_after'], half_half, half_half_line, ext_fov) else: im = ring_correction (im, ringrem, False, False, half_half, half_half_line, ext_fov) # Write down reconstructed preview file (file name modified with metadata): im = im.astype(float32) outfile = outfile + '_' + str(im.shape[1]) + 'x' + str(im.shape[0]) + '_' + str( nanmin(im)) + '$' + str( nanmax(im) ) im.tofile(outfile)
def main(argv): """To do... Usage ----- Parameters --------- Example -------------------------- """ lock = Lock() # Get the from and to number of files to process: int_from = int(argv[0]) int_to = int(argv[1]) # Get paths: infile_1 = argv[2] infile_2 = argv[3] infile_3 = argv[4] outfile_abs = argv[5] outfile_ref = argv[6] outfile_sca = argv[7] # Normalization parameters: norm_sx = int(argv[8]) norm_dx = int(argv[9]) # Params for flat fielding with post flats/darks: flat_end = True if argv[10] == "True" else False half_half = True if argv[11] == "True" else False half_half_line = int(argv[12]) # Params for extended FOV: ext_fov = True if argv[13] == "True" else False ext_fov_rot_right = argv[14] if ext_fov_rot_right == "True": ext_fov_rot_right = True if (ext_fov): norm_sx = 0 else: ext_fov_rot_right = False if (ext_fov): norm_dx = 0 ext_fov_overlap = int(argv[15]) ext_fov_normalize = True if argv[16] == "True" else False ext_fov_average = True if argv[17] == "True" else False # Method and parameters coded into a string: ringrem = argv[18] # Flat fielding method (conventional or dynamic): dynamic_ff = True if argv[19] == "True" else False # Shift parameters: shiftVert_1 = int(argv[20]) shiftHoriz_1 = int(argv[21]) shiftVert_2 = int(argv[22]) shiftHoriz_2 = int(argv[23]) shiftVert_3 = int(argv[24]) shiftHoriz_3 = int(argv[25]) # DEI coefficients: r1 = float(argv[26]) r2 = float(argv[27]) r3 = float(argv[28]) d1 = float(argv[29]) d2 = float(argv[30]) d3 = float(argv[31]) dd1 = float(argv[32]) dd2 = float(argv[33]) dd3 = float(argv[34]) # Nr of threads and log file: nr_threads = int(argv[35]) logfilename = argv[36] # Log input parameters: log = open(logfilename, "w") log.write(linesep + "\tInput TDF file #1: %s" % (infile_1)) log.write(linesep + "\tInput TDF file #2: %s" % (infile_2)) log.write(linesep + "\tInput TDF file #3: %s" % (infile_3)) log.write(linesep + "\tOutput TDF file for Absorption: %s" % (outfile_abs)) log.write(linesep + "\tOutput TDF file for Refraction: %s" % (outfile_ref)) log.write(linesep + "\tOutput TDF file for Scattering: %s" % (outfile_sca)) log.write(linesep + "\t--------------") log.write(linesep + "\tOpening input dataset...") log.close() # Remove a previous copy of output: #if exists(outfile): # remove(outfile) # Open the HDF5 files: f_in_1 = getHDF5(infile_1, 'r') f_in_2 = getHDF5(infile_2, 'r') f_in_3 = getHDF5(infile_3, 'r') if "/tomo" in f_in_1: dset_1 = f_in_1['tomo'] tomoprefix_1 = 'tomo' flatprefix_1 = 'flat' darkprefix_1 = 'dark' else: dset_1 = f_in_1['exchange/data'] if "/provenance/detector_output" in f_in_1: prov_dset_1 = f_in_1['provenance/detector_output'] tomoprefix_1 = prov_dset_1.attrs['tomo_prefix'] flatprefix_1 = prov_dset_1.attrs['flat_prefix'] darkprefix_1 = prov_dset_1.attrs['dark_prefix'] if "/tomo" in f_in_2: dset_2 = f_in_2['tomo'] tomoprefix_2 = 'tomo' flatprefix_2 = 'flat' darkprefix_2 = 'dark' else: dset_2 = f_in_2['exchange/data'] if "/provenance/detector_output" in f_in_2: prov_dset_2 = f_in_2['provenance/detector_output'] tomoprefix_2 = prov_dset_2.attrs['tomo_prefix'] flatprefix_2 = prov_dset_2.attrs['flat_prefix'] darkprefix_2 = prov_dset_2.attrs['dark_prefix'] if "/tomo" in f_in_3: dset_3 = f_in_3['tomo'] tomoprefix_3 = 'tomo' flatprefix_3 = 'flat' darkprefix_3 = 'dark' else: dset_3 = f_in_3['exchange/data'] if "/provenance/detector_output" in f_in_3: prov_dset_3 = f_in_1['provenance/detector_output'] tomoprefix_3 = prov_dset_3.attrs['tomo_prefix'] flatprefix_3 = prov_dset_3.attrs['flat_prefix'] darkprefix_3 = prov_dset_3.attrs['dark_prefix'] # Assuming that what works for the dataset #1 works for the other two: num_proj = tdf.get_nr_projs(dset_1) num_sinos = tdf.get_nr_sinos(dset_1) if (num_sinos == 0): log = open(logfilename, "a") log.write(linesep + "\tNo projections found. Process will end.") log.close() exit() # Check extrema (int_to == -1 means all files): if ((int_to >= num_sinos) or (int_to == -1)): int_to = num_sinos - 1 # Prepare the work plan for flat and dark images: log = open(logfilename, "a") log.write(linesep + "\t--------------") log.write(linesep + "\tPreparing the work plan...") log.close() # Extract flat and darks: skipflat_1 = False skipdark_1 = False skipflat_2 = False skipdark_2 = False skipflat_3 = False skipdark_3 = False # Following variables make sense only for dynamic flat fielding: EFF_1 = -1 filtEFF_1 = -1 im_dark_1 = -1 EFF_2 = -1 filtEFF_2 = -1 im_dark_2 = -1 EFF_3 = -1 filtEFF_3 = -1 im_dark_3 = -1 # Following variable makes sense only for conventional flat fielding: plan_1 = -1 plan_2 = -1 plan_3 = -1 if not dynamic_ff: plan_1 = extract_flatdark(f_in_1, flat_end, logfilename) if (isscalar(plan_1['im_flat']) and isscalar(plan_1['im_flat_after'])): skipflat_1 = True else: skipflat_1 = False plan_2 = extract_flatdark(f_in_2, flat_end, logfilename) if (isscalar(plan_2['im_flat']) and isscalar(plan_2['im_flat_after'])): skipflat_2 = True else: skipflat_2 = False plan_3 = extract_flatdark(f_in_3, flat_end, logfilename) if (isscalar(plan_3['im_flat']) and isscalar(plan_3['im_flat_after'])): skipflat_3 = True else: skipflat_3 = False else: # Dynamic flat fielding: if "/tomo" in f_in_1: if "/flat" in f_in_1: flat_dset_1 = f_in_1['flat'] if "/dark" in f_in_1: im_dark_1 = _medianize(f_in_1['dark']) else: skipdark_1 = True else: skipflat_1 = True # Nothing to do in this case else: if "/exchange/data_white" in f_in_1: flat_dset_1 = f_in_1['/exchange/data_white'] if "/exchange/data_dark" in f_in_1: im_dark_1 = _medianize(f_in_1['/exchange/data_dark']) else: skipdark_1 = True else: skipflat_1 = True # Nothing to do in this case # Prepare plan for dynamic flat fielding with 16 repetitions: if not skipflat_1: EFF_1, filtEFF_1 = dff_prepare_plan(flat_dset_1, 16, im_dark_1) # Dynamic flat fielding: if "/tomo" in f_in_2: if "/flat" in f_in_2: flat_dset_2 = f_in_2['flat'] if "/dark" in f_in_2: im_dark_2 = _medianize(f_in_2['dark']) else: skipdark_2 = True else: skipflat_2 = True # Nothing to do in this case else: if "/exchange/data_white" in f_in_2: flat_dset_2 = f_in_2['/exchange/data_white'] if "/exchange/data_dark" in f_in_2: im_dark_2 = _medianize(f_in_2['/exchange/data_dark']) else: skipdark_2 = True else: skipflat_2 = True # Nothing to do in this case # Prepare plan for dynamic flat fielding with 16 repetitions: if not skipflat_2: EFF_2, filtEFF_2 = dff_prepare_plan(flat_dset_2, 16, im_dark_2) # Dynamic flat fielding: if "/tomo" in f_in_3: if "/flat" in f_in_3: flat_dset_3 = f_in_3['flat'] if "/dark" in f_in_3: im_dark_3 = _medianize(f_in_3['dark']) else: skipdark_3 = True else: skipflat_3 = True # Nothing to do in this case else: if "/exchange/data_white" in f_in_3: flat_dset_3 = f_in_3['/exchange/data_white'] if "/exchange/data_dark" in f_in_3: im_dark_3 = _medianize(f_in_3['/exchange/data_dark']) else: skipdark_3 = True else: skipflat_3 = True # Nothing to do in this case # Prepare plan for dynamic flat fielding with 16 repetitions: if not skipflat_3: EFF_3, filtEFF_3 = dff_prepare_plan(flat_dset_3, 16, im_dark_3) # Outfile shape can be determined only after first processing in ext FOV mode: if (ext_fov): # Read input sino: idx = num_sinos / 2 im = tdf.read_sino(dset_1, idx).astype(float32) im = extfov_correction(im, ext_fov_rot_right, ext_fov_overlap, ext_fov_normalize, ext_fov_average) # Get the corrected outshape: outshape = tdf.get_dset_shape(im.shape[1], num_sinos, im.shape[0]) else: # Get the corrected outshape (in this case it's easy): im = tdf.read_tomo(dset_1, 0).astype(float32) outshape = tdf.get_dset_shape(im.shape[1], im.shape[0], num_proj) f_in_1.close() f_in_2.close() f_in_3.close() # Create the output HDF5 files: f_out_abs = getHDF5(outfile_abs, 'w') f_out_dset_abs = f_out_abs.create_dataset('exchange/data', outshape, float32) f_out_dset_abs.attrs['min'] = str(finfo(float32).max) f_out_dset_abs.attrs['max'] = str(finfo(float32).min) f_out_dset_abs.attrs['version'] = '1.0' f_out_dset_abs.attrs['axes'] = "y:theta:x" f_out_abs.close() f_out_ref = getHDF5(outfile_ref, 'w') f_out_dset_ref = f_out_ref.create_dataset('exchange/data', outshape, float32) f_out_dset_ref.attrs['min'] = str(finfo(float32).max) f_out_dset_ref.attrs['max'] = str(finfo(float32).min) f_out_dset_ref.attrs['version'] = '1.0' f_out_dset_ref.attrs['axes'] = "y:theta:x" f_out_ref.close() f_out_sca = getHDF5(outfile_sca, 'w') f_out_dset_sca = f_out_sca.create_dataset('exchange/data', outshape, float32) f_out_dset_sca.attrs['min'] = str(finfo(float32).max) f_out_dset_sca.attrs['max'] = str(finfo(float32).min) f_out_dset_sca.attrs['version'] = '1.0' f_out_dset_sca.attrs['axes'] = "y:theta:x" f_out_sca.close() # Log infos: log = open(logfilename, "a") log.write(linesep + "\tWork plan prepared correctly.") log.write(linesep + "\t--------------") log.write(linesep + "\tPerforming GDEI...") log.close() # Run several threads for independent computation without waiting for threads # completion: for num in range(nr_threads): start = (num_sinos / nr_threads) * num if (num == nr_threads - 1): end = num_sinos - 1 else: end = (num_sinos / nr_threads) * (num + 1) - 1 Process( target=_process, args=(lock, start, end, num_sinos, infile_1, infile_2, infile_3, outfile_abs, outfile_ref, outfile_sca, r1, r2, r3, d1, d2, d3, dd1, dd2, dd3, shiftVert_1, shiftHoriz_1, shiftVert_2, shiftHoriz_2, shiftVert_3, shiftHoriz_3, outshape, float32, skipflat_1, skipflat_2, skipflat_3, plan_1, plan_2, plan_3, norm_sx, norm_dx, flat_end, half_half, half_half_line, ext_fov, ext_fov_rot_right, ext_fov_overlap, ext_fov_normalize, ext_fov_average, ringrem, dynamic_ff, EFF_1, EFF_2, EFF_3, filtEFF_1, filtEFF_2, filtEFF_3, im_dark_1, im_dark_2, im_dark_3, logfilename)).start()
def main(argv): """Try to guess the center of rotation of the input CT dataset. Parameters ---------- infile : array_like HDF5 input dataset outfile : string Full path where the identified center of rotation will be written as output scale : int If sub-pixel precision is interesting, use e.g. 2.0 to get a center of rotation of .5 value. Use 1.0 if sub-pixel precision is not required angles : int Total number of angles of the input dataset method : string One of the following options: "registration" tmppath : string Temporary path where look for cached flat/dark files """ # Get path: infile = argv[0] # The HDF5 file on the outfile = argv[1] # The txt file with the proposed center scale = float(argv[2]) angles = float(argv[3]) method = argv[4] tmppath = argv[5] if not tmppath.endswith(sep): tmppath += sep pyfftw_cache_disable() pyfftw_cache_enable() pyfftw_set_keepalive_time(1800) # Create a silly temporary log: tmplog = tmppath + basename(infile) + str(time.time()) # Open the HDF5 file (take into account also older TDF versions): f_in = getHDF5( infile, 'r' ) if "/tomo" in f_in: dset = f_in['tomo'] else: dset = f_in['exchange/data'] num_proj = tdf.get_nr_projs(dset) num_sinos = tdf.get_nr_sinos(dset) # Get flats and darks from cache or from file: try: corrplan = cache2plan(infile, tmppath) except Exception as e: #print "Error(s) when reading from cache" corrplan = extract_flatdark(f_in, True, tmplog) remove(tmplog) plan2cache(corrplan, infile, tmppath) # Get first and the 180 deg projections: im1 = tdf.read_tomo(dset,0).astype(float32) idx = int(round(num_proj/angles * pi)) - 1 im2 = tdf.read_tomo(dset,idx).astype(float32) # Apply simple flat fielding (if applicable): if (isinstance(corrplan['im_flat_after'], ndarray) and isinstance(corrplan['im_flat'], ndarray) and isinstance(corrplan['im_dark'], ndarray) and isinstance(corrplan['im_dark_after'], ndarray)) : im1 = ((abs(im1 - corrplan['im_dark'])) / (abs(corrplan['im_flat'] - corrplan['im_dark']) + finfo(float32).eps)).astype(float32) im2 = ((abs(im2 - corrplan['im_dark_after'])) / (abs(corrplan['im_flat_after'] - corrplan['im_dark_after']) + finfo(float32).eps)).astype(float32) # Scale projections (if required) to get subpixel estimation: if ( abs(scale - 1.0) > finfo(float32).eps ): im1 = imresize(im1, (int(round(scale*im1.shape[0])), int(round(scale*im1.shape[1]))), interp='bicubic', mode='F'); im2 = imresize(im2, (int(round(scale*im2.shape[0])), int(round(scale*im2.shape[1]))), interp='bicubic', mode='F'); # Find the center (flipping left-right im2): cen = findcenter.usecorrelation(im1, im2[ :,::-1]) cen = cen / scale # Print center to output file: text_file = open(outfile, "w") text_file.write(str(int(cen))) text_file.close() # Close input HDF5: f_in.close()
def main(argv): """Try to guess the amount of overlap in the case of extended FOV CT. Parameters ---------- infile : array_like HDF5 input dataset outfile : string Full path where the identified overlap will be written as output scale : int If sub-pixel precision is interesting, use e.g. 2.0 to get an overlap of .5 value. Use 1.0 if sub-pixel precision is not required tmppath : int Temporary path where look for cached flat/dark files """ # Get path: infile = argv[0] # The HDF5 file on the SSD outfile = argv[1] # The txt file with the proposed center scale = float(argv[2]) tmppath = argv[3] if not tmppath.endswith(sep): tmppath += sep # Create a silly temporary log: tmplog = tmppath + basename(infile) + str(time.time()) # Open the HDF5 file: f_in = getHDF5(infile, 'r') if "/tomo" in f_in: dset = f_in['tomo'] else: dset = f_in['exchange/data'] num_proj = tdf.get_nr_projs(dset) # Get first and 180 deg projections: im1 = tdf.read_tomo(dset, 0).astype(float32) im2 = tdf.read_tomo(dset, num_proj / 2).astype(float32) # Get flats and darks from cache or from file: try: corrplan = cache2plan(infile, tmppath) except Exception as e: #print "Error(s) when reading from cache" corrplan = extract_flatdark(f_in, True, tmplog) remove(tmplog) plan2cache(corrplan, infile, tmppath) # Apply simple flat fielding (if applicable): if (isinstance(corrplan['im_flat_after'], ndarray) and isinstance(corrplan['im_flat'], ndarray) and isinstance(corrplan['im_dark'], ndarray) and isinstance(corrplan['im_dark_after'], ndarray)): im1 = ((abs(im1 - corrplan['im_dark'])) / (abs(corrplan['im_flat'] - corrplan['im_dark']) + finfo(float32).eps)).astype(float32) im2 = ((abs(im2 - corrplan['im_dark_after'])) / (abs(corrplan['im_flat_after'] - corrplan['im_dark_after']) + finfo(float32).eps)).astype(float32) # Scale projections (if required) to get subpixel estimation: if (abs(scale - 1.0) > finfo(float32).eps): im1 = imresize(im1, (int(round( scale * im1.shape[0])), int(round(scale * im1.shape[1]))), interp='bicubic', mode='F') im2 = imresize(im2, (int(round( scale * im2.shape[0])), int(round(scale * im2.shape[1]))), interp='bicubic', mode='F') # Find the center (flipping left-right im2): DISTINGUISH BETWEEN AIR ON THE RIGHT AND ON THE LEFT?????? cen = findcenter.usecorrelation(im1, im2[:, ::-1]) cen = (cen / scale) * 2.0 # Print center to output file: text_file = open(outfile, "w") text_file.write(str(int(abs(cen)))) text_file.close() # Close input HDF5: f_in.close()
def process(sino_idx, num_sinos, infile, outfile, preprocessing_required, corr_plan, skipflat, norm_sx, norm_dx, flat_end, half_half, half_half_line, ext_fov, ext_fov_rot_right, ext_fov_overlap, ext_fov_normalize, ext_fov_average, ringrem, phaseretrieval_required, phrtmethod, phrt_param1, phrt_param2, energy, distance, pixsize, phrtpad, approx_win, angles, angles_projfrom, angles_projto, offset, logtransform, recpar, circle, scale, pad, method, rolling, roll_shift, zerone_mode, dset_min, dset_max, decim_factor, downsc_factor, corr_offset, postprocess_required, polarfilt_opt, convert_opt, crop_opt, dynamic_ff, EFF, filtEFF, im_dark, nr_threads, logfilename, tmppath): """To do... """ # Perform reconstruction (on-the-fly preprocessing and phase retrieval, if # required): if (phaseretrieval_required): # In this case a bunch of sinograms is loaded into memory: # # Load the temporary data structure reading the input TDF file. # To know the right dimension the first sinogram is pre-processed. # # Open the TDF file and get the dataset: f_in = getHDF5(infile, 'r') if "/tomo" in f_in: dset = f_in['tomo'] else: dset = f_in['exchange/data'] # Downscaling and decimation factors considered when determining the # approximation window: zrange = arange(sino_idx - approx_win * downsc_factor / 2, sino_idx + approx_win * downsc_factor / 2, downsc_factor) zrange = zrange[(zrange >= 0)] zrange = zrange[(zrange < num_sinos)] approx_win = zrange.shape[0] # Approximation window cannot be odd: if (approx_win % 2 == 1): approx_win = approx_win - 1 zrange = zrange[0:approx_win] # Read one sinogram to get the proper dimensions: test_im = tdf.read_sino(dset, zrange[0]*downsc_factor).astype(float32) # Apply projection removal (if required): test_im = test_im[angles_projfrom:angles_projto, :] # Apply decimation and downscaling (if required): test_im = test_im[::decim_factor, ::downsc_factor] # Perform the pre-processing of the first sinogram to get the right # dimension: if (preprocessing_required): if not skipflat: if dynamic_ff: # Dynamic flat fielding with downsampling = 2: #test_im = dynamic_flat_fielding(test_im, zrange[0] / downsc_factor, EFF, filtEFF, 2, im_dark, norm_sx, norm_dx) test_im = dynamic_flat_fielding(test_im, zrange[0] , EFF, filtEFF, 2, im_dark, norm_sx, norm_dx) else: #test_im = flat_fielding(test_im, zrange[0] / downsc_factor, corr_plan, flat_end, half_half, # half_half_line / decim_factor, norm_sx, norm_dx).astype(float32) test_im = flat_fielding(test_im, zrange[0], corr_plan, flat_end, half_half, half_half_line / decim_factor, norm_sx, norm_dx).astype(float32) if ext_fov: test_im = extfov_correction(test_im, ext_fov_rot_right, ext_fov_overlap / downsc_factor, ext_fov_normalize, ext_fov_average).astype(float32) if not skipflat and not dynamic_ff: test_im = ring_correction(test_im, ringrem, flat_end, corr_plan['skip_flat_after'], half_half, half_half_line / decim_factor, ext_fov).astype(float32) else: test_im = ring_correction(test_im, ringrem, False, False, half_half, half_half_line / decim_factor, ext_fov).astype(float32) # Now we can allocate memory for the bunch of slices: tmp_im = empty((approx_win, test_im.shape[0], test_im.shape[1]), dtype=float32) tmp_im[0,:,:] = test_im # Reading all the the sinos from TDF file and close: for ct in range(1, approx_win): # Read the sinogram: test_im = tdf.read_sino(dset, zrange[ct]*downsc_factor).astype(float32) # Apply projection removal (if required): test_im = test_im[angles_projfrom:angles_projto, :] # Apply decimation and downscaling (if required): test_im = test_im[::decim_factor, ::downsc_factor] # Perform the pre-processing for each sinogram of the bunch: if (preprocessing_required): if not skipflat: if dynamic_ff: # Dynamic flat fielding with downsampling = 2: #test_im = dynamic_flat_fielding(test_im, zrange[ct] / downsc_factor, EFF, filtEFF, 2, im_dark, norm_sx, norm_dx) test_im = dynamic_flat_fielding(test_im, zrange[ct], EFF, filtEFF, 2, im_dark, norm_sx, norm_dx) else: #test_im = flat_fielding(test_im, zrange[ct] / downsc_factor, corr_plan, flat_end, half_half, # half_half_line / decim_factor, norm_sx, norm_dx).astype(float32) test_im = flat_fielding(test_im, zrange[ct], corr_plan, flat_end, half_half, half_half_line / decim_factor, norm_sx, norm_dx).astype(float32) if ext_fov: test_im = extfov_correction(test_im, ext_fov_rot_right, ext_fov_overlap / downsc_factor, ext_fov_normalize, ext_fov_average).astype(float32) if not skipflat and not dynamic_ff: test_im = ring_correction(test_im, ringrem, flat_end, corr_plan['skip_flat_after'], half_half, half_half_line / decim_factor, ext_fov).astype(float32) else: test_im = ring_correction(test_im, ringrem, False, False, half_half, half_half_line / decim_factor, ext_fov).astype(float32) tmp_im[ct,:,:] = test_im f_in.close() # Now everything has to refer to a downscaled dataset: sino_idx = ((zrange == sino_idx).nonzero()) # # Perform phase retrieval: # # Prepare the plan: if (phrtmethod == 0): # Paganin's: phrtplan = tiehom_plan(tmp_im[:,0,:], phrt_param1, phrt_param2, energy, distance, pixsize * downsc_factor, phrtpad) else: phrtplan = phrt_plan(tmp_im[:,0,:], energy, distance, pixsize * downsc_factor, phrt_param2, phrt_param1, phrtmethod, phrtpad) #phrtplan = prepare_plan (tmp_im[:,0,:], beta, delta, energy, distance, #pixsize*downsc_factor, padding=phrtpad) # Process each projection (whose height depends on the size of the bunch): for ct in range(0, tmp_im.shape[1]): #tmp_im[:,ct,:] = phase_retrieval(tmp_im[:,ct,:], phrtplan).astype(float32) if (phrtmethod == 0): tmp_im[:,ct,:] = tiehom(tmp_im[:,ct,:], phrtplan).astype(float32) else: tmp_im[:,ct,:] = phrt(tmp_im[:,ct,:], phrtplan, phrtmethod).astype(float32) # Extract the requested sinogram: im = tmp_im[sino_idx[0],:,:].squeeze() else: # Read only one sinogram: f_in = getHDF5(infile, 'r') if "/tomo" in f_in: dset = f_in['tomo'] else: dset = f_in['exchange/data'] im = tdf.read_sino(dset,sino_idx * downsc_factor).astype(float32) f_in.close() # Apply projection removal (if required): im = im[angles_projfrom:angles_projto, :] # Apply decimation and downscaling (if required): im = im[::decim_factor,::downsc_factor] #sino_idx = sino_idx / downsc_factor # Downscaling for the index already applied # Perform the preprocessing of the sinogram (if required): if (preprocessing_required): if not skipflat: if dynamic_ff: # Dynamic flat fielding with downsampling = 2: im = dynamic_flat_fielding(im, sino_idx, EFF, filtEFF, 2, im_dark, norm_sx, norm_dx) else: im = flat_fielding(im, sino_idx, corr_plan, flat_end, half_half, half_half_line / decim_factor, norm_sx, norm_dx).astype(float32) if ext_fov: im = extfov_correction(im, ext_fov_rot_right, ext_fov_overlap / downsc_factor, ext_fov_normalize, ext_fov_average) if not skipflat and not dynamic_ff: im = ring_correction(im, ringrem, flat_end, corr_plan['skip_flat_after'], half_half, half_half_line / decim_factor, ext_fov) else: im = ring_correction(im, ringrem, False, False, half_half, half_half_line / decim_factor, ext_fov) # Additional ring removal before reconstruction: #im = boinhaibel(im, '11;') #im = munchetal(im, '5;1.8') #im = rivers(im, '13;') #im = raven(im, '11;0.8') #im = oimoen(im, '51;51') # Actual reconstruction: im = reconstruct(im, angles, offset / downsc_factor, logtransform, recpar, circle, scale, pad, method, zerone_mode, dset_min, dset_max, corr_offset, rolling, roll_shift, tmppath).astype(float32) # Apply post-processing (if required): if postprocess_required: im = polarfilter(im, polarfilt_opt) im = croprescale(im, convert_opt, crop_opt) else: # Create the circle mask for fancy output: if (circle == True): siz = im.shape[1] if siz % 2: rang = arange(-siz / 2 + 1, siz / 2 + 1) else: rang = arange(-siz / 2,siz / 2) x,y = meshgrid(rang,rang) z = x ** 2 + y ** 2 a = (z < (siz / 2 - int(round(abs(offset) / downsc_factor))) ** 2) im = im * a # Write down reconstructed preview file (file name modified with metadata): im = im.astype(float32) outfile = outfile + '_' + str(im.shape[1]) + 'x' + str(im.shape[0]) + '_' + str(amin(im)) + '$' + str(amax(im)) im.tofile(outfile)
def process_gridrec(lock, int_from, int_to, num_sinos, infile, outpath, preprocessing_required, skipflat, corr_plan, norm_sx, norm_dx, flat_end, half_half, half_half_line, ext_fov, ext_fov_rot_right, ext_fov_overlap, ext_fov_normalize, ext_fov_average, ringrem, angles, angles_projfrom, angles_projto, offset, logtransform, param1, circle, scale, pad, rolling, roll_shift, zerone_mode, dset_min, dset_max, decim_factor, downsc_factor, corr_offset, postprocess_required, polarfilt_opt, convert_opt, crop_opt, dynamic_ff, EFF, filtEFF, im_dark, outprefix, logfilename): """To do... """ # Process the required subset of images: for i in range(int_from, int_to + 1, 2): # Read two sinograms: t0 = time() f_in = getHDF5(infile, 'r') if "/tomo" in f_in: dset = f_in['tomo'] else: dset = f_in['exchange/data'] im1 = tdf.read_sino(dset, i * downsc_factor).astype(float32) if ((i + downsc_factor) <= (int_to + 1)): im2 = tdf.read_sino(dset, i * downsc_factor + downsc_factor).astype(float32) else: im2 = im1 f_in.close() t1 = time() # Apply projection removal (if required): im1 = im1[angles_projfrom:angles_projto, :] im2 = im2[angles_projfrom:angles_projto, :] # Apply decimation and downscaling (if required): im1 = im1[::decim_factor, ::downsc_factor] im2 = im2[::decim_factor, ::downsc_factor] #i = i / downsc_factor # Perform the preprocessing of the sinograms (if required): if (preprocessing_required): if not skipflat: if dynamic_ff: # Dynamic flat fielding with downsampling = 2: im1 = dynamic_flat_fielding(im1, i, EFF, filtEFF, 2, im_dark, norm_sx, norm_dx) else: im1 = flat_fielding(im1, i, corr_plan, flat_end, half_half, half_half_line / decim_factor, norm_sx, norm_dx).astype(float32) if ext_fov: im1 = extfov_correction(im1, ext_fov_rot_right, ext_fov_overlap / downsc_factor, ext_fov_normalize, ext_fov_average) if not skipflat: im1 = ring_correction(im1, ringrem, flat_end, corr_plan['skip_flat_after'], half_half, half_half_line / decim_factor, ext_fov) else: im1 = ring_correction(im1, ringrem, False, False, half_half, half_half_line / decim_factor, ext_fov) if not skipflat: if dynamic_ff: # Dynamic flat fielding with downsampling = 2: im2 = dynamic_flat_fielding(im2, i + 1, EFF, filtEFF, 2, im_dark, norm_sx, norm_dx) else: im2 = flat_fielding(im2, i + 1, corr_plan, flat_end, half_half, half_half_line / decim_factor, norm_sx, norm_dx).astype(float32) if ext_fov: im2 = extfov_correction(im2, ext_fov_rot_right, ext_fov_overlap / downsc_factor, ext_fov_normalize, ext_fov_average) if not skipflat and not dynamic_ff: im2 = ring_correction(im2, ringrem, flat_end, corr_plan['skip_flat_after'], half_half, half_half_line / decim_factor, ext_fov) else: im2 = ring_correction(im2, ringrem, False, False, half_half, half_half_line, ext_fov) # Actual reconstruction: [im1, im2] = reconstruct_gridrec(im1, im2, angles, offset / downsc_factor, logtransform, param1, circle, scale, pad, rolling, roll_shift, zerone_mode, dset_min, dset_max, decim_factor, downsc_factor, corr_offset) # Appy post-processing (if required): if postprocess_required: # Filter (if required): im1 = polarfilter(im1, polarfilt_opt) im2 = polarfilter(im2, polarfilt_opt) im1 = croprescale(im1, convert_opt, crop_opt, circle) im2 = croprescale(im2, convert_opt, crop_opt, circle) else: # Create the circle mask for fancy output: if (circle == True): siz = im1.shape[1] if siz % 2: rang = arange(-siz / 2 + 1, siz / 2 + 1) else: rang = arange(-siz / 2, siz / 2) x, y = meshgrid(rang, rang) z = x**2 + y**2 a = (z < (siz / 2 - int(round(abs(offset) / downsc_factor)))**2) im1 = im1 * a im2 = im2 * a # Write down reconstructed slices: t2 = time() fname1 = outpath + outprefix + '_' + str(i).zfill(4) + '.tif' imsave(fname1, im1) fname2 = outpath + outprefix + '_' + str(i + 1).zfill(4) + '.tif' imsave(fname2, im2) t3 = time() # Write log (atomic procedure - lock used): write_log_gridrec(lock, fname1, fname2, logfilename, t2 - t1, (t3 - t2) + (t1 - t0))
def process_gridrec(lock, int_from, int_to, num_sinos, infile, outpath, preprocessing_required, skipflat, corr_plan, norm_sx, norm_dx, flat_end, half_half, half_half_line, ext_fov, ext_fov_rot_right, ext_fov_overlap, ext_fov_normalize, ext_fov_average, ringrem, angles, angles_projfrom, angles_projto, offset, logtransform, param1, circle, scale, pad, rolling, roll_shift, zerone_mode, dset_min, dset_max, decim_factor, downsc_factor, corr_offset, postprocess_required, polarfilt_opt, convert_opt, crop_opt, dynamic_ff, EFF, filtEFF, im_dark, outprefix, logfilename): """To do... """ # Process the required subset of images: for i in range(int_from, int_to + 1, 2): # Read two sinograms: t0 = time() f_in = getHDF5(infile, 'r') if "/tomo" in f_in: dset = f_in['tomo'] else: dset = f_in['exchange/data'] im1 = tdf.read_sino(dset,i*downsc_factor).astype(float32) if ( (i + downsc_factor) <= (int_to + 1) ): im2 = tdf.read_sino(dset,i*downsc_factor + downsc_factor).astype(float32) else: im2 = im1 f_in.close() t1 = time() # Apply projection removal (if required): im1 = im1[angles_projfrom:angles_projto, :] im2 = im2[angles_projfrom:angles_projto, :] # Apply decimation and downscaling (if required): im1 = im1[::decim_factor,::downsc_factor] im2 = im2[::decim_factor,::downsc_factor] #i = i / downsc_factor # Perform the preprocessing of the sinograms (if required): if (preprocessing_required): if not skipflat: if dynamic_ff: # Dynamic flat fielding with downsampling = 2: im1 = dynamic_flat_fielding(im1, i, EFF, filtEFF, 2, im_dark, norm_sx, norm_dx) else: im1 = flat_fielding (im1, i, corr_plan, flat_end, half_half, half_half_line / decim_factor, norm_sx, norm_dx).astype(float32) if ext_fov: im1 = extfov_correction (im1, ext_fov_rot_right, ext_fov_overlap / downsc_factor, ext_fov_normalize, ext_fov_average) if not skipflat: im1 = ring_correction (im1, ringrem, flat_end, corr_plan['skip_flat_after'], half_half, half_half_line / decim_factor, ext_fov) else: im1 = ring_correction (im1, ringrem, False, False, half_half, half_half_line / decim_factor, ext_fov) if not skipflat: if dynamic_ff: # Dynamic flat fielding with downsampling = 2: im2 = dynamic_flat_fielding(im2, i + 1, EFF, filtEFF, 2, im_dark, norm_sx, norm_dx) else: im2 = flat_fielding (im2, i + 1, corr_plan, flat_end, half_half, half_half_line / decim_factor, norm_sx, norm_dx).astype(float32) if ext_fov: im2 = extfov_correction (im2, ext_fov_rot_right, ext_fov_overlap / downsc_factor, ext_fov_normalize, ext_fov_average) if not skipflat and not dynamic_ff: im2 = ring_correction (im2, ringrem, flat_end, corr_plan['skip_flat_after'], half_half, half_half_line / decim_factor, ext_fov) else: im2 = ring_correction (im2, ringrem, False, False, half_half, half_half_line, ext_fov) # Actual reconstruction: [im1, im2] = reconstruct_gridrec(im1, im2, angles, offset / downsc_factor, logtransform, param1, circle, scale, pad, rolling, roll_shift, zerone_mode, dset_min, dset_max, decim_factor, downsc_factor, corr_offset) # Appy post-processing (if required): if postprocess_required: # Filter (if required): im1 = polarfilter(im1, polarfilt_opt) im2 = polarfilter(im2, polarfilt_opt) im1 = croprescale(im1, convert_opt, crop_opt, circle) im2 = croprescale(im2, convert_opt, crop_opt, circle) else: # Create the circle mask for fancy output: if (circle == True): siz = im1.shape[1] if siz % 2: rang = arange(-siz / 2 + 1, siz / 2 + 1) else: rang = arange(-siz / 2,siz / 2) x,y = meshgrid(rang,rang) z = x ** 2 + y ** 2 a = (z < (siz / 2 - int(round(abs(offset)/downsc_factor)) ) ** 2) im1 = im1 * a im2 = im2 * a # Write down reconstructed slices: t2 = time() fname1 = outpath + outprefix + '_' + str(i).zfill(4) + '.tif' imsave(fname1, im1) fname2 = outpath + outprefix + '_' + str(i + 1).zfill(4) + '.tif' imsave(fname2, im2) t3 = time() # Write log (atomic procedure - lock used): write_log_gridrec(lock, fname1, fname2, logfilename, t2 - t1, (t3 - t2) + (t1 - t0) )
def process(lock, int_from, int_to, num_sinos, infile, outpath, preprocessing_required, skipflat, corr_plan, norm_sx, norm_dx, flat_end, half_half, half_half_line, ext_fov, ext_fov_rot_right, ext_fov_overlap, ext_fov_normalize, ext_fov_average, ringrem, angles, angles_projfrom, angles_projto, offset, logtransform, param1, circle, scale, pad, method, rolling, roll_shift, zerone_mode, dset_min, dset_max, decim_factor, downsc_factor, corr_offset, postprocess_required, polarfilt_opt, convert_opt, crop_opt, dynamic_ff, EFF, filtEFF, im_dark, outprefix, logfilename): """To do... """ # Process the required subset of images: for i in range(int_from, int_to + 1): # Perform reconstruction (on-the-fly preprocessing and phase retrieval, if # required): #if (phaseretrieval_required): # # Load into memory a bunch of sinograms: # t0 = time() # # Open the TDF file for reading: # f_in = getHDF5(infile, 'r') # if "/tomo" in f_in: # dset = f_in['tomo'] # else: # dset = f_in['exchange/data'] # # Prepare the data structure according to the approximation window: # tmp_im = numpy.empty((tdf.get_nr_projs(dset),tdf.get_det_size(dset), # approx_win), dtype=float32) # # Load the temporary data structure reading the input TDF file: # # (It can be parallelized Open-MP style) # ct = 0 # for j in range(i - approx_win/2, i + approx_win/2 + 1): # if (j < 0): # j = 0 # if (j >= num_sinos): # j = num_sinos - 1 # a = tdf.read_sino(dset,j).astype(float32) # tmp_im[:,:,ct] = a # ct = ct + 1 # # Close the TDF file: # f_in.close() # t1 = time() # # Perform the processing: # if (preprocessing_required): # ct = 0 # # (It can be parallelized Open-MP style) # for j in range(i - approx_win/2, i + approx_win/2 + 1): # if (j < 0): # j = 0 # if (j >= num_sinos): # j = num_sinos - 1 # tmp_im[:,:,ct] = flat_fielding (tmp_im[:,:,ct], j, corr_plan, flat_end, # half_half, half_half_line, norm_sx, norm_dx).astype(float32) # tmp_im[:,:,ct] = extfov_correction (tmp_im[:,:,ct], ext_fov, # ext_fov_rot_right, ext_fov_overlap).astype(float32) # tmp_im[:,:,ct] = ring_correction (tmp_im[:,:,ct], ringrem, flat_end, # corr_plan['skip_flat_after'], half_half, half_half_line, # ext_fov).astype(float32) # ct = ct + 1 # # Perform phase retrieval: # # (It can be parallelized Open-MP style) # for ct in range(0, tmp_im.shape[0]): # tmp_im[ct,:,:] = phase_retrieval(tmp_im[ct,:,:].T, # phrt_plan).astype(float32).T # ct = ct + 1 # # Extract the central processed sinogram: # im = tmp_im[:,:,approx_win/2] #else: # Read only one sinogram: t0 = time() f_in = getHDF5(infile, 'r') if "/tomo" in f_in: dset = f_in['tomo'] else: dset = f_in['exchange/data'] im = tdf.read_sino(dset, i * downsc_factor).astype(float32) f_in.close() t1 = time() # Apply projection removal (if required): im = im[angles_projfrom:angles_projto, :] # Apply decimation and downscaling (if required): im = im[::decim_factor, ::downsc_factor] #i = i / downsc_factor # Perform the preprocessing of the sinogram (if required): if (preprocessing_required): if not skipflat: if dynamic_ff: # Dynamic flat fielding with downsampling = 2: im = dynamic_flat_fielding(im, i, EFF, filtEFF, 2, im_dark, norm_sx, norm_dx).astype(float32) else: im = flat_fielding(im, i, corr_plan, flat_end, half_half, half_half_line / decim_factor, norm_sx, norm_dx).astype(float32) if ext_fov: im = extfov_correction(im, ext_fov_rot_right, ext_fov_overlap / downsc_factor, ext_fov_normalize, ext_fov_average) if not skipflat and not dynamic_ff: im = ring_correction(im, ringrem, flat_end, corr_plan['skip_flat_after'], half_half, half_half_line / decim_factor, ext_fov) else: im = ring_correction(im, ringrem, False, False, half_half, half_half_line, ext_fov) # Actual reconstruction: im = reconstruct(im, angles, offset / downsc_factor, logtransform, param1, circle, scale, pad, method, rolling, roll_shift, zerone_mode, dset_min, dset_max, decim_factor, downsc_factor, corr_offset).astype(float32) # Apply post-processing (if required): if postprocess_required: im = polarfilter(im, polarfilt_opt) im = croprescale(im, convert_opt, crop_opt) else: # Create the circle mask for fancy output: if (circle == True): siz = im.shape[1] if siz % 2: rang = arange(-siz / 2 + 1, siz / 2 + 1) else: rang = arange(-siz / 2, siz / 2) x, y = meshgrid(rang, rang) z = x**2 + y**2 a = (z < (siz / 2 - abs(offset))**2) im = im * a # Write down reconstructed slice: t2 = time() fname = outpath + outprefix + '_' + str(i).zfill(4) + '.tif' imsave(fname, im) t3 = time() # Write log (atomic procedure - lock used): write_log(lock, fname, logfilename, t2 - t1, (t3 - t2) + (t1 - t0))
def main(argv): """To do... """ lock = Lock() skip_flat = True first_done = False pyfftw_cache_disable() pyfftw_cache_enable() pyfftw_set_keepalive_time(1800) # Get the from and to number of files to process: int_from = int(argv[0]) int_to = int(argv[1]) # Get full paths of input TDF and output TDF: infile = argv[2] outfile = argv[3] # Get the phase retrieval parameters: method = int(argv[4]) param1 = double(argv[5]) # e.g. regParam, or beta param2 = double(argv[6]) # e.g. thresh or delta energy = double(argv[7]) distance = double(argv[8]) pixsize = double(argv[9]) / 1000.0 # pixsixe from micron to mm: pad = True if argv[10] == "True" else False # Number of threads (actually processes) to use and logfile: nr_threads = int(argv[11]) logfilename = argv[12] # Log infos: log = open(logfilename, "w") log.write(linesep + "\tInput TDF file: %s" % (infile)) log.write(linesep + "\tOutput TDF file: %s" % (outfile)) log.write(linesep + "\t--------------") if (method == 0): log.write(linesep + "\tMethod: TIE-Hom (Paganin et al., 2002)") log.write(linesep + "\t--------------") log.write(linesep + "\tDelta/Beta: %0.1f" % ((param2 / param1))) elif (method == 1): log.write(linesep + "\tMethod: Generalized TIE-Hom (Paganin et al., 2020)") log.write(linesep + "\t--------------") log.write(linesep + "\tDelta/Beta: %0.1f" % ((param2 / param1))) else: log.write(linesep + "\tMethod: Projected CTF (Moosmann et al., 2011)") log.write(linesep + "\t--------------") log.write(linesep + "\tRegularization: %0.3f" % (param2)) log.write(linesep + "\tThreshold: %0.3f" % (param1)) log.write(linesep + "\tEnergy: %0.1f keV" % (energy)) log.write(linesep + "\tDistance: %0.1f mm" % (distance)) log.write(linesep + "\tPixel size: %0.3f micron" % (pixsize * 1000)) log.write(linesep + "\t--------------") log.write(linesep + "\tBrowsing input files...") log.close() # Remove a previous copy of output: if exists(outfile): remove(outfile) # Open the HDF5 file: f_in = getHDF5(infile, 'r') if "/tomo" in f_in: dset = f_in['tomo'] else: dset = f_in['exchange/data'] num_proj = tdf.get_nr_projs(dset) num_sinos = tdf.get_nr_sinos(dset) if (num_proj == 0): log = open(logfilename, "a") log.write(linesep + "\tNo projections found. Process will end.") log.close() exit() log = open(logfilename, "a") log.write(linesep + "\tInput files browsed correctly.") log.close() # Check extrema (int_to == -1 means all files): if ((int_to >= num_proj) or (int_to == -1)): int_to = num_proj - 1 if ((int_from < 0)): int_from = 0 # Prepare the plan: log = open(logfilename, "a") log.write(linesep + "\tPreparing the work plan...") log.close() im = tdf.read_tomo(dset, 0).astype(float32) outshape = tdf.get_dset_shape(im.shape[1], im.shape[0], num_proj) f_out = getHDF5(outfile, 'w') f_out_dset = f_out.create_dataset('exchange/data', outshape, float32) f_out_dset.attrs['min'] = str(amin(im[:])) f_out_dset.attrs['max'] = str(amax(im[:])) f_out_dset.attrs['version'] = '1.0' f_out_dset.attrs['axes'] = "y:theta:x" f_in.close() f_out.close() if (method == 0): # Paganin 2020: plan = tiehom_plan(im, param1, param2, energy, distance, pixsize, pad) elif (method == 1): # Paganin 2020: plan = tiehom_plan2020(im, param1, param2, energy, distance, pixsize, pad) else: plan = phrt_plan(im, energy, distance, pixsize, param2, param1, method, pad) # Run several threads for independent computation without waiting for threads completion: for num in range(nr_threads): start = (num_proj / nr_threads) * num if (num == nr_threads - 1): end = num_proj - 1 else: end = (num_proj / nr_threads) * (num + 1) - 1 Process(target=_process, args=(lock, start, end, infile, outfile, outshape, float32, method, plan, logfilename)).start()
def main(argv): """To do... Usage ----- Parameters --------- Example -------------------------- The following line processes the first ten TIFF files of input path "/home/in" and saves the processed files to "/home/out" with the application of the Boin and Haibel filter with smoothing via a Butterworth filter of order 4 and cutoff frequency 0.01: reconstruct 0 4 C:\Temp\Dullin_Aug_2012\sino_noflat C:\Temp\Dullin_Aug_2012\sino_noflat\output 9.0 10.0 0.0 0.0 0.0 true sino slice C:\Temp\Dullin_Aug_2012\sino_noflat\tomo_conv flat dark """ lock = Lock() skip_flat = False skip_flat_after = True # Get the from and to number of files to process: int_from = int(argv[0]) int_to = int(argv[1]) # Get paths: infile = argv[2] outpath = argv[3] # Essential reconstruction parameters: angles = float(argv[4]) offset = float(argv[5]) param1 = argv[6] scale = int(float(argv[7])) overpad = True if argv[8] == "True" else False logtrsf = True if argv[9] == "True" else False circle = True if argv[10] == "True" else False outprefix = argv[11] # Parameters for on-the-fly pre-processing: preprocessing_required = True if argv[12] == "True" else False flat_end = True if argv[13] == "True" else False half_half = True if argv[14] == "True" else False half_half_line = int(argv[15]) ext_fov = True if argv[16] == "True" else False norm_sx = int(argv[21]) norm_dx = int(argv[22]) ext_fov_rot_right = argv[17] if ext_fov_rot_right == "True": ext_fov_rot_right = True if (ext_fov): norm_sx = 0 else: ext_fov_rot_right = False if (ext_fov): norm_dx = 0 ext_fov_overlap = int(argv[18]) ext_fov_normalize = True if argv[19] == "True" else False ext_fov_average = True if argv[20] == "True" else False skip_ringrem = True if argv[23] == "True" else False ringrem = argv[24] # Extra reconstruction parameters: zerone_mode = True if argv[25] == "True" else False corr_offset = float(argv[26]) reconmethod = argv[27] # Force overpadding in case of GRIDREC for unknown reasons: if reconmethod == "GRIDREC": overpad = True decim_factor = int(argv[28]) downsc_factor = int(argv[29]) # Parameters for postprocessing: postprocess_required = True if argv[30] == "True" else False polarfilt_opt = argv[31] convert_opt = argv[32] crop_opt = argv[33] angles_projfrom = int(argv[34]) angles_projto = int(argv[35]) rolling = True if argv[36] == "True" else False roll_shift = int(int(argv[37]) / decim_factor) dynamic_ff = True if argv[38] == "True" else False nr_threads = int(argv[39]) logfilename = argv[40] process_id = int(logfilename[-6:-4]) # Check prefixes and path: #if not infile.endswith(sep): infile += sep if not exists(outpath): makedirs(outpath) if not outpath.endswith(sep): outpath += sep # Open the HDF5 file: f_in = getHDF5(infile, 'r') if "/tomo" in f_in: dset = f_in['tomo'] tomoprefix = 'tomo' flatprefix = 'flat' darkprefix = 'dark' else: dset = f_in['exchange/data'] if "/provenance/detector_output" in f_in: prov_dset = f_in['provenance/detector_output'] tomoprefix = prov_dset.attrs['tomo_prefix'] flatprefix = prov_dset.attrs['flat_prefix'] darkprefix = prov_dset.attrs['dark_prefix'] dset_min = -1 dset_max = -1 if (zerone_mode): if ('min' in dset.attrs): dset_min = float(dset.attrs['min']) else: zerone_mode = False if ('max' in dset.attrs): dset_max = float(dset.attrs['max']) else: zerone_mode = False num_sinos = tdf.get_nr_sinos(dset) # Pay attention to the downscale factor if (num_sinos == 0): log = open(logfilename, "a") log.write(linesep + "\tNo projections found. Process will end.") log.close() exit() # Check extrema (int_to == -1 means all files): if ((int_to >= num_sinos / downsc_factor) or (int_to == -1)): int_to = num_sinos / downsc_factor - 1 # Log info: log = open(logfilename, "w") log.write(linesep + "\tInput file: %s" % (infile)) log.write(linesep + "\tOutput path: %s" % (outpath)) log.write(linesep + "\t--------------") log.write(linesep + "\tPreparing the work plan...") log.close() # Get correction plan and phase retrieval plan (if required): corrplan = -1 phrtplan = -1 skipflat = False im_dark = -1 EFF = -1 filtEFF = -1 if (preprocessing_required): if not dynamic_ff: # Load flat fielding plan either from cache (if required) or from TDF file # and cache it for faster re-use: corrplan = extract_flatdark(f_in, flat_end, logfilename) if (isscalar(corrplan['im_flat']) and isscalar(corrplan['im_flat_after'])): skipflat = True # Dowscale flat and dark images if necessary: if isinstance(corrplan['im_flat'], ndarray): corrplan['im_flat'] = corrplan[ 'im_flat'][::downsc_factor, ::downsc_factor] if isinstance(corrplan['im_dark'], ndarray): corrplan['im_dark'] = corrplan[ 'im_dark'][::downsc_factor, ::downsc_factor] if isinstance(corrplan['im_flat_after'], ndarray): corrplan['im_flat_after'] = corrplan[ 'im_flat_after'][::downsc_factor, ::downsc_factor] if isinstance(corrplan['im_dark_after'], ndarray): corrplan['im_dark_after'] = corrplan[ 'im_dark_after'][::downsc_factor, ::downsc_factor] else: # Dynamic flat fielding: if "/tomo" in f_in: if "/flat" in f_in: flat_dset = f_in['flat'] if "/dark" in f_in: im_dark = _medianize(f_in['dark']) else: skipdark = True else: skipflat = True # Nothing to do in this case else: if "/exchange/data_white" in f_in: flat_dset = f_in['/exchange/data_white'] if "/exchange/data_dark" in f_in: im_dark = _medianize(f_in['/exchange/data_dark']) else: skipdark = True else: skipflat = True # Nothing to do in this case # Prepare plan for dynamic flat fielding with 16 repetitions: if not skipflat: EFF, filtEFF = dff_prepare_plan(flat_dset, 16, im_dark) # Downscale images if necessary: im_dark = im_dark[::downsc_factor, ::downsc_factor] EFF = EFF[::downsc_factor, ::downsc_factor, :] filtEFF = filtEFF[::downsc_factor, ::downsc_factor, :] f_in.close() # Log infos: log = open(logfilename, "a") log.write(linesep + "\tWork plan prepared correctly.") log.write(linesep + "\t--------------") log.write(linesep + "\tPerforming reconstruction...") log.close() # Run several threads for independent computation without waiting for threads # completion: for num in range(nr_threads): start = ((int_to - int_from + 1) / nr_threads) * num + int_from if (num == nr_threads - 1): end = int_to else: end = ((int_to - int_from + 1) / nr_threads) * (num + 1) + int_from - 1 if (reconmethod == 'GRIDREC'): Process( target=process_gridrec, args=(lock, start, end, num_sinos, infile, outpath, preprocessing_required, skipflat, corrplan, norm_sx, norm_dx, flat_end, half_half, half_half_line, ext_fov, ext_fov_rot_right, ext_fov_overlap, ext_fov_normalize, ext_fov_average, ringrem, angles, angles_projfrom, angles_projto, offset, logtrsf, param1, circle, scale, overpad, rolling, roll_shift, zerone_mode, dset_min, dset_max, decim_factor, downsc_factor, corr_offset, postprocess_required, polarfilt_opt, convert_opt, crop_opt, dynamic_ff, EFF, filtEFF, im_dark, outprefix, logfilename)).start() else: Process( target=process, args=(lock, start, end, num_sinos, infile, outpath, preprocessing_required, skipflat, corrplan, norm_sx, norm_dx, flat_end, half_half, half_half_line, ext_fov, ext_fov_rot_right, ext_fov_overlap, ext_fov_normalize, ext_fov_average, ringrem, angles, angles_projfrom, angles_projto, offset, logtrsf, param1, circle, scale, overpad, reconmethod, rolling, roll_shift, zerone_mode, dset_min, dset_max, decim_factor, downsc_factor, corr_offset, postprocess_required, polarfilt_opt, convert_opt, crop_opt, dynamic_ff, EFF, filtEFF, im_dark, outprefix, logfilename)).start()
def main(argv): """To do... Usage ----- Parameters --------- Example -------------------------- The following line processes the first ten TIFF files of input path "/home/in" and saves the processed files to "/home/out" with the application of the Boin and Haibel filter with smoothing via a Butterworth filter of order 4 and cutoff frequency 0.01: destripe /home/in /home/out 1 10 1 0.01 4 """ lock = Lock() rescale_factor = 10000.0 # For 16-bit floating point # Get the from and to number of files to process: int_from = int(argv[0]) int_to = int(argv[1]) # Get paths: infile = argv[2] outfile = argv[3] # Params for flat fielding with post flats/darks: flat_end = True if argv[4] == "True" else False half_half = True if argv[5] == "True" else False half_half_line = int(argv[6]) # Flat fielding method (conventional or dynamic): dynamic_ff = True if argv[7] == "True" else False # Parameters for rotation: rotation = float(argv[8]) interp = argv[9] border = argv[10] # Nr of threads and log file: nr_threads = int(argv[11]) logfilename = argv[12] # Log input parameters: log = open(logfilename, "w") log.write(linesep + "\tInput TDF file: %s" % (infile)) log.write(linesep + "\tOutput TDF file: %s" % (outfile)) log.write(linesep + "\t--------------") log.write(linesep + "\tOpening input dataset...") log.close() # Remove a previous copy of output: if exists(outfile): remove(outfile) # Open the HDF5 file: f_in = getHDF5(infile, 'r') if "/tomo" in f_in: dset = f_in['tomo'] tomoprefix = 'tomo' flatprefix = 'flat' darkprefix = 'dark' else: dset = f_in['exchange/data'] if "/provenance/detector_output" in f_in: prov_dset = f_in['provenance/detector_output'] tomoprefix = prov_dset.attrs['tomo_prefix'] flatprefix = prov_dset.attrs['flat_prefix'] darkprefix = prov_dset.attrs['dark_prefix'] num_proj = tdf.get_nr_projs(dset) num_sinos = tdf.get_nr_sinos(dset) if (num_sinos == 0): log = open(logfilename, "a") log.write(linesep + "\tNo projections found. Process will end.") log.close() exit() # Check extrema (int_to == -1 means all files): if ((int_to >= num_proj) or (int_to == -1)): int_to = num_proj - 1 # Prepare the work plan for flat and dark images: log = open(logfilename, "a") log.write(linesep + "\t--------------") log.write(linesep + "\tPreparing the work plan...") log.close() # Extract flat and darks: skipflat = False skipdark = False # Following variables make sense only for dynamic flat fielding: EFF = -1 filtEFF = -1 im_dark = -1 # Following variable makes sense only for conventional flat fielding: plan = -1 if not dynamic_ff: plan = extract_flatdark(f_in, flat_end, logfilename) if (isscalar(plan['im_flat']) and isscalar(plan['im_flat_after'])): skipflat = True else: skipflat = False else: # Dynamic flat fielding: if "/tomo" in f_in: if "/flat" in f_in: flat_dset = f_in['flat'] if "/dark" in f_in: im_dark = _medianize(f_in['dark']) else: skipdark = True else: skipflat = True # Nothing to do in this case else: if "/exchange/data_white" in f_in: flat_dset = f_in['/exchange/data_white'] if "/exchange/data_dark" in f_in: im_dark = _medianize(f_in['/exchange/data_dark']) else: skipdark = True else: skipflat = True # Nothing to do in this case # Prepare plan for dynamic flat fielding with 16 repetitions: if not skipflat: EFF, filtEFF = dff_prepare_plan(flat_dset, 16, im_dark) # Get the corrected outshape (in this case it's easy): im = tdf.read_tomo(dset, 0).astype(float32) outshape = tdf.get_dset_shape(im.shape[1], im.shape[0], num_proj) # Create the output HDF5 file: f_out = getHDF5(outfile, 'w') #f_out_dset = f_out.create_dataset('exchange/data', outshape, im.dtype) f_out_dset = f_out.create_dataset('exchange/data', outshape, float16) f_out_dset.attrs['min'] = str(amin(im[:])) f_out_dset.attrs['max'] = str(amax(im[:])) f_out_dset.attrs['version'] = '1.0' f_out_dset.attrs['axes'] = "y:theta:x" f_out_dset.attrs['rescale_factor'] = str(rescale_factor) f_out.close() f_in.close() # Log infos: log = open(logfilename, "a") log.write(linesep + "\tWork plan prepared correctly.") log.write(linesep + "\t--------------") log.write(linesep + "\tPerforming pre processing...") log.close() # Run several threads for independent computation without waiting for threads # completion: for num in range(nr_threads): start = (num_proj / nr_threads) * num if (num == nr_threads - 1): end = num_proj - 1 else: end = (num_proj / nr_threads) * (num + 1) - 1 Process(target=_process, args=(lock, start, end, infile, outfile, outshape, float16, skipflat, plan, flat_end, half_half, half_half_line, dynamic_ff, EFF, filtEFF, im_dark, rotation, interp, border, rescale_factor, logfilename)).start()
def main(argv): """To do... Usage ----- Parameters --------- Example -------------------------- The following line processes the first ten TIFF files of input path "/home/in" and saves the processed files to "/home/out" with the application of the Boin and Haibel filter with smoothing via a Butterworth filter of order 4 and cutoff frequency 0.01: reconstruct 0 4 C:\Temp\Dullin_Aug_2012\sino_noflat C:\Temp\Dullin_Aug_2012\sino_noflat\output 9.0 10.0 0.0 0.0 0.0 true sino slice C:\Temp\Dullin_Aug_2012\sino_noflat\tomo_conv flat dark """ lock = Lock() skip_flat = False skip_flat_after = True # Get the from and to number of files to process: int_from = int(argv[0]) int_to = int(argv[1]) # Get paths: infile = argv[2] outpath = argv[3] # Essential reconstruction parameters: angles = float(argv[4]) offset = float(argv[5]) param1 = argv[6] scale = int(float(argv[7])) overpad = True if argv[8] == "True" else False logtrsf = True if argv[9] == "True" else False circle = True if argv[10] == "True" else False outprefix = argv[11] # Parameters for on-the-fly pre-processing: preprocessing_required = True if argv[12] == "True" else False flat_end = True if argv[13] == "True" else False half_half = True if argv[14] == "True" else False half_half_line = int(argv[15]) ext_fov = True if argv[16] == "True" else False norm_sx = int(argv[21]) norm_dx = int(argv[22]) ext_fov_rot_right = argv[17] if ext_fov_rot_right == "True": ext_fov_rot_right = True if (ext_fov): norm_sx = 0 else: ext_fov_rot_right = False if (ext_fov): norm_dx = 0 ext_fov_overlap = int(argv[18]) ext_fov_normalize = True if argv[19] == "True" else False ext_fov_average = True if argv[20] == "True" else False skip_ringrem = True if argv[23] == "True" else False ringrem = argv[24] # Extra reconstruction parameters: zerone_mode = True if argv[25] == "True" else False corr_offset = float(argv[26]) reconmethod = argv[27] # Force overpadding in case of GRIDREC for unknown reasons: if reconmethod == "GRIDREC": overpad = True decim_factor = int(argv[28]) downsc_factor = int(argv[29]) # Parameters for postprocessing: postprocess_required = True if argv[30] == "True" else False polarfilt_opt = argv[31] convert_opt = argv[32] crop_opt = argv[33] angles_projfrom = int(argv[34]) angles_projto = int(argv[35]) rolling = True if argv[36] == "True" else False roll_shift = int(int(argv[37]) / decim_factor) dynamic_ff = True if argv[38] == "True" else False nr_threads = int(argv[39]) logfilename = argv[40] process_id = int(logfilename[-6:-4]) # Check prefixes and path: #if not infile.endswith(sep): infile += sep if not exists(outpath): makedirs(outpath) if not outpath.endswith(sep): outpath += sep # Open the HDF5 file: f_in = getHDF5(infile, 'r') if "/tomo" in f_in: dset = f_in['tomo'] tomoprefix = 'tomo' flatprefix = 'flat' darkprefix = 'dark' else: dset = f_in['exchange/data'] if "/provenance/detector_output" in f_in: prov_dset = f_in['provenance/detector_output'] tomoprefix = prov_dset.attrs['tomo_prefix'] flatprefix = prov_dset.attrs['flat_prefix'] darkprefix = prov_dset.attrs['dark_prefix'] dset_min = -1 dset_max = -1 if (zerone_mode): if ('min' in dset.attrs): dset_min = float(dset.attrs['min']) else: zerone_mode = False if ('max' in dset.attrs): dset_max = float(dset.attrs['max']) else: zerone_mode = False num_sinos = tdf.get_nr_sinos(dset) # Pay attention to the downscale factor if (num_sinos == 0): log = open(logfilename,"a") log.write(linesep + "\tNo projections found. Process will end.") log.close() exit() # Check extrema (int_to == -1 means all files): if ((int_to >= num_sinos / downsc_factor) or (int_to == -1)): int_to = num_sinos / downsc_factor - 1 # Log info: log = open(logfilename,"w") log.write(linesep + "\tInput file: %s" % (infile)) log.write(linesep + "\tOutput path: %s" % (outpath)) log.write(linesep + "\t--------------") log.write(linesep + "\tPreparing the work plan...") log.close() # Get correction plan and phase retrieval plan (if required): corrplan = -1 phrtplan = -1 skipflat = False im_dark = -1 EFF = -1 filtEFF = -1 if (preprocessing_required): if not dynamic_ff: # Load flat fielding plan either from cache (if required) or from TDF file and cache it for faster re-use: corrplan = extract_flatdark(f_in, flat_end, logfilename) if (isscalar(corrplan['im_flat']) and isscalar(corrplan['im_flat_after']) ): skipflat = True # Dowscale flat and dark images if necessary: if isinstance(corrplan['im_flat'], ndarray): corrplan['im_flat'] = corrplan['im_flat'][::downsc_factor,::downsc_factor] if isinstance(corrplan['im_dark'], ndarray): corrplan['im_dark'] = corrplan['im_dark'][::downsc_factor,::downsc_factor] if isinstance(corrplan['im_flat_after'], ndarray): corrplan['im_flat_after'] = corrplan['im_flat_after'][::downsc_factor,::downsc_factor] if isinstance(corrplan['im_dark_after'], ndarray): corrplan['im_dark_after'] = corrplan['im_dark_after'][::downsc_factor,::downsc_factor] else: # Dynamic flat fielding: if "/tomo" in f_in: if "/flat" in f_in: flat_dset = f_in['flat'] if "/dark" in f_in: im_dark = _medianize(f_in['dark']) else: skipdark = True else: skipflat = True # Nothing to do in this case else: if "/exchange/data_white" in f_in: flat_dset = f_in['/exchange/data_white'] if "/exchange/data_dark" in f_in: im_dark = _medianize(f_in['/exchange/data_dark']) else: skipdark = True else: skipflat = True # Nothing to do in this case # Prepare plan for dynamic flat fielding with 16 repetitions: if not skipflat: EFF, filtEFF = dff_prepare_plan(flat_dset, 16, im_dark) # Downscale images if necessary: im_dark = im_dark[::downsc_factor,::downsc_factor] EFF = EFF[::downsc_factor,::downsc_factor,:] filtEFF = filtEFF[::downsc_factor,::downsc_factor,:] f_in.close() # Log infos: log = open(logfilename,"a") log.write(linesep + "\tWork plan prepared correctly.") log.write(linesep + "\t--------------") log.write(linesep + "\tPerforming reconstruction...") log.close() # Run several threads for independent computation without waiting for threads completion: for num in range(nr_threads): start = ( (int_to - int_from + 1) / nr_threads)*num + int_from if (num == nr_threads - 1): end = int_to else: end = ( (int_to - int_from + 1) / nr_threads)*(num + 1) + int_from - 1 if (reconmethod == 'GRIDREC'): Process(target=process_gridrec, args=(lock, start, end, num_sinos, infile, outpath, preprocessing_required, skipflat, corrplan, norm_sx, norm_dx, flat_end, half_half, half_half_line, ext_fov, ext_fov_rot_right, ext_fov_overlap, ext_fov_normalize, ext_fov_average, ringrem, angles, angles_projfrom, angles_projto, offset, logtrsf, param1, circle, scale, overpad, rolling, roll_shift, zerone_mode, dset_min, dset_max, decim_factor, downsc_factor, corr_offset, postprocess_required, polarfilt_opt, convert_opt, crop_opt, dynamic_ff, EFF, filtEFF, im_dark, outprefix, logfilename )).start() else: Process(target=process, args=(lock, start, end, num_sinos, infile, outpath, preprocessing_required, skipflat, corrplan, norm_sx, norm_dx, flat_end, half_half, half_half_line, ext_fov, ext_fov_rot_right, ext_fov_overlap, ext_fov_normalize, ext_fov_average, ringrem, angles, angles_projfrom, angles_projto, offset, logtrsf, param1, circle, scale, overpad, reconmethod, rolling, roll_shift, zerone_mode, dset_min, dset_max, decim_factor, downsc_factor, corr_offset, postprocess_required, polarfilt_opt, convert_opt, crop_opt, dynamic_ff, EFF, filtEFF, im_dark, outprefix, logfilename )).start()
def _process(lock, int_from, int_to, infile, outfile, outshape, outtype, skipflat, plan, flat_end, half_half, half_half_line, dynamic_ff, EFF, filtEFF, im_dark, rotation, interp, border, rescale_factor, logfilename): # Process the required subset of images: for i in range(int_from, int_to + 1): # Read input image: t0 = time() f_in = getHDF5(infile, 'r') if "/tomo" in f_in: dset = f_in['tomo'] else: dset = f_in['exchange/data'] im = tdf.read_tomo(dset, i).astype(float32) f_in.close() t1 = time() # Perform pre-processing (flat fielding, extended FOV, ring removal): if not skipflat: if dynamic_ff: # Dynamic flat fielding with downsampling = 2: im = dynamic_flat_fielding(im, EFF, filtEFF, 2, im_dark) else: im = flat_fielding(im, i, plan, flat_end, half_half, half_half_line, norm_sx, norm_dx) t2 = time() # Rotate: rows, cols = im.shape #if (cols > rows): # angle = - arctan(2.0 / cols) * (rows / 2.0) / 2.0 #else: # angle = - arctan(2.0 / rows) * (cols / 2.0) / 2.0 #print(angle) M = cv2.getRotationMatrix2D((cols / 2, rows / 2), rotation, 1) if interp == 'nearest': interpflag = cv2.INTER_NEAREST elif interp == 'cubic': interpflag = cv2.INTER_CUBIC elif interp == 'lanczos': interpflag = cv2.INTER_LANCZOS4 else: interpflag = cv2.INTER_LINEAR if border == 'constant': borderflag = cv2.BORDER_CONSTANT else: borderflag = cv2.BORDER_REPLICATE im = cv2.warpAffine(im, M, (cols, rows), flags=interpflag, borderMode=borderflag) # Save processed image to HDF5 file (atomic procedure - lock used): _write_data(lock, im, i, outfile, outshape, outtype, rescale_factor, logfilename, t2 - t1, t1 - t0)
def process(lock, int_from, int_to, num_sinos, infile, outpath, preprocessing_required, corr_plan, norm_sx, norm_dx, flat_end, half_half, half_half_line, ext_fov, ext_fov_rot_right, ext_fov_overlap, ringrem, angles, offset, logtransform, param1, circle, scale, pad, method, zerone_mode, dset_min, dset_max, decim_factor, downsc_factor, corr_offset, postprocess_required, convert_opt, crop_opt, outprefix, logfilename): """To do... """ # Process the required subset of images: for i in range(int_from, int_to + 1): # Perform reconstruction (on-the-fly preprocessing and phase retrieval, if required): #if (phaseretrieval_required): # # Load into memory a bunch of sinograms: # t0 = time() # # Open the TDF file for reading: # f_in = getHDF5(infile, 'r') # if "/tomo" in f_in: # dset = f_in['tomo'] # else: # dset = f_in['exchange/data'] # # Prepare the data structure according to the approximation window: # tmp_im = numpy.empty((tdf.get_nr_projs(dset),tdf.get_det_size(dset), approx_win), dtype=float32) # # Load the temporary data structure reading the input TDF file: # # (It can be parallelized Open-MP style) # ct = 0 # for j in range(i - approx_win/2, i + approx_win/2 + 1): # if (j < 0): # j = 0 # if (j >= num_sinos): # j = num_sinos - 1 # a = tdf.read_sino(dset,j).astype(float32) # tmp_im[:,:,ct] = a # ct = ct + 1 # # Close the TDF file: # f_in.close() # t1 = time() # # Perform the processing: # if (preprocessing_required): # ct = 0 # # (It can be parallelized Open-MP style) # for j in range(i - approx_win/2, i + approx_win/2 + 1): # if (j < 0): # j = 0 # if (j >= num_sinos): # j = num_sinos - 1 # tmp_im[:,:,ct] = flat_fielding (tmp_im[:,:,ct], j, corr_plan, flat_end, half_half, half_half_line, norm_sx, norm_dx).astype(float32) # tmp_im[:,:,ct] = extfov_correction (tmp_im[:,:,ct], ext_fov, ext_fov_rot_right, ext_fov_overlap).astype(float32) # tmp_im[:,:,ct] = ring_correction (tmp_im[:,:,ct], ringrem, flat_end, corr_plan['skip_flat_after'], half_half, half_half_line, ext_fov).astype(float32) # ct = ct + 1 # # Perform phase retrieval: # # (It can be parallelized Open-MP style) # for ct in range(0, tmp_im.shape[0]): # tmp_im[ct,:,:] = phase_retrieval(tmp_im[ct,:,:].T, phrt_plan).astype(float32).T # ct = ct + 1 # # Extract the central processed sinogram: # im = tmp_im[:,:,approx_win/2] #else: # Read only one sinogram: t0 = time() f_in = getHDF5(infile, 'r') if "/tomo" in f_in: dset = f_in['tomo'] else: dset = f_in['exchange/data'] im = tdf.read_sino(dset,i).astype(float32) f_in.close() t1 = time() # Perform the preprocessing of the sinogram (if required): if (preprocessing_required): im = flat_fielding (im, i, corr_plan, flat_end, half_half, half_half_line, norm_sx, norm_dx).astype(float32) im = extfov_correction (im, ext_fov, ext_fov_rot_right, ext_fov_overlap) im = ring_correction (im, ringrem, flat_end, corr_plan['skip_flat_after'], half_half, half_half_line, ext_fov) # Actual reconstruction: im = reconstruct(im, angles, offset, logtransform, param1, circle, scale, pad, method, zerone_mode, dset_min, dset_max, decim_factor, downsc_factor, corr_offset).astype(float32) # Appy post-processing (if required): if postprocess_required: im = postprocess(im, convert_opt, crop_opt) else: # Create the circle mask for fancy output: if (circle == True): siz = im.shape[1] if siz % 2: rang = arange(-siz / 2 + 1, siz / 2 + 1) else: rang = arange(-siz / 2,siz / 2) x,y = meshgrid(rang,rang) z = x ** 2 + y ** 2 a = (z < (siz / 2 - abs(offset) ) ** 2) im = im * a # Write down reconstructed slice: t2 = time() fname = outpath + outprefix + '_' + str(i).zfill(4) + '.tif' imsave(fname, im) t3 = time() # Write log (atomic procedure - lock used): write_log(lock, fname, logfilename, t2 - t1, (t3 - t2) + (t1 - t0) )
def main(argv): """ Converts a TDF file (HDF5 Tomo Data Format) into a sequence of TIFF (uncompressed) files. Parameters ---------- from : scalar, integer among all the projections (or sinogram) data, a subset of the volume can be specified, ranging from the parameter "from" to the parameter "to" (see next). In most cases, this parameter is 0. to : scalar, integer among all the projections (or sinogram) data, a subset of the volume can be specified, ranging from the parameter "from" (see previous parameter) to the parameter "to". If the value -1 is specified, all the projection (or sinogram) data will be considered. in_file : string path with filename of the TDF to read from (e.g. "Z:\\sample1.tdf"). out_path : string path that will contain the sequence of TIFF files (e.g. "Z:\\sample1\\tomo\\"). WARNING: the program does NOT automatically create non-existing folders and subfolders specified in the path. Moreover, if files with the same name already exist they will be automatically overwritten. file_prefix : string string to be assumed as the filename prefix of the TIFF files to create for the projection (or sinogram) data. E.g. "tomo" will create files having name "tomo_0001.tif", "tomo_0002.tif". flat_prefix : string string to be assumed as the filename prefix of the TIFF files to create for the flat (white field) data. E.g. "flat" will create files having name "flat_1.tif", "flat_2.tif". If dark or flat data have to be skipped the string "-" can be specified. dark_prefix : string string to be assumed as the filename prefix of the TIFF files to create for the dark (dark field) data. E.g. "dark" will create files having name "dark_1.tif", "dark_2.tif". If dark or flat data have to be skipped the string "-" can be specified. projection_order : boolean string specify the string "True" to create TIFF files for projections (the most common case), "False" for sinograms. TIFF_format : boolean string specify the string "True" to create TIFF files, "False" for RAW files. log_file : string path with filename of a log file (e.g. "R:\\log.txt") where info about the conversion is reported. Returns ------- no return value Example ------- Example call to convert all the projections data to a sequence of tomo*.tif files: python tdf2tiff.py 0 -1 "C:\Temp\wet12T4part2.tdf" "C:\Temp\tomo" tomo flat dark True True "C:\Temp\log.txt" Requirements ------- - Python 2.7 with the latest NumPy, SciPy, H5Py. - TIFFFile from C. Gohlke's website http://www.lfd.uci.edu/~gohlke/ (consider also to install TIFFFile.c for performances). - tdf.py Tests ------- Tested with WinPython-64bit-2.7.6.3 (Windows) and Anaconda 2.1.0 (Linux 64-bit). """ # To be used without flat fielding (just conversion): first_done = False # Get the from and to number of files to process: int_from = int(argv[0]) int_to = int(argv[1]) # -1 means "all files" # Get paths: infile = argv[2] outpath = argv[3] fileprefix = argv[4] flatprefix = argv[5] darkprefix = argv[6] if (flatprefix == "-") or (darkprefix == "-"): skipflat = True else: skipflat = False projorder = argv[7] if projorder == "True": projorder = True else: projorder = False TIFFFormat = argv[8] if TIFFFormat == "True": TIFFFormat = True else: TIFFFormat = False logfilename = argv[9] # Check prefixes and path: if not outpath.endswith(os.path.sep): outpath += os.path.sep # Init variables: num_files = 0 num_flats = 0 num_darks = 0 # Get the files in infile: log = open(logfilename,"w") log.write(os.linesep + "\tInput TDF: %s" % (infile)) if ( TIFFFormat ): log.write(os.linesep + "\tOutput path where TIFF files will be created: %s" % (outpath)) else: log.write(os.linesep + "\tOutput path where RAW files will be created: %s" % (outpath)) log.write(os.linesep + "\t--------------") log.write(os.linesep + "\tFile output prefix: %s" % (fileprefix)) log.write(os.linesep + "\tDark images output prefix: %s" % (flatprefix)) log.write(os.linesep + "\tFlat images output prefix: %s" % (darkprefix)) log.write(os.linesep + "\t--------------") if (int_to != -1): log.write(os.linesep + "\tThe subset [%d,%d] of the data will be considered." % (int_from, int_to)) if (projorder): log.write(os.linesep + "\tProjection order assumed.") else: log.write(os.linesep + "\tSinogram order assumed.") if (skipflat): log.write(os.linesep + "\tWarning: flat/dark images (if any) will not be considered.") log.write(os.linesep + "\t--------------") log.close() if not os.path.exists(infile): log = open(logfilename,"a") log.write(os.linesep + "\tError: input TDF file not found. Process will end.") log.close() exit() # Open the HDF5 file: f = getHDF5( infile, 'r' ) oldTDF = False; try: dset = f['tomo'] oldTDF = True except Exception: pass if not oldTDF: #try: dset = f['exchange/data'] #except Exception: # log = open(logfilename,"a") # log.write(os.linesep + "\tError: invalid TDF format. Process will end.") # log.close() # exit() if projorder: num_files = tdf.get_nr_projs(dset) else: num_files = tdf.get_nr_sinos(dset) f.close() # Get attributes: try: f = getHDF5( infile, 'r' ) if ('version' in f.attrs) and (f.attrs['version'] == 'TDF 1.0'): log = open(logfilename,"a") log.write(os.linesep + "\tTDF version 1.0 found.") log.write(os.linesep + "\t--------------") log.close() f.close() except: log = open(logfilename,"a") log.write(os.linesep + "\tWarning: TDF version unknown. Some features will not be available.") log.write(os.linesep + "\t--------------") log.close() # Check extrema (int_to == -1 means all files): if ( (int_to >= num_files) or (int_to == -1) ): int_to = num_files - 1 # Get the files in infile: if not skipflat: # # Flat part # try: t0 = time.time() f = getHDF5( infile, 'r' ) if oldTDF: dset = f['flat'] else: dset = f['exchange/data_white'] num_flats = tdf.get_nr_projs(dset) #if ('version' in f.attrs) and (f.attrs['version'] == "TDF 1.0"): if ('version' in f.attrs): if (f.attrs['version'] == '1.0'): provenance_dset = f['provenance/detector_output'] # Process the required subset of images: for i in range(0, num_flats): # Read input image: t1 = time.time() im = tdf.read_tomo( dset, i ) if ( TIFFFormat ): if ('version' in f.attrs): if (f.attrs['version'] == '1.0'): if (os.path.splitext(provenance_dset["filename", i])[0] == provenance_dset["filename", i]): fname = outpath + provenance_dset["filename", i] + '.tif' else: fname = outpath + provenance_dset["filename", i] else: fname = outpath + flatprefix + '_' + str(i + 1).zfill(4) + '.tif' else: fname = outpath + flatprefix + '_' + str(i + 1).zfill(4) + '_' + str(im.shape[1]) + 'x' + str(im.shape[0]) + '_' + str(im.dtype) + '.raw' # Cast type (if required but it should never occur): if ((im.dtype).type is float64): im = im.astype(float32, copy=False) if ( TIFFFormat ): imsave(fname, im) else: im.tofile(fname) try: if ('version' in f.attrs): if (f.attrs['version'] == '1.0'): t = int(time.mktime(datetime.datetime.strptime(provenance_dset["timestamp", i], "%Y-%m-%d %H:%M:%S.%f").timetuple())) os.utime(fname, (t,t) ) except: pass t2 = time.time() # Print out execution time: log = open(logfilename,"a") log.write(os.linesep + "\t%s created in %0.3f sec." % (os.path.basename(fname), t2 - t1)) log.close() f.close() except Exception: log = open(logfilename,"a") log.write(os.linesep + "\tWarning: no dataset named \"flat\" found.") log.close() pass # # Dark part # try: t0 = time.time() f = getHDF5( infile, 'r' ) if oldTDF: dset = f['dark'] else: dset = f['exchange/data_dark'] num_darks = tdf.get_nr_projs(dset) if ('version' in f.attrs): if (f.attrs['version'] == '1.0'): provenance_dset = f['provenance/detector_output'] # Process the required subset of images: for i in range(0, num_darks): # Read input image: t1 = time.time() im = tdf.read_tomo( dset, i ) if ( TIFFFormat ): if ('version' in f.attrs): if (f.attrs['version'] == '1.0'): if (os.path.splitext(provenance_dset["filename", num_flats + i])[0] == provenance_dset["filename", num_flats + i]): fname = outpath + provenance_dset["filename", num_flats + i] + '.tif' else: fname = outpath + provenance_dset["filename", num_flats + i] else: fname = outpath + darkprefix + '_' + str(i + 1).zfill(4) + '.tif' else: fname = outpath + darkprefix + '_' + str(i + 1).zfill(4) + '_' + str(im.shape[1]) + 'x' + str(im.shape[0]) + '_' + str(im.dtype) + '.raw' # Cast type (if required but it should never occur): if ((im.dtype).type is float64): im = im.astype(float32, copy=False) if ( TIFFFormat ): imsave(fname, im) else: im.tofile(fname) try: if ('version' in f.attrs): if (f.attrs['version'] == '1.0'): t = int(time.mktime(datetime.datetime.strptime(provenance_dset["timestamp", num_flats + i], "%Y-%m-%d %H:%M:%S.%f").timetuple())) os.utime(fname, (t,t) ) except: pass t2 = time.time() # Print out execution time: log = open(logfilename,"a") log.write(os.linesep + "\t%s created in %0.3f sec." % (os.path.basename(fname), t2 - t1)) log.close() f.close() except Exception: log = open(logfilename,"a") log.write(os.linesep + "\tWarning: no dataset named \"dark\" found.") log.close() pass else: num_flats = 0 num_darks = 0 # # Tomo part # # Read i-th image from input folder: t0 = time.time() f = getHDF5( infile, 'r' ) if oldTDF: dset = f['tomo'] else: dset = f['exchange/data'] if ('version' in f.attrs): if (f.attrs['version'] == '1.0'): provenance_dset = f['provenance/detector_output'] if not skipflat: offset = num_flats + num_darks else: offset = 0 # Process the required subset of images: for i in range(int_from, int_to + 1): # Read input image: t1 = time.time() if projorder: im = tdf.read_tomo( dset, i ) else: im = tdf.read_sino( dset, i ) # Cast type (if required but it should never occur): if ((im.dtype).type is float64): im = im.astype(float32, copy=False) # Save file: if ( TIFFFormat ): if ('version' in f.attrs): if (f.attrs['version'] == '1.0'): if (os.path.splitext(provenance_dset["filename", offset + i])[0] == provenance_dset["filename", offset + i]): fname = outpath + provenance_dset["filename", offset + i] + '.tif' else: fname = outpath + provenance_dset["filename", offset + i] else: fname = outpath + fileprefix + '_' + str(i + 1).zfill(4) + '.tif' imsave(fname, im) else: fname = outpath + fileprefix + '_' + str(i + 1).zfill(4) + '_' + str(im.shape[1]) + 'x' + str(im.shape[0]) + '_' + str(im.dtype) + '.raw' im.tofile(fname) # Change modified date: try: if ('version' in f.attrs): if (f.attrs['version'] == '1.0'): t = int(time.mktime(datetime.datetime.strptime(provenance_dset["timestamp", offset + i], "%Y-%m-%d %H:%M:%S.%f").timetuple())) os.utime(fname, (t,t) ) except: pass t2 = time.time() # Print out execution time: log = open(logfilename,"a") log.write(os.linesep + "\t%s created in %0.3f sec." % (os.path.basename(fname), t2 - t1)) log.close() f.close()
def process(sino_idx, num_sinos, infile, outpath, preprocessing_required, corr_plan, norm_sx, norm_dx, flat_end, half_half, half_half_line, ext_fov, ext_fov_rot_right, ext_fov_overlap, ext_fov_normalize, ext_fov_average, ringrem, phaseretrieval_required, phrtmethod, phrt_param1, phrt_param2, energy, distance, pixsize, phrtpad, approx_win, angles, angles_projfrom, angles_projto, offset, logtransform, param1, circle, scale, pad, method, zerone_mode, dset_min, dset_max, decim_factor, downsc_factor, corr_offset, postprocess_required, convert_opt, crop_opt, nr_threads, off_from, off_to, logfilename, lock, slice_prefix): """To do... """ slice_nr = sino_idx # Perform reconstruction (on-the-fly preprocessing and phase retrieval, if required): if (phaseretrieval_required): # In this case a bunch of sinograms is loaded into memory: # # Load the temporary data structure reading the input TDF file. # To know the right dimension the first sinogram is pre-processed. # # Open the TDF file and get the dataset: f_in = getHDF5(infile, 'r') if "/tomo" in f_in: dset = f_in['tomo'] else: dset = f_in['exchange/data'] # Downscaling and decimation factors considered when determining the approximation window: zrange = arange(sino_idx - approx_win * downsc_factor / 2, sino_idx + approx_win * downsc_factor / 2, downsc_factor) zrange = zrange[(zrange >= 0)] zrange = zrange[(zrange < num_sinos)] approx_win = zrange.shape[0] # Approximation window cannot be odd: if (approx_win % 2 == 1): approx_win = approx_win - 1 zrange = zrange[0:approx_win] # Read one sinogram to get the proper dimensions: test_im = tdf.read_sino(dset, zrange[0] * downsc_factor).astype(float32) # Apply projection removal (if required): test_im = test_im[angles_projfrom:angles_projto, :] # Apply decimation and downscaling (if required): test_im = test_im[::decim_factor, ::downsc_factor] # Perform the pre-processing of the first sinogram to get the right dimension: if (preprocessing_required): test_im = flat_fielding(test_im, zrange[0], corr_plan, flat_end, half_half, half_half_line / decim_factor, norm_sx, norm_dx).astype(float32) if ext_fov: test_im = extfov_correction(test_im, ext_fov_rot_right, ext_fov_overlap / downsc_factor, ext_fov_normalize, ext_fov_average).astype(float32) test_im = ring_correction(test_im, ringrem, flat_end, corr_plan['skip_flat_after'], half_half, half_half_line / decim_factor, ext_fov).astype(float32) # Now we can allocate memory for the bunch of slices: tmp_im = empty((approx_win, test_im.shape[0], test_im.shape[1]), dtype=float32) tmp_im[0, :, :] = test_im # Reading all the the sinos from TDF file and close: for ct in range(1, approx_win): test_im = tdf.read_sino(dset, zrange[ct] * downsc_factor).astype(float32) # Apply projection removal (if required): test_im = test_im[angles_projfrom:angles_projto, :] # Apply decimation and downscaling (if required): test_im = test_im[::decim_factor, ::downsc_factor] # Perform the pre-processing for each sinogram of the bunch: if (preprocessing_required): test_im = flat_fielding(test_im, zrange[ct], corr_plan, flat_end, half_half, half_half_line / decim_factor, norm_sx, norm_dx).astype(float32) if ext_fov: test_im = extfov_correction( test_im, ext_fov_rot_right, ext_fov_overlap / downsc_factor, ext_fov_normalize, ext_fov_average).astype(float32) test_im = ring_correction(test_im, ringrem, flat_end, corr_plan['skip_flat_after'], half_half, half_half_line / decim_factor, ext_fov).astype(float32) tmp_im[ct, :, :] = test_im f_in.close() # Now everything has to refer to a downscaled dataset: sino_idx = ((zrange == sino_idx).nonzero()) # # Perform phase retrieval: # # Prepare the plan: if (phrtmethod == 0): # Paganin's: phrtplan = tiehom_plan(tmp_im[:, 0, :], phrt_param1, phrt_param2, energy, distance, pixsize * downsc_factor, padding=phrtpad) else: phrtplan = phrt_plan(tmp_im[:, 0, :], energy, distance, pixsize * downsc_factor, phrt_param2, phrt_param1, phrtmethod, padding=phrtpad) # Process each projection (whose height depends on the size of the bunch): for ct in range(0, tmp_im.shape[1]): if (phrtmethod == 0): tmp_im[:, ct, :] = tiehom(tmp_im[:, ct, :], phrtplan).astype(float32) else: tmp_im[:, ct, :] = phrt(tmp_im[:, ct, :], phrtplan, phrtmethod).astype(float32) # Extract the requested sinogram: im = tmp_im[sino_idx[0], :, :].squeeze() else: # Read only one sinogram: f_in = getHDF5(infile, 'r') if "/tomo" in f_in: dset = f_in['tomo'] else: dset = f_in['exchange/data'] im = tdf.read_sino(dset, sino_idx * downsc_factor).astype(float32) f_in.close() # Apply projection removal (if required): im = im[angles_projfrom:angles_projto, :] # Downscale and decimate the sinogram: im = im[::decim_factor, ::downsc_factor] #sino_idx = sino_idx/downsc_factor # Perform the preprocessing of the sinogram (if required): if (preprocessing_required): im = flat_fielding(im, sino_idx, corr_plan, flat_end, half_half, half_half_line / decim_factor, norm_sx, norm_dx).astype(float32) if ext_fov: im = extfov_correction(im, ext_fov_rot_right, ext_fov_overlap / downsc_factor, ext_fov_normalize, ext_fov_average) im = ring_correction(im, ringrem, flat_end, corr_plan['skip_flat_after'], half_half, half_half_line / decim_factor, ext_fov) # Log infos: log = open(logfilename, "a") log.write( linesep + "\tPerforming reconstruction with multiple centers of rotation...") log.write(linesep + "\t--------------") log.close() # Split the computation into multiple processes: for num in range(nr_threads): start = ((off_to - off_from + 1) / nr_threads) * num + off_from if (num == nr_threads - 1): end = off_to else: end = ((off_to - off_from + 1) / nr_threads) * (num + 1) + off_from - 1 Process(target=reconstruct, args=(im, angles, angles_projfrom, angles_projto, offset / downsc_factor, logtransform, param1, circle, scale, pad, method, zerone_mode, dset_min, dset_max, corr_offset, postprocess_required, convert_opt, crop_opt, start, end, outpath, slice_nr, downsc_factor, logfilename, lock, slice_prefix)).start()
def main(argv): """To do... Usage ----- Parameters --------- Example -------------------------- The following line processes the first ten TIFF files of input path "/home/in" and saves the processed files to "/home/out" with the application of the Boin and Haibel filter with smoothing via a Butterworth filter of order 4 and cutoff frequency 0.01: destripe /home/in /home/out 1 10 1 0.01 4 """ lock = Lock() skip_ringrem = False skip_flat = False skip_flat_after = True first_done = False # Get the number of sino to pre-process: idx = int(argv[0]) # Get paths: infile = argv[1] outfile = argv[2] # Normalization parameters: norm_sx = int(argv[3]) norm_dx = int(argv[4]) # Params for flat fielding with post flats/darks: flat_end = True if argv[5] == "True" else False half_half = True if argv[6] == "True" else False half_half_line = int(argv[7]) # Params for extended FOV: ext_fov = True if argv[8] == "True" else False ext_fov_rot_right = argv[9] if ext_fov_rot_right == "True": ext_fov_rot_right = True if (ext_fov): norm_sx = 0 else: ext_fov_rot_right = False if (ext_fov): norm_dx = 0 ext_fov_overlap = int(argv[10]) # Method and parameters coded into a string: ringrem = argv[11] # Tmp path and log file: tmppath = argv[12] if not tmppath.endswith(sep): tmppath += sep logfilename = argv[13] # Open the HDF5 file: f_in = getHDF5(infile, 'r') if "/tomo" in f_in: dset = f_in['tomo'] else: dset = f_in['exchange/data'] prov_dset = f_in['provenance/detector_output'] num_proj = tdf.get_nr_projs(dset) num_sinos = tdf.get_nr_sinos(dset) # Check if the HDF5 makes sense: if (num_sinos == 0): log = open(logfilename,"a") log.write(linesep + "\tNo projections found. Process will end.") log.close() exit() # Get flat and darks from cache or from file: try: corrplan = cache2plan(infile, tmppath) except Exception as e: #print "Error(s) when reading from cache" corrplan = extract_flatdark(f_in, flat_end, logfilename) plan2cache(corrplan, infile, tmppath) # Read input image: im = tdf.read_sino(dset,idx).astype(float32) f_in.close() # Perform pre-processing (flat fielding, extended FOV, ring removal): im = flat_fielding(im, idx, corrplan, flat_end, half_half, half_half_line, norm_sx, norm_dx) im = extfov_correction(im, ext_fov, ext_fov_rot_right, ext_fov_overlap) im = ring_correction (im, ringrem, flat_end, corrplan['skip_flat_after'], half_half, half_half_line, ext_fov) # Write down reconstructed preview file (file name modified with metadata): im = im.astype(float32) outfile = outfile + '_' + str(im.shape[1]) + 'x' + str(im.shape[0]) + '_' + str( nanmin(im)) + '$' + str( nanmax(im) ) im.tofile(outfile)
def process(sino_idx, num_sinos, infile, outfile, preprocessing_required, corr_plan, skipflat, norm_sx, norm_dx, flat_end, half_half, half_half_line, ext_fov, ext_fov_rot_right, ext_fov_overlap, ext_fov_normalize, ext_fov_average, ringrem, phaseretrieval_required, phrtmethod, phrt_param1, phrt_param2, energy, distance, pixsize, phrtpad, approx_win, angles, angles_projfrom, angles_projto, offset, logtransform, recpar, circle, scale, pad, method, rolling, roll_shift, zerone_mode, dset_min, dset_max, decim_factor, downsc_factor, corr_offset, postprocess_required, polarfilt_opt, convert_opt, crop_opt, dynamic_ff, EFF, filtEFF, im_dark, nr_threads, logfilename, tmppath): """To do... """ # Perform reconstruction (on-the-fly preprocessing and phase retrieval, if # required): if (phaseretrieval_required): # In this case a bunch of sinograms is loaded into memory: # # Load the temporary data structure reading the input TDF file. # To know the right dimension the first sinogram is pre-processed. # # Open the TDF file and get the dataset: f_in = getHDF5(infile, 'r') if "/tomo" in f_in: dset = f_in['tomo'] else: dset = f_in['exchange/data'] # Downscaling and decimation factors considered when determining the # approximation window: zrange = arange(sino_idx - approx_win * downsc_factor / 2, sino_idx + approx_win * downsc_factor / 2, downsc_factor) zrange = zrange[(zrange >= 0)] zrange = zrange[(zrange < num_sinos)] approx_win = zrange.shape[0] # Approximation window cannot be odd: if (approx_win % 2 == 1): approx_win = approx_win - 1 zrange = zrange[0:approx_win] # Read one sinogram to get the proper dimensions: test_im = tdf.read_sino(dset, zrange[0] * downsc_factor).astype(float32) # Apply projection removal (if required): test_im = test_im[angles_projfrom:angles_projto, :] # Apply decimation and downscaling (if required): test_im = test_im[::decim_factor, ::downsc_factor] # Perform the pre-processing of the first sinogram to get the right # dimension: if (preprocessing_required): if not skipflat: if dynamic_ff: # Dynamic flat fielding with downsampling = 2: #test_im = dynamic_flat_fielding(test_im, zrange[0] / downsc_factor, EFF, filtEFF, 2, im_dark, norm_sx, norm_dx) test_im = dynamic_flat_fielding(test_im, zrange[0], EFF, filtEFF, 2, im_dark, norm_sx, norm_dx) else: #test_im = flat_fielding(test_im, zrange[0] / downsc_factor, corr_plan, flat_end, half_half, # half_half_line / decim_factor, norm_sx, norm_dx).astype(float32) test_im = flat_fielding(test_im, zrange[0], corr_plan, flat_end, half_half, half_half_line / decim_factor, norm_sx, norm_dx).astype(float32) if ext_fov: test_im = extfov_correction(test_im, ext_fov_rot_right, ext_fov_overlap / downsc_factor, ext_fov_normalize, ext_fov_average).astype(float32) if not skipflat and not dynamic_ff: test_im = ring_correction(test_im, ringrem, flat_end, corr_plan['skip_flat_after'], half_half, half_half_line / decim_factor, ext_fov).astype(float32) else: test_im = ring_correction(test_im, ringrem, False, False, half_half, half_half_line / decim_factor, ext_fov).astype(float32) # Now we can allocate memory for the bunch of slices: tmp_im = empty((approx_win, test_im.shape[0], test_im.shape[1]), dtype=float32) tmp_im[0, :, :] = test_im # Reading all the the sinos from TDF file and close: for ct in range(1, approx_win): # Read the sinogram: test_im = tdf.read_sino(dset, zrange[ct] * downsc_factor).astype(float32) # Apply projection removal (if required): test_im = test_im[angles_projfrom:angles_projto, :] # Apply decimation and downscaling (if required): test_im = test_im[::decim_factor, ::downsc_factor] # Perform the pre-processing for each sinogram of the bunch: if (preprocessing_required): if not skipflat: if dynamic_ff: # Dynamic flat fielding with downsampling = 2: #test_im = dynamic_flat_fielding(test_im, zrange[ct] / downsc_factor, EFF, filtEFF, 2, im_dark, norm_sx, norm_dx) test_im = dynamic_flat_fielding( test_im, zrange[ct], EFF, filtEFF, 2, im_dark, norm_sx, norm_dx) else: #test_im = flat_fielding(test_im, zrange[ct] / downsc_factor, corr_plan, flat_end, half_half, # half_half_line / decim_factor, norm_sx, norm_dx).astype(float32) test_im = flat_fielding(test_im, zrange[ct], corr_plan, flat_end, half_half, half_half_line / decim_factor, norm_sx, norm_dx).astype(float32) if ext_fov: test_im = extfov_correction( test_im, ext_fov_rot_right, ext_fov_overlap / downsc_factor, ext_fov_normalize, ext_fov_average).astype(float32) if not skipflat and not dynamic_ff: test_im = ring_correction(test_im, ringrem, flat_end, corr_plan['skip_flat_after'], half_half, half_half_line / decim_factor, ext_fov).astype(float32) else: test_im = ring_correction(test_im, ringrem, False, False, half_half, half_half_line / decim_factor, ext_fov).astype(float32) tmp_im[ct, :, :] = test_im f_in.close() # Now everything has to refer to a downscaled dataset: sino_idx = ((zrange == sino_idx).nonzero()) # # Perform phase retrieval: # # Prepare the plan: if (phrtmethod == 0): # Paganin 2002: phrtplan = tiehom_plan(tmp_im[:, 0, :], phrt_param1, phrt_param2, energy, distance, pixsize * downsc_factor, phrtpad) elif (phrtmethod == 1): # Paganin 2020: phrtplan = tiehom_plan2020(tmp_im[:, 0, :], phrt_param1, phrt_param2, energy, distance, pixsize * downsc_factor, phrtpad) else: phrtplan = phrt_plan(tmp_im[:, 0, :], energy, distance, pixsize * downsc_factor, phrt_param2, phrt_param1, phrtmethod, phrtpad) #phrtplan = prepare_plan (tmp_im[:,0,:], beta, delta, energy, distance, #pixsize*downsc_factor, padding=phrtpad) # Process each projection (whose height depends on the size of the bunch): for ct in range(0, tmp_im.shape[1]): #tmp_im[:,ct,:] = phase_retrieval(tmp_im[:,ct,:], phrtplan).astype(float32) if (phrtmethod == 0): tmp_im[:, ct, :] = tiehom(tmp_im[:, ct, :], phrtplan).astype(float32) elif (phrtmethod == 1): tmp_im[:, ct, :] = tiehom2020(tmp_im[:, ct, :], phrtplan).astype(float32) else: tmp_im[:, ct, :] = phrt(tmp_im[:, ct, :], phrtplan, phrtmethod).astype(float32) # Extract the requested sinogram: im = tmp_im[sino_idx[0], :, :].squeeze() else: # Read only one sinogram: f_in = getHDF5(infile, 'r') if "/tomo" in f_in: dset = f_in['tomo'] else: dset = f_in['exchange/data'] im = tdf.read_sino(dset, sino_idx * downsc_factor).astype(float32) f_in.close() # Apply projection removal (if required): im = im[angles_projfrom:angles_projto, :] # Apply decimation and downscaling (if required): im = im[::decim_factor, ::downsc_factor] #sino_idx = sino_idx / downsc_factor # Downscaling for the index already applied # Perform the preprocessing of the sinogram (if required): if (preprocessing_required): if not skipflat: if dynamic_ff: # Dynamic flat fielding with downsampling = 2: im = dynamic_flat_fielding(im, sino_idx, EFF, filtEFF, 2, im_dark, norm_sx, norm_dx) else: im = flat_fielding(im, sino_idx, corr_plan, flat_end, half_half, half_half_line / decim_factor, norm_sx, norm_dx).astype(float32) if ext_fov: im = extfov_correction(im, ext_fov_rot_right, ext_fov_overlap / downsc_factor, ext_fov_normalize, ext_fov_average) if not skipflat and not dynamic_ff: im = ring_correction(im, ringrem, flat_end, corr_plan['skip_flat_after'], half_half, half_half_line / decim_factor, ext_fov) else: im = ring_correction(im, ringrem, False, False, half_half, half_half_line / decim_factor, ext_fov) # Additional ring removal before reconstruction: #im = boinhaibel(im, '11;') #im = munchetal(im, '5;1.8') #im = rivers(im, '13;') #im = raven(im, '11;0.8') #im = oimoen(im, '51;51') # Actual reconstruction: im = reconstruct(im, angles, offset / downsc_factor, logtransform, recpar, circle, scale, pad, method, zerone_mode, dset_min, dset_max, corr_offset, rolling, roll_shift, tmppath).astype(float32) # Apply post-processing (if required): if postprocess_required: im = polarfilter(im, polarfilt_opt) im = croprescale(im, convert_opt, crop_opt) else: # Create the circle mask for fancy output: if (circle == True): siz = im.shape[1] if siz % 2: rang = arange(-siz / 2 + 1, siz / 2 + 1) else: rang = arange(-siz / 2, siz / 2) x, y = meshgrid(rang, rang) z = x**2 + y**2 a = (z < (siz / 2 - int(round(abs(offset) / downsc_factor)))**2) im = im * a # Write down reconstructed preview file (file name modified with metadata): im = im.astype(float32) outfile = outfile + '_' + str(im.shape[1]) + 'x' + str( im.shape[0]) + '_' + str(amin(im)) + '$' + str(amax(im)) im.tofile(outfile)
def process_gridrec(lock, int_from, int_to, num_sinos, infile, outpath, preprocessing_required, corr_plan, norm_sx, norm_dx, flat_end, half_half, half_half_line, ext_fov, ext_fov_rot_right, ext_fov_overlap, ringrem, angles, offset, logtransform, param1, circle, scale, pad, zerone_mode, dset_min, dset_max, decim_factor, downsc_factor, corr_offset, postprocess_required, convert_opt, crop_opt, outprefix, logfilename): """To do... """ # Process the required subset of images: for i in range(int_from, int_to + 1, 2): # Read two sinograms: t0 = time() f_in = getHDF5(infile, 'r') if "/tomo" in f_in: dset = f_in['tomo'] else: dset = f_in['exchange/data'] im1 = tdf.read_sino(dset,i).astype(float32) if ( (i + 1) <= (int_to + 1) ): im2 = tdf.read_sino(dset,i + 1).astype(float32) else: im2 = im1 f_in.close() t1 = time() # Perform the preprocessing of the sinograms (if required): if (preprocessing_required): im1 = flat_fielding (im1, i, corr_plan, flat_end, half_half, half_half_line, norm_sx, norm_dx).astype(float32) im1 = extfov_correction (im1, ext_fov, ext_fov_rot_right, ext_fov_overlap) im1 = ring_correction (im1, ringrem, flat_end, corr_plan['skip_flat_after'], half_half, half_half_line, ext_fov) im2 = flat_fielding (im2, i + 1, corr_plan, flat_end, half_half, half_half_line, norm_sx, norm_dx).astype(float32) im2 = extfov_correction (im2, ext_fov, ext_fov_rot_right, ext_fov_overlap) im2 = ring_correction (im2, ringrem, flat_end, corr_plan['skip_flat_after'], half_half, half_half_line, ext_fov) # Actual reconstruction: [im1, im2] = reconstruct_gridrec(im1, im2, angles, offset, logtransform, param1, circle, scale, pad, zerone_mode, dset_min, dset_max, decim_factor, downsc_factor, corr_offset) # Appy post-processing (if required): if postprocess_required: im1 = postprocess(im1, convert_opt, crop_opt, circle) im2 = postprocess(im2, convert_opt, crop_opt, circle) else: # Create the circle mask for fancy output: if (circle == True): siz = im1.shape[1] if siz % 2: rang = arange(-siz / 2 + 1, siz / 2 + 1) else: rang = arange(-siz / 2,siz / 2) x,y = meshgrid(rang,rang) z = x ** 2 + y ** 2 a = (z < (siz / 2 - int(round(abs(offset)/downsc_factor)) ) ** 2) im1 = im1 * a im2 = im2 * a # Write down reconstructed slices: t2 = time() fname1 = outpath + outprefix + '_' + str(i).zfill(4) + '.tif' imsave(fname1, im1) fname2 = outpath + outprefix + '_' + str(i + 1).zfill(4) + '.tif' imsave(fname2, im2) t3 = time() # Write log (atomic procedure - lock used): write_log_gridrec(lock, fname1, fname2, logfilename, t2 - t1, (t3 - t2) + (t1 - t0) )
def main(argv): """To do... """ lock = Lock() skip_flat = True first_done = False pyfftw_cache_disable() pyfftw_cache_enable() pyfftw_set_keepalive_time(1800) # Get the from and to number of files to process: idx = int(argv[0]) # Get full paths of input TDF and output TDF: infile = argv[1] outfile = argv[2] # Get the phase retrieval parameters: method = int(argv[3]) param1 = double(argv[4]) # param1( e.g. regParam, or beta) param2 = double(argv[5]) # param2( e.g. thresh or delta) energy = double(argv[6]) distance = double(argv[7]) pixsize = double(argv[8]) / 1000.0 # pixsixe from micron to mm: pad = True if argv[9] == "True" else False # Tmp path and log file: tmppath = argv[10] if not tmppath.endswith(sep): tmppath += sep logfilename = argv[11] # Open the HDF5 file and check it contains flat files: skipflat = False f_in = getHDF5(infile, 'r') if "/tomo" in f_in: dset = f_in['tomo'] if not "/flat" in f_in: skipflat = True else: dset = f_in['exchange/data'] if not "/exchange/data_white" in f_in: skipflat = True num_proj = tdf.get_nr_projs(dset) num_sinos = tdf.get_nr_sinos(dset) # Check if the HDF5 makes sense: if (num_proj == 0): log = open(logfilename, "a") log.write(linesep + "\tNo projections found. Process will end.") log.close() exit() # Get flats and darks from cache or from file: if not skipflat: try: corrplan = cache2plan(infile, tmppath) except Exception as e: #print "Error(s) when reading from cache" corrplan = extract_flatdark(f_in, True, logfilename) remove(logfilename) plan2cache(corrplan, infile, tmppath) # Read projection: im = tdf.read_tomo(dset, idx).astype(float32) f_in.close() # Apply simple flat fielding (if applicable): if not skipflat: if (isinstance(corrplan['im_flat_after'], ndarray) and isinstance(corrplan['im_flat'], ndarray) and isinstance(corrplan['im_dark'], ndarray) and isinstance(corrplan['im_dark_after'], ndarray)): if (idx < num_proj / 2): im = (im - corrplan['im_dark']) / ( abs(corrplan['im_flat'] - corrplan['im_dark']) + finfo(float32).eps) else: im = (im - corrplan['im_dark_after']) / ( abs(corrplan['im_flat_after'] - corrplan['im_dark_after']) + finfo(float32).eps) # Prepare plan: im = im.astype(float32) if (method == 0): # Paganin 2002: plan = tiehom_plan(im, param1, param2, energy, distance, pixsize, pad) im = tiehom(im, plan).astype(float32) elif (method == 1): # Paganin 2020: plan = tiehom_plan2020(im, param1, param2, energy, distance, pixsize, pad) im = tiehom2020(im, plan).astype(float32) else: plan = phrt_plan(im, energy, distance, pixsize, param2, param1, method, pad) im = phrt(im, plan, method).astype(float32) # Write down reconstructed preview file (file name modified with metadata): im = im.astype(float32) outfile = outfile + '_' + str(im.shape[1]) + 'x' + str( im.shape[0]) + '_' + str(nanmin(im)) + '$' + str(nanmax(im)) im.tofile(outfile)
def main(argv): """To do... Usage ----- Parameters --------- Example -------------------------- The following line processes the first ten TIFF files of input path "/home/in" and saves the processed files to "/home/out" with the application of the Boin and Haibel filter with smoothing via a Butterworth filter of order 4 and cutoff frequency 0.01: reconstruct 0 4 C:\Temp\Dullin_Aug_2012\sino_noflat C:\Temp\Dullin_Aug_2012\sino_noflat\output 9.0 10.0 0.0 0.0 0.0 true sino slice C:\Temp\Dullin_Aug_2012\sino_noflat\tomo_conv flat dark """ lock = Lock() skip_flat = False skip_flat_after = True # Get the from and to number of files to process: int_from = int(argv[0]) int_to = int(argv[1]) # Get paths: infile = argv[2] outpath = argv[3] # Essential reconstruction parameters:: angles = float(argv[4]) offset = float(argv[5]) param1 = argv[6] scale = int(float(argv[7])) overpad = True if argv[8] == "True" else False logtrsf = True if argv[9] == "True" else False circle = True if argv[10] == "True" else False outprefix = argv[11] # Parameters for on-the-fly pre-processing: preprocessing_required = True if argv[12] == "True" else False flat_end = True if argv[13] == "True" else False half_half = True if argv[14] == "True" else False half_half_line = int(argv[15]) ext_fov = True if argv[16] == "True" else False norm_sx = int(argv[19]) norm_dx = int(argv[20]) ext_fov_rot_right = argv[17] if ext_fov_rot_right == "True": ext_fov_rot_right = True if (ext_fov): norm_sx = 0 else: ext_fov_rot_right = False if (ext_fov): norm_dx = 0 ext_fov_overlap = int(argv[18]) skip_ringrem = True if argv[21] == "True" else False ringrem = argv[22] # Extra reconstruction parameters: zerone_mode = True if argv[23] == "True" else False corr_offset = float(argv[24]) reconmethod = argv[25] decim_factor = int(argv[26]) downsc_factor = int(argv[27]) # Parameters for postprocessing: postprocess_required = True if argv[28] == "True" else False convert_opt = argv[29] crop_opt = argv[30] nr_threads = int(argv[31]) logfilename = argv[32] process_id = int(logfilename[-6:-4]) # Check prefixes and path: #if not infile.endswith(sep): infile += sep if not exists(outpath): makedirs(outpath) if not outpath.endswith(sep): outpath += sep # Open the HDF5 file: f_in = getHDF5(infile, 'r') if "/tomo" in f_in: dset = f_in['tomo'] tomoprefix = 'tomo' flatprefix = 'flat' darkprefix = 'dark' else: dset = f_in['exchange/data'] if "/provenance/detector_output" in f_in: prov_dset = f_in['provenance/detector_output'] tomoprefix = prov_dset.attrs['tomo_prefix'] flatprefix = prov_dset.attrs['flat_prefix'] darkprefix = prov_dset.attrs['dark_prefix'] dset_min = -1 dset_max = -1 if (zerone_mode): if ('min' in dset.attrs): dset_min = float(dset.attrs['min']) else: zerone_mode = False if ('max' in dset.attrs): dset_max = float(dset.attrs['max']) else: zerone_mode = False num_sinos = tdf.get_nr_sinos(dset) # Pay attention to the downscale factor if (num_sinos == 0): log = open(logfilename,"a") log.write(linesep + "\tNo projections found. Process will end.") log.close() exit() # Check extrema (int_to == -1 means all files): if ((int_to >= num_sinos) or (int_to == -1)): int_to = num_sinos - 1 # Log info: log = open(logfilename,"w") log.write(linesep + "\tInput file: %s" % (infile)) log.write(linesep + "\tOutput path: %s" % (outpath)) log.write(linesep + "\t--------------") log.write(linesep + "\tPreparing the working plan...") log.close() # Get correction plan and phase retrieval plan (if required): corrplan = -1 phrtplan = -1 if (preprocessing_required): corrplan = extract_flatdark(f_in, flat_end, logfilename) f_in.close() # Log infos: log = open(logfilename,"a") log.write(linesep + "\tWorking plan prepared correctly.") log.write(linesep + "\t-------") log.write(linesep + "\tPerforming reconstruction...") log.close() # Run several threads for independent computation without waiting for threads completion: for num in range(nr_threads): start = ( (int_to - int_from + 1) / nr_threads)*num + int_from if (num == nr_threads - 1): end = int_to else: end = ( (int_to - int_from + 1) / nr_threads)*(num + 1) + int_from - 1 if (reconmethod == 'GRIDREC'): Process(target=process_gridrec, args=(lock, start, end, num_sinos, infile, outpath, preprocessing_required, corrplan, norm_sx, norm_dx, flat_end, half_half, half_half_line, ext_fov, ext_fov_rot_right, ext_fov_overlap, ringrem, angles, offset, logtrsf, param1, circle, scale, overpad, zerone_mode, dset_min, dset_max, decim_factor, downsc_factor, corr_offset, postprocess_required, convert_opt, crop_opt, outprefix, logfilename )).start() else: Process(target=process, args=(lock, start, end, num_sinos, infile, outpath, preprocessing_required, corrplan, norm_sx, norm_dx, flat_end, half_half, half_half_line, ext_fov, ext_fov_rot_right, ext_fov_overlap, ringrem, angles, offset, logtrsf, param1, circle, scale, overpad, reconmethod, zerone_mode, dset_min, dset_max, decim_factor, downsc_factor, corr_offset, postprocess_required, convert_opt, crop_opt, outprefix, logfilename )).start()
def main(argv): """ Converts a sequence of TIFF files into a TDF file (HDF5 Tomo Data Format). Parameters ---------- from : scalar, integer among all the projections (or sinogram) files, a subset of files can be specified, ranging from the parameter "from" to the parameter "to" (see next). In most cases, this parameter is 0. to : scalar, integer among all the projections (or sinogram) files, a subset of files can be specified, ranging from the parameter "from" (see previous parameter) to the parameter "to". If the value -1 is specified, all the projection files will be considered. in_path : string path containing the sequence of TIFF files to consider (e.g. "Z:\\sample1\\tomo\\"). out_file : string path with filename of the TDF to create (e.g. "Z:\\sample1.tdf"). WARNING: the program does NOT automatically create non-existing folders and subfolders specified in the path. Moreover, if a file with the same name already exists it will be automatically deleted and overwritten. crop_top : scalar, integer during the conversion, images can be cropped if required. This parameter specifies the number of pixels to crop from the top of the image. Leave 0 for no cropping. crop_bottom : scalar, integer during the conversion, images can be cropped if required. This parameter specifies the number of pixels to crop from the bottom of the image. Leave 0 for no cropping. crop_left : scalar, integer during the conversion, images can be cropped if required. This parameter specifies the number of pixels to crop from the left of the image. Leave 0 for no cropping. crop_right : scalar, integer during the conversion, images can be cropped if required. This parameter specifies the number of pixels to crop from the right of the image. Leave 0 for no cropping. file_prefix : string string to be assumed as the filename prefix of the TIFF files to consider for the projection (or sinogram) files. E.g. "tomo" will consider files having name "tomo_0001.tif", "tomo_0002.tif". flat_prefix : string string to be assumed as the filename prefix of the TIFF files to consider for the flat (white field) files. E.g. "flat" will consider files having name "flat_1.tif", "flat_2.tif". If dark or flat files have to be skipped the string "-" can be specified. dark_prefix : string string to be assumed as the filename prefix of the TIFF files to consider for the dark (dark field) files. E.g. "dark" will consider files having name "dark_1.tif", "dark_2.tif". If dark or flat files have to be skipped the string "-" can be specified. projection_order : boolean string specify the string "True" if the TIFF files represent projections (the most common case), "False" for sinograms. privilege_sino : boolean string specify the string "True" if the TDF will privilege a fast read/write of sinograms (the most common case), "False" for fast read/write of projections. compression : scalar, integer an integer value in the range of [1,9] to be used as GZIP compression factor in the HDF5 file, where 1 is the minimum compression (and maximum speed) and 9 is the maximum (and slow) compression. The value 0 can be specified with the meaning of no compression. nr_threads : int number of multiple threads (actually processes) to consider to speed up the whole conversion process. log_file : string path with filename of a log file (e.g. "R:\\log.txt") where info about the conversion is reported. Returns ------- no return value Example ------- Example call to convert all the tomo*.tif* projections to a TDF with no cropping and minimum compression: python tiff2tdf.py 0 -1 "Z:\\rawdata\\c_1\\tomo\\" "Z:\\work\\c1_compr9.tdf" 0 0 0 0 tomo flat dark True True 1 "S:\\conversion.txt" Requirements ------- - Python 2.7 with the latest NumPy, SciPy, H5Py. - TIFFFile from C. Gohlke's website http://www.lfd.uci.edu/~gohlke/ (consider also to install TIFFFile.c for performances). - tdf.py Tests ------- Tested with WinPython-64bit-2.7.6.3 (Windows) and Anaconda 2.1.0 (Linux 64-bit). """ lock = Lock() # Get the from and to number of files to process: int_from = int(argv[0]) int_to = int(argv[1]) # -1 means "all files" # Get paths: inpath = argv[2] outfile = argv[3] crop_top = int(argv[4]) # 0 for all means "no cropping" crop_bottom = int(argv[5]) crop_left = int(argv[6]) crop_right = int(argv[7]) tomoprefix = argv[8] flatprefix = argv[9] # - means "do not consider flat or darks" darkprefix = argv[10] # - means "do not consider flat or darks" if (flatprefix == "-") or (darkprefix == "-"): skipflat = True else: skipflat = False projorder = True if argv[11] == "True" else False privilege_sino = True if argv[12] == "True" else False # Get compression factor: compr_opts = int(argv[13]) compressionFlag = True if (compr_opts <= 0): compressionFlag = False elif (compr_opts > 9): compr_opts = 9 nr_threads = int(argv[14]) logfilename = argv[15] # Check prefixes and path: if not inpath.endswith(os.path.sep): inpath += os.path.sep # Get the files in inpath: log = open(logfilename, "w") log.write(os.linesep + "\tInput path: %s" % (inpath)) log.write(os.linesep + "\tOutput TDF file: %s" % (outfile)) log.write(os.linesep + "\t--------------") log.write(os.linesep + "\tProjection file prefix: %s" % (tomoprefix)) log.write(os.linesep + "\tDark file prefix: %s" % (darkprefix)) log.write(os.linesep + "\tFlat file prefix: %s" % (flatprefix)) log.write(os.linesep + "\t--------------") log.write(os.linesep + "\tCropping:") log.write(os.linesep + "\t\tTop: %d pixels" % (crop_top)) log.write(os.linesep + "\t\tBottom: %d pixels" % (crop_bottom)) log.write(os.linesep + "\t\tLeft: %d pixels" % (crop_left)) log.write(os.linesep + "\t\tRight: %d pixels" % (crop_right)) if (int_to != -1): log.write( os.linesep + "\tThe subset [%d,%d] of the input files will be considered." % (int_from, int_to)) if (projorder): log.write(os.linesep + "\tProjection order assumed.") else: log.write(os.linesep + "\tSinogram order assumed.") if (privilege_sino): log.write(os.linesep + "\tFast I/O for sinograms privileged.") else: log.write(os.linesep + "\tFast I/O for projections privileged.") if (compressionFlag): log.write(os.linesep + "\tTDF compression factor: %d" % (compr_opts)) else: log.write(os.linesep + "\tTDF compression: none.") if (skipflat): log.write( os.linesep + "\tWarning: flat/dark images (if any) will not be considered.") log.write(os.linesep + "\t--------------") log.close() # Remove a previous copy of output: if os.path.exists(outfile): log = open(logfilename, "a") log.write( os.linesep + "\tWarning: an output file with the same name was overwritten.") os.remove(outfile) log.close() log = open(logfilename, "a") log.write(os.linesep + "\tBrowsing input files...") log.close() # Pythonic way to get file list: if os.path.exists(inpath): tomo_files = sorted(glob(inpath + tomoprefix + '*.tif*')) num_files = len(tomo_files) else: log = open(logfilename, "a") log.write(os.linesep + "\tError: input path does not exist. Process will end.") log.close() exit() if (num_files == 0): log = open(logfilename, "a") log.write( os.linesep + "\tError: no projection files found. Check input path and file prefixes." ) log.close() exit() log = open(logfilename, "a") log.write(os.linesep + "\tInput files browsed correctly.") log.close() # Check extrema (int_to == -1 means all files): if ((int_to >= num_files) or (int_to == -1)): int_from = 0 int_to = num_files - 1 # In case of subset specified: num_files = int_to - int_from + 1 # Prepare output HDF5 output (should this be atomic?): im = imread(tomo_files[0]) # Crop: im = im[crop_top:im.shape[0] - crop_bottom, crop_left:im.shape[1] - crop_right] log = open(logfilename, "a") log.write(os.linesep + "\tPreparing the work plan...") log.close() #dsetshape = (num_files,) + im.shape if projorder: #dsetshape = tdf.get_dset_shape(privilege_sino, im.shape[1], im.shape[0], #num_files) datashape = tdf.get_dset_shape(im.shape[1], im.shape[0], num_files) else: #dsetshape = tdf.get_dset_shape(privilege_sino, im.shape[1], num_files, #im.shape[0]) datashape = tdf.get_dset_shape(im.shape[1], num_files, im.shape[0]) if not os.path.isfile(outfile): f = getHDF5(outfile, 'w') f.attrs['version'] = '1.0' f.attrs['implements'] = "exchange:provenance" echange_group = f.create_group('exchange') if (compressionFlag): dset = f.create_dataset('exchange/data', datashape, im.dtype, chunks=tdf.get_dset_chunks(im.shape[1]), compression="gzip", compression_opts=compr_opts, shuffle=True, fletcher32=True) else: dset = f.create_dataset('exchange/data', datashape, im.dtype) if privilege_sino: dset.attrs['axes'] = "y:theta:x" else: dset.attrs['axes'] = "theta:y:x" tmp = im[:].astype(numpy.float32) tmp = tmp[numpy.nonzero(numpy.isfinite(tmp))] dset.attrs['min'] = str(numpy.amin(tmp[:])) dset.attrs['max'] = str(numpy.amax(tmp[:])) dset.attrs['avg'] = str(0.0) # Get the total number of files to consider: tot_files = num_files if not skipflat: num_flats = len(sorted(glob(inpath + flatprefix + '*.tif*'))) num_darks = len(sorted(glob(inpath + darkprefix + '*.tif*'))) tot_files = tot_files + num_flats + num_darks # Create provenance dataset: provenance_dt = numpy.dtype([("filename", numpy.dtype("S255")), ("timestamp", numpy.dtype("S255"))]) metadata_group = f.create_group('provenance') provenance_dset = metadata_group.create_dataset('detector_output', (tot_files, ), dtype=provenance_dt) provenance_dset.attrs['tomo_prefix'] = tomoprefix provenance_dset.attrs['dark_prefix'] = darkprefix provenance_dset.attrs['flat_prefix'] = flatprefix provenance_dset.attrs['first_index'] = int(tomo_files[0][-8:-4]) # Handle the metadata: if (os.path.isfile(inpath + 'logfile.xml')): with open(inpath + 'logfile.xml', "r") as file: xml_command = file.read() tdf.parse_metadata(f, xml_command) f.close() # Print out about plan preparation: log = open(logfilename, "a") log.write(os.linesep + "\tWork plan prepared succesfully.") log.close() # Get the files in inpath: if not skipflat: # # Flat part # flat_files = sorted(glob(inpath + flatprefix + '*.tif*')) num_flats = len(flat_files) if (num_flats > 0): # Create acquisition group: im = imread(flat_files[0]) im = im[crop_top:im.shape[0] - crop_bottom, crop_left:im.shape[1] - crop_right] #flatshape = tdf.get_dset_shape(privilege_sino, im.shape[1], im.shape[0], #num_flats) flatshape = tdf.get_dset_shape(im.shape[1], im.shape[0], num_flats) f = getHDF5(outfile, 'a') if (compressionFlag): dset = f.create_dataset('exchange/data_white', flatshape, im.dtype, chunks=tdf.get_dset_chunks( im.shape[1]), compression="gzip", compression_opts=compr_opts, shuffle=True, fletcher32=True) else: dset = f.create_dataset('exchange/data_white', flatshape, im.dtype) tmp = im[:].astype(numpy.float32) tmp = tmp[numpy.nonzero(numpy.isfinite(tmp))] dset.attrs['min'] = str(numpy.amin(tmp[:])) dset.attrs['max'] = str(numpy.amax(tmp[:])) dset.attrs['avg'] = str(0.0) if privilege_sino: dset.attrs['axes'] = "y:theta:x" else: dset.attrs['axes'] = "theta:y:x" f.close() #process(lock, 0, num_flats - 1, 0, flat_files, True, outfile, #'exchange/data_white', dsetshape, im.dtype, # crop_top, crop_bottom, crop_left, crop_right, tot_files, provenance_dt, # logfilename ) else: log = open(logfilename, "a") log.write(os.linesep + "\tWarning: flat files not found.") log.close() # # Dark part # dark_files = sorted(glob(inpath + darkprefix + '*.tif*')) num_darks = len(dark_files) if (num_darks > 0): im = imread(dark_files[0]) im = im[crop_top:im.shape[0] - crop_bottom, crop_left:im.shape[1] - crop_right] #darkshape = tdf.get_dset_shape(privilege_sino, im.shape[1], im.shape[0], #num_flats) darkshape = tdf.get_dset_shape(im.shape[1], im.shape[0], num_darks) f = getHDF5(outfile, 'a') if (compressionFlag): dset = f.create_dataset('exchange/data_dark', darkshape, im.dtype, chunks=tdf.get_dset_chunks( im.shape[1]), compression="gzip", compression_opts=compr_opts, shuffle=True, fletcher32=True) else: dset = f.create_dataset('exchange/data_dark', darkshape, im.dtype) tmp = im[:].astype(numpy.float32) tmp = tmp[numpy.nonzero(numpy.isfinite(tmp))] dset.attrs['min'] = str(numpy.amin(tmp)) dset.attrs['max'] = str(numpy.amax(tmp)) dset.attrs['avg'] = str(0.0) if privilege_sino: dset.attrs['axes'] = "y:theta:x" else: dset.attrs['axes'] = "theta:y:x" f.close() #process(lock, 0, num_darks - 1, num_flats, dark_files, True, outfile, #'exchange/data_dark', dsetshape, im.dtype, # crop_top, crop_bottom, crop_left, crop_right, tot_files, provenance_dt, # logfilename ) else: log = open(logfilename, "a") log.write(os.linesep + "\tWarning: dark files not found.") log.close() # Process the required subset of images: if not skipflat: flatdark_offset = num_flats + num_darks else: flatdark_offset = 0 # Spawn the process for the conversion of flat images: if (num_flats > 0): Process(target=_process, args=(lock, 0, num_flats - 1, 0, 0, flat_files, True, outfile, 'exchange/data_white', flatshape, im.dtype, crop_top, crop_bottom, crop_left, crop_right, tot_files, provenance_dt, logfilename)).start() # Spawn the process for the conversion of dark images: if (num_darks > 0): Process(target=_process, args=(lock, 0, num_darks - 1, num_flats, 0, dark_files, True, outfile, 'exchange/data_dark', darkshape, im.dtype, crop_top, crop_bottom, crop_left, crop_right, tot_files, provenance_dt, logfilename)).start() # Start the process for the conversion of the projections (or sinograms) in a multi-threaded way: for num in range(nr_threads): start = ((int_to - int_from + 1) / nr_threads) * num + int_from if (num == nr_threads - 1): end = int_to else: end = ((int_to - int_from + 1) / nr_threads) * (num + 1) + int_from - 1 Process(target=_process, args=(lock, start, end, flatdark_offset, int_from, tomo_files, projorder, outfile, 'exchange/data', datashape, im.dtype, crop_top, crop_bottom, crop_left, crop_right, tot_files, provenance_dt, logfilename)).start()
def main(argv): """To do... """ lock = Lock() skip_flat = False skip_flat_after = True # Get the from and to number of files to process: sino_idx = int(argv[0]) # Get paths: infile = argv[1] outpath = argv[2] # Essential reconstruction parameters:: angles = float(argv[3]) off_step = float(argv[4]) param1 = argv[5] scale = int(float(argv[6])) overpad = True if argv[7] == "True" else False logtrsf = True if argv[8] == "True" else False circle = True if argv[9] == "True" else False # Parameters for on-the-fly pre-processing: preprocessing_required = True if argv[10] == "True" else False flat_end = True if argv[11] == "True" else False half_half = True if argv[12] == "True" else False half_half_line = int(argv[13]) ext_fov = True if argv[14] == "True" else False norm_sx = int(argv[19]) norm_dx = int(argv[20]) ext_fov_rot_right = argv[15] if ext_fov_rot_right == "True": ext_fov_rot_right = True if (ext_fov): norm_sx = 0 else: ext_fov_rot_right = False if (ext_fov): norm_dx = 0 ext_fov_overlap = int(argv[16]) ext_fov_normalize = True if argv[17] == "True" else False ext_fov_average = True if argv[18] == "True" else False skip_ringrem = True if argv[21] == "True" else False ringrem = argv[22] # Extra reconstruction parameters: zerone_mode = True if argv[23] == "True" else False corr_offset = float(argv[24]) reconmethod = argv[25] decim_factor = int(argv[26]) downsc_factor = int(argv[27]) # Parameters for postprocessing: postprocess_required = True if argv[28] == "True" else False convert_opt = argv[29] crop_opt = argv[30] # Parameters for on-the-fly phase retrieval: phaseretrieval_required = True if argv[31] == "True" else False phrtmethod = int(argv[32]) phrt_param1 = double(argv[33]) # param1( e.g. regParam, or beta) phrt_param2 = double(argv[34]) # param2( e.g. thresh or delta) energy = double(argv[35]) distance = double(argv[36]) pixsize = double(argv[37]) / 1000.0 # pixsixe from micron to mm: phrtpad = True if argv[38] == "True" else False approx_win = int(argv[39]) angles_projfrom = int(argv[40]) angles_projto = int(argv[41]) preprocessingplan_fromcache = True if argv[42] == "True" else False tmppath = argv[43] if not tmppath.endswith(sep): tmppath += sep nr_threads = int(argv[44]) off_from = float(argv[45]) off_to = float(argv[46]) slice_prefix = argv[47] logfilename = argv[48] if not exists(outpath): makedirs(outpath) if not outpath.endswith(sep): outpath += sep # Log info: log = open(logfilename, "w") log.write(linesep + "\tInput dataset: %s" % (infile)) log.write(linesep + "\tOutput path: %s" % (outpath)) log.write(linesep + "\t--------------") log.write(linesep + "\tLoading flat and dark images...") log.close() # Open the HDF5 file: f_in = getHDF5(infile, 'r') if "/tomo" in f_in: dset = f_in['tomo'] else: dset = f_in['exchange/data'] if "/provenance/detector_output" in f_in: prov_dset = f_in['provenance/detector_output'] dset_min = -1 dset_max = -1 if (zerone_mode): if ('min' in dset.attrs): dset_min = float(dset.attrs['min']) else: zerone_mode = False if ('max' in dset.attrs): dset_max = float(dset.attrs['max']) else: zerone_mode = False num_sinos = tdf.get_nr_sinos(dset) # Pay attention to the downscale factor if (num_sinos == 0): exit() # Check extrema: if (sino_idx >= num_sinos / downsc_factor): sino_idx = num_sinos / downsc_factor - 1 # Get correction plan and phase retrieval plan (if required): corrplan = 0 if (preprocessing_required): # Load flat fielding plan either from cache (if required) or from TDF file and cache it for faster re-use: if (preprocessingplan_fromcache): try: corrplan = cache2plan(infile, tmppath) except Exception as e: #print "Error(s) when reading from cache" corrplan = extract_flatdark(f_in, flat_end, logfilename) plan2cache(corrplan, infile, tmppath) else: corrplan = extract_flatdark(f_in, flat_end, logfilename) plan2cache(corrplan, infile, tmppath) # Dowscale flat and dark images if necessary: if isinstance(corrplan['im_flat'], ndarray): corrplan['im_flat'] = corrplan[ 'im_flat'][::downsc_factor, ::downsc_factor] if isinstance(corrplan['im_dark'], ndarray): corrplan['im_dark'] = corrplan[ 'im_dark'][::downsc_factor, ::downsc_factor] if isinstance(corrplan['im_flat_after'], ndarray): corrplan['im_flat_after'] = corrplan[ 'im_flat_after'][::downsc_factor, ::downsc_factor] if isinstance(corrplan['im_dark_after'], ndarray): corrplan['im_dark_after'] = corrplan[ 'im_dark_after'][::downsc_factor, ::downsc_factor] f_in.close() # Log infos: log = open(logfilename, "a") log.write(linesep + "\tPerforming preprocessing...") log.close() # Run computation: process(sino_idx, num_sinos, infile, outpath, preprocessing_required, corrplan, norm_sx, norm_dx, flat_end, half_half, half_half_line, ext_fov, ext_fov_rot_right, ext_fov_overlap, ext_fov_normalize, ext_fov_average, ringrem, phaseretrieval_required, phrtmethod, phrt_param1, phrt_param2, energy, distance, pixsize, phrtpad, approx_win, angles, angles_projfrom, angles_projto, off_step, logtrsf, param1, circle, scale, overpad, reconmethod, zerone_mode, dset_min, dset_max, decim_factor, downsc_factor, corr_offset, postprocess_required, convert_opt, crop_opt, nr_threads, off_from, off_to, logfilename, lock, slice_prefix)
def main(argv): """Try to guess the center of rotation of the input CT dataset. Parameters ---------- infile : array_like HDF5 input dataset outfile : string Full path where the identified center of rotation will be written as output scale : int If sub-pixel precision is interesting, use e.g. 2.0 to get a center of rotation of .5 value. Use 1.0 if sub-pixel precision is not required angles : int Total number of angles of the input dataset proj_from : int Initial projections to consider for the assumed angles proj_to : int Final projections to consider for the assumed angles method : string (not implemented yet) tmppath : string Temporary path where look for cached flat/dark files """ # Get path: infile = argv[0] # The HDF5 file on the outfile = argv[1] # The txt file with the proposed center scale = float(argv[2]) angles = float(argv[3]) proj_from = int(argv[4]) proj_to = int(argv[5]) method = argv[6] tmppath = argv[7] if not tmppath.endswith(sep): tmppath += sep pyfftw_cache_disable() pyfftw_cache_enable() pyfftw_set_keepalive_time(1800) # Create a silly temporary log: tmplog = tmppath + basename(infile) + str(time.time()) # Open the HDF5 file (take into account also older TDF versions): f_in = getHDF5(infile, 'r') if "/tomo" in f_in: dset = f_in['tomo'] else: dset = f_in['exchange/data'] num_proj = tdf.get_nr_projs(dset) num_sinos = tdf.get_nr_sinos(dset) # Get flats and darks from cache or from file: try: corrplan = cache2plan(infile, tmppath) except Exception as e: #print "Error(s) when reading from cache" corrplan = extract_flatdark(f_in, True, tmplog) remove(tmplog) plan2cache(corrplan, infile, tmppath) # Get first and the 180 deg projections: im1 = tdf.read_tomo(dset, proj_from).astype(float32) idx = int(round((proj_to - proj_from) / angles * pi)) + proj_from im2 = tdf.read_tomo(dset, idx).astype(float32) # Apply simple flat fielding (if applicable): if (isinstance(corrplan['im_flat_after'], ndarray) and isinstance(corrplan['im_flat'], ndarray) and isinstance(corrplan['im_dark'], ndarray) and isinstance(corrplan['im_dark_after'], ndarray)): im1 = ((abs(im1 - corrplan['im_dark'])) / (abs(corrplan['im_flat'] - corrplan['im_dark']) + finfo(float32).eps)).astype(float32) im2 = ((abs(im2 - corrplan['im_dark_after'])) / (abs(corrplan['im_flat_after'] - corrplan['im_dark_after']) + finfo(float32).eps)).astype(float32) # Scale projections (if required) to get subpixel estimation: if (abs(scale - 1.0) > finfo(float32).eps): im1 = imresize(im1, (int(round( scale * im1.shape[0])), int(round(scale * im1.shape[1]))), interp='bicubic', mode='F') im2 = imresize(im2, (int(round( scale * im2.shape[0])), int(round(scale * im2.shape[1]))), interp='bicubic', mode='F') # Find the center (flipping left-right im2): cen = findcenter.usecorrelation(im1, im2[:, ::-1]) cen = cen / scale # Print center to output file: text_file = open(outfile, "w") text_file.write(str(int(cen))) text_file.close() # Close input HDF5: f_in.close()