def read_projection(fname, element, theta_index): """ Reads a projection for a given element from an hdf file. Parameters ---------- fname : str String defining the file name element : String defining the element to select theta_index : index where theta is saved under in the hdf MAPS/extra_pvs_as_csv tag. For unknown reason 2-ID-E and the Bio Nano proble save this information in different index location (663 and 657) Returns ------- float projection angle ndarray projection """ projections = dxchange.read_hdf5(fname, "MAPS/XRF_roi") theta = float(dxchange.read_hdf5(fname, "MAPS/extra_pvs_as_csv")[theta_index].split(b',')[1]) elements = read_elements(fname) try: if find_index(elements, element) != None: return projections[find_index(elements, element),:, :], theta else: raise TypeError except TypeError: print("**** ERROR: Element %s does exist in the file: %s " % (element, fname)) return None
def read_theta(path_files, theta_index, hdf_tag): """ Reads hdf file and returns theta Parameters ---------- path_files: list List of path+filenames theta_index : int Index where theta is saved under in the hdf MAPS/extra_pvs_as_csv tag This is: 2-ID-E: 663; 2-ID-E (prior 2017): *657*; BNP: 8 hdf_tag : str String defining the hdf5 data_tag name (ex. MAPS) Returns ------- ndarray: ndarray 1D array [theta] """ for i in range(len(path_files)): if theta_index == None: theta = float(dxchange.read_hdf5(path_files[i], "{}/theta".format(hdf_tag))) print(theta) else: theta = float(dxchange.read_hdf5(path_files[i], "{}/extra_pvs_as_csv".format(hdf_tag))[theta_index].split(b',')[1]) print(theta) return theta
def read_scaler(fname, hdf_tag, scaler_name): try: scaler_names = read_channel_names(fname, hdf_tag, 'scaler_names') all_scaler = dxchange.read_hdf5(fname, "{}/{}".format(hdf_tag, 'scalers')) except: scaler_names = read_channel_names(fname, 'MAPS', 'scaler_names') all_scaler = dxchange.read_hdf5(fname, "{}/{}".format('MAPS', 'scalers')) return all_scaler[find_index(scaler_names, scaler_name)]
def read_dx_xrf(fname): data = dxchange.read_hdf5( fname, dataset='/exchange/data').astype('float32').copy() b_elements = dxchange.read_hdf5(fname, dataset='/exchange/elements') elements = [] for i, e in enumerate(b_elements): elements.append(e.decode('utf-8')) theta = dxchange.read_hdf5( fname, dataset='/exchange/theta').astype('float32').copy() theta *= np.pi / 180. return data, theta, elements
def read_quant(fname, element, hdf_tag, quant_name, channel_tag): elements = read_channel_names(fname, hdf_tag, channel_tag) elem_idx = find_index(elements, element) try: quant_names = read_channel_names(fname, hdf_tag, 'quant_names') all_quants = dxchange.read_hdf5(fname, "{}/{}".format(hdf_tag, 'quant')) except: quant_names = ['SRcurrent', 'us_ic', 'ds_ic'] all_quants = dxchange.read_hdf5( fname, "{}/{}".format('MAPS', 'XRF_roi_quant')) quant_idx = find_index(quant_names, quant_name) return all_quants[quant_idx][0][elem_idx]
def read_elements(h5fname): b_elements = dxchange.read_hdf5(h5fname, "MAPS/channel_names") elements = [] for i, e in enumerate(b_elements): elements.append(e.decode('utf-8')) return(elements)
def read_elements(h5fname): b_elements = dxchange.read_hdf5(h5fname, "MAPS/channel_names") elements = [] for i, e in enumerate(b_elements): elements.append(e.decode('utf-8')) return (elements)
def read_projection(fname, element, hdf_tag, roi_tag, channel_tag): """ Reads a projection for a given element from a single xrf hdf file Parameters ---------- fname : str String defining the file name element : str String defining the element to select hdf_tag : str String defining the hdf5 data_tag name (ex. MAPS) roi_tag: str data tag for corresponding roi_tag (ex. XRF_roi) channel_tag : str String defining the hdf5 channel tag name (ex. channel_names) Returns ------- ndarray: ndarray projection """ elements = read_channel_names(fname, hdf_tag, channel_tag) print(fname) projections = dxchange.read_hdf5(fname, "{}/{}".format(hdf_tag, roi_tag)) return projections[find_index(elements, element)]
def read_channel_names(fname, hdf_tag, channel_tag): """ Read the channel names Parameters ---------- fname : str String defining the file name hdf_tag : str String defining the hdf5 data_tag name (ex. MAPS) channel_tag : str String defining the hdf5 channel tag name (ex. channel_names) Returns ------- channel_names : list List of channel names """ try: b_channel_names = dxchange.read_hdf5( fname, "{}/{}".format(hdf_tag, channel_tag)) except: print("File signature not found for {}, check file integrity".format( fname)) return [] channel_names = [] for i, e in enumerate(b_channel_names): channel_names.append(e.decode('utf-8')) return (channel_names)
def read_projection(fname, element, theta_index): """ Reads a projection for a given element from a single xrf hdf file. Parameters ---------- fname : str String defining the file name element : String defining the element to select theta_index : Index where theta is saved under in the hdf MAPS/extra_pvs_as_csv tag. This is: 2-ID-E: 663 2-ID-E prior 2017: 657 BNP: 8 Returns ------- ndarray projection float projection angle """ projections = dxchange.read_hdf5(fname, "MAPS/XRF_roi") theta = float( dxchange.read_hdf5( fname, "MAPS/extra_pvs_as_csv")[theta_index].split(b',')[1]) elements = read_channel_names(fname) try: if find_index(elements, element) != None: return projections[find_index(elements, element), :, :], theta else: raise TypeError except TypeError: print("**** ERROR: Element %s does exist in the file: %s " % (element, fname)) return None
def read_projection(fname, element, theta_index): """ Reads a projection for a given element from an hdf file. Parameters ---------- fname : str String defining the file name element : String defining the element to select theta_index : index where theta is saved under in the hdf MAPS/extra_pvs_as_csv tag. For unknown reason 2-ID-E and the Bio Nano proble save this information in different index location (663 and 657) Returns ------- float projection angle ndarray projection """ projections = dxchange.read_hdf5(fname, "MAPS/XRF_roi") theta = float( dxchange.read_hdf5( fname, "MAPS/extra_pvs_as_csv")[theta_index].split(b',')[1]) elements = read_elements(fname) try: if find_index(elements, element) != None: return projections[find_index(elements, element), :, :], theta else: raise TypeError except TypeError: print("**** ERROR: Element %s does exist in the file: %s " % (element, fname)) return None
def read_channel_names(fname, hdf_tag, channel_tag): """ Read the channel names Parameters ---------- fname : str String defining the file name hdf_tag : str String defining the hdf5 data_tag name (ex. MAPS) channel_tag : str String defining the hdf5 channel tag name (ex. channel_names) Returns ------- channel_names : list List of channel names """ b_channel_names = dxchange.read_hdf5(fname, "{}/{}".format(hdf_tag, channel_tag)) channel_names = [] for i, e in enumerate(b_channel_names): channel_names.append(e.decode('utf-8')) return(channel_names)
scanlog = p05tools.file.parse_scanlog(rawdir + scanname + 'scan.log') ################################ # tomopy options ncore = 40 nchunk = 3 ################################ # Projection preprocessing use_preprocessed_projs = False if use_preprocessed_projs: proj = dxchange.read_hdf5(recodir + 'norm.h5', 'exchange/data') theta = dxchange.read_hdf5(recodir + 'theta.h5', 'exchange/data') logger.info('loaded proj from file: %s' % (recodir + 'norm.h5')) logger.info('loaded theta from file: %s' % (recodir + 'theta.h5')) else: proj, flat, dark, theta = p05tools.reco.get_rawdata(scanlog, rawdir, verbose=True) binfactor = 2 proj = p05tools.reco.rebin_stack(proj, binfactor, descriptor='proj') flat = p05tools.reco.rebin_stack(flat, binfactor, descriptor='flat') dark = p05tools.reco.rebin_stack(dark, binfactor, descriptor='dark') # correlate flats and save array with related proj/flats corr_flat = p05tools.reco.correrlate_flat(proj, flat, verbose=True) dxchange.write_hdf5(corr_flat, fname=recodir + 'corr_flat.h5', overwrite=True) logger.info('saved corr_flat array to file: %s' % (recodir + 'corr_flat.h5'))
def read_channel_names(h5fname): b_channel_names = dxchange.read_hdf5(h5fname, "MAPS/channel_names") channel_names = [] for i, e in enumerate(b_channel_names): channel_names.append(e.decode('utf-8')) return (channel_names)
def ds(img): il, ih, iw = img.shape img_tmp = np.zeros((il, ih / 2, iw / 2)) for i in range(len(img)): img_tmp[i] = downscale_local_mean(img[i], (2, 2)) return img_tmp fnames = glob.glob('/home/xiyang/data_20170608/') wpath = 'weights/200ms_all' for i in range(len(wpath)): fname = fnames[i] spath = '/home/xiyang/data_20170608/' + fname[:-3] + '/' if not os.path.exists(spath): os.makedirs(spath) data = dxchange.read_hdf5(fname, '/exchange/data') dark = dxchange.read_hdf5(fname, '/exchange/data_dark') white = dxchange.read_hdf5(fname, '/exchange/data_white') print data.shape, dark.shape, white.shape print data.min(), data.max() data = tomopy.normalize(data, white, dark) print data.min(), data.max() print data.shape data[data > 10] = 0.5 data[data < -10] = 0 data = ds(data) predict_patch(data, patch_size, 1, nb_filters, nb_conv, batch_size, wpath, spath)
import numpy as np import dxchange import matplotlib.pyplot as plt from skimage.measure import compare_ssim from skimage.transform import downscale_local_mean def ds(img): il, ih, iw = img.shape img_tmp = np.zeros((il, ih / 2, iw / 2)) for i in range(len(img)): img_tmp[i] = downscale_local_mean(img[i], (2, 2)) return img_tmp data = dxchange.read_hdf5('/home/beams1/YANGX/cnn_prj_enhance/battery_201703/H01C16_0.1_5_252.h5', '/exchange/data') dark = dxchange.read_hdf5('/home/beams1/YANGX/cnn_prj_enhance/battery_201703/H01C16_0.1_5_252.h5', '/exchange/data_dark') white = dxchange.read_hdf5('/home/beams1/YANGX/cnn_prj_enhance/battery_201703/H01C16_0.1_5_252.h5', '/exchange/data_white') print data.min(), data.max() data = tomopy.normalize(data, white, dark) data[data>10] = 0.5 data[data<-10] = 0 print data.min(), data.max() data = ds(data) data = data[:, 0:950, 130:1090] print data.min(), data.max()
def normalize_dawa_radio(flat_file, radio_file, first_image=None, last_image=None, xshift=0, yshift=0, dark_value=None, zinger_threshold=0, zinger_filter_size=3): logging.basicConfig(filename=radio_file + '_normalized.log', filemode='w', format='%(asctime)s %(levelname)-8s %(message)s', level=logging.INFO, datefmt='%Y-%m-%d %H:%M:%S') logging.info('Reading data') flat_file_name = flat_file + '.h5' radio_file_name = radio_file + '.h5' flat = dxchange.read_hdf5(flat_file_name, '/exchange/data_white') dark_field_value = dxchange.read_hdf5( flat_file_name, '/process/acquisition/dark_fields/dark_field_value')[0] dark = flat[0, :, :] * 0 + dark_field_value first = 0 if (first_image != None): first = first_image with h5py.File(radio_file_name, "r") as f: # data = f['/exchange/data_white'] print('Reading /exchange/data') data = f['/exchange/data'] last = data.shape[0] if (last_image != None): last = last_image logging.info('first image: %d', first) logging.info('last image: %d', last) #radio = dxchange.read_hdf5(radio_file_name, '/exchange/data_white', slc=((first, last, 1), None)) radio = dxchange.read_hdf5(radio_file_name, '/exchange/data', slc=((first, last, 1), None)) if (xshift != 0): flat = np.roll(flat, xshift, axis=2) if (yshift != 0): flat = np.roll(flat, yshift, axis=1) logging.info('xshift: %d', xshift) logging.info('yshift: %d', yshift) logging.info('radio.shape: %s', radio.shape) logging.info('flat.shape: %s', flat.shape) logging.info('dark.shape: %s', dark.shape) if (dark_value != None): dark = dark * 0 + dark_value logging.info('dark[0]: %s', dark.item(0)) logging.info('Normalizing') radio = tomopy.normalize(radio, flat, dark) logging.info("After normalizing, min: %f, max: %f", radio.min(), radio.max()) if (zinger_threshold != 0): logging.info('Removing zingers. threshold:%f, size:%d', zinger_threshold, zinger_filter_size) radio = tomopy.misc.corr.remove_outlier(radio, zinger_threshold, size=zinger_filter_size) logging.info("After remove_outlier, min: %f, max: %f", radio.min(), radio.max()) logging.info('Converting to integer') radio = (10000. * radio).astype(np.int16) logging.info('Writing normalized file') dxchange.write_hdf5(radio, fname=radio_file + '_normalized.h5', overwrite=True) logging.info('Normalized file written')
mydir = '/Users/decarlo/conda/mic_tools/data' fname = [] fname = list(filter(lambda x: x.endswith(('.h5', '.hdf')), os.listdir(mydir))) print(fname) f = fname[4] print("FFFFF: ", f) prj = np.zeros((61, 101, 151), dtype='float32') for m in range(len(fname)): print(fname[m]) prj += dx.read_hdf5(os.path.join(mydir, fname[m]), dataset='/exchange/data').astype('float32').copy() ang = dx.read_hdf5(os.path.join(mydir, f), dataset='/exchange/theta').astype('float32').copy() ang *= np.pi / 180. # Clean folder. try: shutil.rmtree('tmp/iters') except: pass prj = tomopy.remove_nan(prj, val=0.0) prj = tomopy.remove_neg(prj, val=0.0) prj[np.where(prj == np.inf)] = 0.0 # prj = tomopy.median_filter(prj, size=3)
import dxchange import matplotlib.pyplot as plt from skimage.measure import compare_ssim from skimage.transform import downscale_local_mean def ds(img): il, ih, iw = img.shape img_tmp = np.zeros((il, ih / 2, iw / 2)) for i in range(len(img)): img_tmp[i] = downscale_local_mean(img[i], (2, 2)) return img_tmp data = dxchange.read_hdf5( '/home/beams1/YANGX/cnn_prj_enhance/exposure_tests/training_insitu_1_6650eV_5X_5_prj_100msx300_0_1681.h5', '/exchange/data') dark = dxchange.read_hdf5( '/home/beams1/YANGX/cnn_prj_enhance/exposure_tests/training_insitu_1_6650eV_5X_5_prj_100msx300_0_1681.h5', '/exchange/data_dark') white = dxchange.read_hdf5( '/home/beams1/YANGX/cnn_prj_enhance/exposure_tests/training_insitu_1_6650eV_5X_5_prj_100msx300_0_1681.h5', '/exchange/data_white') print data.shape, dark.shape, white.shape print data.min(), data.max() data = tomopy.normalize(data[:300], white, dark) data[data > 10] = 0.5 data[data < -10] = 0 print data.min(), data.max() print data.shape