def read_projection(fname, element, theta_index):
"""
Reads a projection for a given element from an hdf file.
Parameters
----------
fname : str
String defining the file name
element :
String defining the element to select
theta_index :
index where theta is saved under in the hdf MAPS/extra_pvs_as_csv tag.
For unknown reason 2-ID-E and the Bio Nano proble save this information
in different index location (663 and 657)
Returns
-------
float
projection angle
ndarray
projection
"""
projections = dxchange.read_hdf5(fname, "MAPS/XRF_roi")
theta = float(dxchange.read_hdf5(fname, "MAPS/extra_pvs_as_csv")[theta_index].split(b',')[1])
elements = read_elements(fname)
try:
if find_index(elements, element) != None:
return projections[find_index(elements, element),:, :], theta
else:
raise TypeError
except TypeError:
print("**** ERROR: Element %s does exist in the file: %s " % (element, fname))
return None
def read_theta(path_files, theta_index, hdf_tag):
"""
Reads hdf file and returns theta
Parameters
----------
path_files: list
List of path+filenames
theta_index : int
Index where theta is saved under in the hdf MAPS/extra_pvs_as_csv tag
This is: 2-ID-E: 663; 2-ID-E (prior 2017): *657*; BNP: 8
hdf_tag : str
String defining the hdf5 data_tag name (ex. MAPS)
Returns
-------
ndarray: ndarray
1D array [theta]
"""
for i in range(len(path_files)):
if theta_index == None:
theta = float(dxchange.read_hdf5(path_files[i], "{}/theta".format(hdf_tag)))
print(theta)
else:
theta = float(dxchange.read_hdf5(path_files[i], "{}/extra_pvs_as_csv".format(hdf_tag))[theta_index].split(b',')[1])
print(theta)
return theta
def read_scaler(fname, hdf_tag, scaler_name):
try:
scaler_names = read_channel_names(fname, hdf_tag, 'scaler_names')
all_scaler = dxchange.read_hdf5(fname,
"{}/{}".format(hdf_tag, 'scalers'))
except:
scaler_names = read_channel_names(fname, 'MAPS', 'scaler_names')
all_scaler = dxchange.read_hdf5(fname,
"{}/{}".format('MAPS', 'scalers'))
return all_scaler[find_index(scaler_names, scaler_name)]
def read_dx_xrf(fname):
data = dxchange.read_hdf5(
fname, dataset='/exchange/data').astype('float32').copy()
b_elements = dxchange.read_hdf5(fname, dataset='/exchange/elements')
elements = []
for i, e in enumerate(b_elements):
elements.append(e.decode('utf-8'))
theta = dxchange.read_hdf5(
fname, dataset='/exchange/theta').astype('float32').copy()
theta *= np.pi / 180.
return data, theta, elements
def read_quant(fname, element, hdf_tag, quant_name, channel_tag):
elements = read_channel_names(fname, hdf_tag, channel_tag)
elem_idx = find_index(elements, element)
try:
quant_names = read_channel_names(fname, hdf_tag, 'quant_names')
all_quants = dxchange.read_hdf5(fname,
"{}/{}".format(hdf_tag, 'quant'))
except:
quant_names = ['SRcurrent', 'us_ic', 'ds_ic']
all_quants = dxchange.read_hdf5(
fname, "{}/{}".format('MAPS', 'XRF_roi_quant'))
quant_idx = find_index(quant_names, quant_name)
return all_quants[quant_idx][0][elem_idx]
def read_elements(h5fname):
b_elements = dxchange.read_hdf5(h5fname, "MAPS/channel_names")
elements = []
for i, e in enumerate(b_elements):
elements.append(e.decode('utf-8'))
return(elements)
def read_elements(h5fname):
b_elements = dxchange.read_hdf5(h5fname, "MAPS/channel_names")
elements = []
for i, e in enumerate(b_elements):
elements.append(e.decode('utf-8'))
return (elements)
def read_projection(fname, element, hdf_tag, roi_tag, channel_tag):
"""
Reads a projection for a given element from a single xrf hdf file
Parameters
----------
fname : str
String defining the file name
element : str
String defining the element to select
hdf_tag : str
String defining the hdf5 data_tag name (ex. MAPS)
roi_tag: str
data tag for corresponding roi_tag (ex. XRF_roi)
channel_tag : str
String defining the hdf5 channel tag name (ex. channel_names)
Returns
-------
ndarray: ndarray
projection
"""
elements = read_channel_names(fname, hdf_tag, channel_tag)
print(fname)
projections = dxchange.read_hdf5(fname, "{}/{}".format(hdf_tag, roi_tag))
return projections[find_index(elements, element)]
def read_channel_names(fname, hdf_tag, channel_tag):
"""
Read the channel names
Parameters
----------
fname : str
String defining the file name
hdf_tag : str
String defining the hdf5 data_tag name (ex. MAPS)
channel_tag : str
String defining the hdf5 channel tag name (ex. channel_names)
Returns
-------
channel_names : list
List of channel names
"""
try:
b_channel_names = dxchange.read_hdf5(
fname, "{}/{}".format(hdf_tag, channel_tag))
except:
print("File signature not found for {}, check file integrity".format(
fname))
return []
channel_names = []
for i, e in enumerate(b_channel_names):
channel_names.append(e.decode('utf-8'))
return (channel_names)
def read_projection(fname, element, theta_index):
"""
Reads a projection for a given element from a single xrf hdf file.
Parameters
----------
fname : str
String defining the file name
element :
String defining the element to select
theta_index :
Index where theta is saved under in the hdf MAPS/extra_pvs_as_csv tag.
This is:
2-ID-E: 663
2-ID-E prior 2017: 657
BNP: 8
Returns
-------
ndarray
projection
float
projection angle
"""
projections = dxchange.read_hdf5(fname, "MAPS/XRF_roi")
theta = float(
dxchange.read_hdf5(
fname, "MAPS/extra_pvs_as_csv")[theta_index].split(b',')[1])
elements = read_channel_names(fname)
try:
if find_index(elements, element) != None:
return projections[find_index(elements, element), :, :], theta
else:
raise TypeError
except TypeError:
print("**** ERROR: Element %s does exist in the file: %s " %
(element, fname))
return None
def read_projection(fname, element, theta_index):
"""
Reads a projection for a given element from an hdf file.
Parameters
----------
fname : str
String defining the file name
element :
String defining the element to select
theta_index :
index where theta is saved under in the hdf MAPS/extra_pvs_as_csv tag.
For unknown reason 2-ID-E and the Bio Nano proble save this information
in different index location (663 and 657)
Returns
-------
float
projection angle
ndarray
projection
"""
projections = dxchange.read_hdf5(fname, "MAPS/XRF_roi")
theta = float(
dxchange.read_hdf5(
fname, "MAPS/extra_pvs_as_csv")[theta_index].split(b',')[1])
elements = read_elements(fname)
try:
if find_index(elements, element) != None:
return projections[find_index(elements, element), :, :], theta
else:
raise TypeError
except TypeError:
print("**** ERROR: Element %s does exist in the file: %s " %
(element, fname))
return None
def read_channel_names(fname, hdf_tag, channel_tag):
"""
Read the channel names
Parameters
----------
fname : str
String defining the file name
hdf_tag : str
String defining the hdf5 data_tag name (ex. MAPS)
channel_tag : str
String defining the hdf5 channel tag name (ex. channel_names)
Returns
-------
channel_names : list
List of channel names
"""
b_channel_names = dxchange.read_hdf5(fname, "{}/{}".format(hdf_tag, channel_tag))
channel_names = []
for i, e in enumerate(b_channel_names):
channel_names.append(e.decode('utf-8'))
return(channel_names)
scanlog = p05tools.file.parse_scanlog(rawdir + scanname + 'scan.log')
################################
# tomopy options
ncore = 40
nchunk = 3
################################
# Projection preprocessing
use_preprocessed_projs = False
if use_preprocessed_projs:
proj = dxchange.read_hdf5(recodir + 'norm.h5', 'exchange/data')
theta = dxchange.read_hdf5(recodir + 'theta.h5', 'exchange/data')
logger.info('loaded proj from file: %s' % (recodir + 'norm.h5'))
logger.info('loaded theta from file: %s' % (recodir + 'theta.h5'))
else:
proj, flat, dark, theta = p05tools.reco.get_rawdata(scanlog, rawdir, verbose=True)
binfactor = 2
proj = p05tools.reco.rebin_stack(proj, binfactor, descriptor='proj')
flat = p05tools.reco.rebin_stack(flat, binfactor, descriptor='flat')
dark = p05tools.reco.rebin_stack(dark, binfactor, descriptor='dark')
# correlate flats and save array with related proj/flats
corr_flat = p05tools.reco.correrlate_flat(proj, flat, verbose=True)
dxchange.write_hdf5(corr_flat, fname=recodir + 'corr_flat.h5', overwrite=True)
logger.info('saved corr_flat array to file: %s' % (recodir + 'corr_flat.h5'))
def read_channel_names(h5fname):
b_channel_names = dxchange.read_hdf5(h5fname, "MAPS/channel_names")
channel_names = []
for i, e in enumerate(b_channel_names):
channel_names.append(e.decode('utf-8'))
return (channel_names)
def ds(img):
il, ih, iw = img.shape
img_tmp = np.zeros((il, ih / 2, iw / 2))
for i in range(len(img)):
img_tmp[i] = downscale_local_mean(img[i], (2, 2))
return img_tmp
fnames = glob.glob('/home/xiyang/data_20170608/')
wpath = 'weights/200ms_all'
for i in range(len(wpath)):
fname = fnames[i]
spath = '/home/xiyang/data_20170608/' + fname[:-3] + '/'
if not os.path.exists(spath):
os.makedirs(spath)
data = dxchange.read_hdf5(fname, '/exchange/data')
dark = dxchange.read_hdf5(fname, '/exchange/data_dark')
white = dxchange.read_hdf5(fname, '/exchange/data_white')
print data.shape, dark.shape, white.shape
print data.min(), data.max()
data = tomopy.normalize(data, white, dark)
print data.min(), data.max()
print data.shape
data[data > 10] = 0.5
data[data < -10] = 0
data = ds(data)
predict_patch(data, patch_size, 1, nb_filters, nb_conv, batch_size, wpath,
spath)
import numpy as np
import dxchange
import matplotlib.pyplot as plt
from skimage.measure import compare_ssim
from skimage.transform import downscale_local_mean
def ds(img):
il, ih, iw = img.shape
img_tmp = np.zeros((il, ih / 2, iw / 2))
for i in range(len(img)):
img_tmp[i] = downscale_local_mean(img[i], (2, 2))
return img_tmp
data = dxchange.read_hdf5('/home/beams1/YANGX/cnn_prj_enhance/battery_201703/H01C16_0.1_5_252.h5',
'/exchange/data')
dark = dxchange.read_hdf5('/home/beams1/YANGX/cnn_prj_enhance/battery_201703/H01C16_0.1_5_252.h5',
'/exchange/data_dark')
white = dxchange.read_hdf5('/home/beams1/YANGX/cnn_prj_enhance/battery_201703/H01C16_0.1_5_252.h5',
'/exchange/data_white')
print data.min(), data.max()
data = tomopy.normalize(data, white, dark)
data[data>10] = 0.5
data[data<-10] = 0
print data.min(), data.max()
data = ds(data)
data = data[:, 0:950, 130:1090]
print data.min(), data.max()
def normalize_dawa_radio(flat_file,
radio_file,
first_image=None,
last_image=None,
xshift=0,
yshift=0,
dark_value=None,
zinger_threshold=0,
zinger_filter_size=3):
logging.basicConfig(filename=radio_file + '_normalized.log',
filemode='w',
format='%(asctime)s %(levelname)-8s %(message)s',
level=logging.INFO,
datefmt='%Y-%m-%d %H:%M:%S')
logging.info('Reading data')
flat_file_name = flat_file + '.h5'
radio_file_name = radio_file + '.h5'
flat = dxchange.read_hdf5(flat_file_name, '/exchange/data_white')
dark_field_value = dxchange.read_hdf5(
flat_file_name, '/process/acquisition/dark_fields/dark_field_value')[0]
dark = flat[0, :, :] * 0 + dark_field_value
first = 0
if (first_image != None):
first = first_image
with h5py.File(radio_file_name, "r") as f:
# data = f['/exchange/data_white']
print('Reading /exchange/data')
data = f['/exchange/data']
last = data.shape[0]
if (last_image != None):
last = last_image
logging.info('first image: %d', first)
logging.info('last image: %d', last)
#radio = dxchange.read_hdf5(radio_file_name, '/exchange/data_white', slc=((first, last, 1), None))
radio = dxchange.read_hdf5(radio_file_name,
'/exchange/data',
slc=((first, last, 1), None))
if (xshift != 0):
flat = np.roll(flat, xshift, axis=2)
if (yshift != 0):
flat = np.roll(flat, yshift, axis=1)
logging.info('xshift: %d', xshift)
logging.info('yshift: %d', yshift)
logging.info('radio.shape: %s', radio.shape)
logging.info('flat.shape: %s', flat.shape)
logging.info('dark.shape: %s', dark.shape)
if (dark_value != None):
dark = dark * 0 + dark_value
logging.info('dark[0]: %s', dark.item(0))
logging.info('Normalizing')
radio = tomopy.normalize(radio, flat, dark)
logging.info("After normalizing, min: %f, max: %f", radio.min(),
radio.max())
if (zinger_threshold != 0):
logging.info('Removing zingers. threshold:%f, size:%d',
zinger_threshold, zinger_filter_size)
radio = tomopy.misc.corr.remove_outlier(radio,
zinger_threshold,
size=zinger_filter_size)
logging.info("After remove_outlier, min: %f, max: %f", radio.min(),
radio.max())
logging.info('Converting to integer')
radio = (10000. * radio).astype(np.int16)
logging.info('Writing normalized file')
dxchange.write_hdf5(radio,
fname=radio_file + '_normalized.h5',
overwrite=True)
logging.info('Normalized file written')
mydir = '/Users/decarlo/conda/mic_tools/data'
fname = []
fname = list(filter(lambda x: x.endswith(('.h5', '.hdf')), os.listdir(mydir)))
print(fname)
f = fname[4]
print("FFFFF: ", f)
prj = np.zeros((61, 101, 151), dtype='float32')
for m in range(len(fname)):
print(fname[m])
prj += dx.read_hdf5(os.path.join(mydir, fname[m]), dataset='/exchange/data').astype('float32').copy()
ang = dx.read_hdf5(os.path.join(mydir, f), dataset='/exchange/theta').astype('float32').copy()
ang *= np.pi / 180.
# Clean folder.
try:
shutil.rmtree('tmp/iters')
except:
pass
prj = tomopy.remove_nan(prj, val=0.0)
prj = tomopy.remove_neg(prj, val=0.0)
prj[np.where(prj == np.inf)] = 0.0
# prj = tomopy.median_filter(prj, size=3)
mydir = '/Users/decarlo/conda/mic_tools/data'
fname = []
fname = list(filter(lambda x: x.endswith(('.h5', '.hdf')), os.listdir(mydir)))
print(fname)
f = fname[4]
print("FFFFF: ", f)
prj = np.zeros((61, 101, 151), dtype='float32')
for m in range(len(fname)):
print(fname[m])
prj += dx.read_hdf5(os.path.join(mydir, fname[m]),
dataset='/exchange/data').astype('float32').copy()
ang = dx.read_hdf5(os.path.join(mydir, f),
dataset='/exchange/theta').astype('float32').copy()
ang *= np.pi / 180.
# Clean folder.
try:
shutil.rmtree('tmp/iters')
except:
pass
prj = tomopy.remove_nan(prj, val=0.0)
prj = tomopy.remove_neg(prj, val=0.0)
prj[np.where(prj == np.inf)] = 0.0
# prj = tomopy.median_filter(prj, size=3)
import dxchange
import matplotlib.pyplot as plt
from skimage.measure import compare_ssim
from skimage.transform import downscale_local_mean
def ds(img):
il, ih, iw = img.shape
img_tmp = np.zeros((il, ih / 2, iw / 2))
for i in range(len(img)):
img_tmp[i] = downscale_local_mean(img[i], (2, 2))
return img_tmp
data = dxchange.read_hdf5(
'/home/beams1/YANGX/cnn_prj_enhance/exposure_tests/training_insitu_1_6650eV_5X_5_prj_100msx300_0_1681.h5',
'/exchange/data')
dark = dxchange.read_hdf5(
'/home/beams1/YANGX/cnn_prj_enhance/exposure_tests/training_insitu_1_6650eV_5X_5_prj_100msx300_0_1681.h5',
'/exchange/data_dark')
white = dxchange.read_hdf5(
'/home/beams1/YANGX/cnn_prj_enhance/exposure_tests/training_insitu_1_6650eV_5X_5_prj_100msx300_0_1681.h5',
'/exchange/data_white')
print data.shape, dark.shape, white.shape
print data.min(), data.max()
data = tomopy.normalize(data[:300], white, dark)
data[data > 10] = 0.5
data[data < -10] = 0
print data.min(), data.max()
print data.shape
