def test_normalization_works(self):
"""assert sample and ob normalization works with and without roi"""
sample_tif_folder = self.data_path + '/tif/sample'
ob_tif_folder = self.data_path + '/tif/ob'
# testing sample with norm_roi
o_norm = Normalization()
o_norm.load(folder=sample_tif_folder, auto_gamma_filter=False)
o_norm.load(folder=ob_tif_folder,
data_type='ob',
auto_gamma_filter=False)
roi = ROI(x0=0, y0=0, x1=3, y1=2)
_sample = o_norm.data['sample']['data'][0]
_expected = _sample / np.mean(_sample[0:3, 0:4])
o_norm.normalization(roi=roi)
_returned = o_norm.data['sample']['data'][0]
assert (_expected == _returned).all()
# testing sample without norm_roi
o_norm1 = Normalization()
o_norm1.load(folder=sample_tif_folder, auto_gamma_filter=False)
o_norm1.load(folder=ob_tif_folder,
data_type='ob',
auto_gamma_filter=False)
_expected = o_norm1.data['sample']['data'][0]
o_norm1.normalization()
_returned = o_norm1.data['sample']['data'][0]
assert (_expected == _returned).all()
# testing ob with norm_roi
o_norm = Normalization()
o_norm.load(folder=sample_tif_folder, auto_gamma_filter=False)
o_norm.load(folder=ob_tif_folder,
data_type='ob',
auto_gamma_filter=False)
norm_roi = ROI(x0=0, y0=0, x1=3, y1=2)
o_norm.normalization(roi=norm_roi)
_ob = o_norm.data['ob']['data'][0]
_expected = _ob / np.mean(_ob[0:3, 0:4])
_returned = o_norm.data['ob']['data'][0]
assert (_expected == _returned).all()
# testing ob without norm_roi
o_norm = Normalization()
o_norm.load(folder=sample_tif_folder, auto_gamma_filter=False)
o_norm.load(folder=ob_tif_folder,
data_type='ob',
auto_gamma_filter=False)
_expected = o_norm.data['ob']['data'][0]
o_norm.normalization()
_returned = o_norm.data['ob']['data'][0]
assert (_expected == _returned).all()
def test_nbr_data_files_same_after_normalization_by_list_roi(self):
"""assert the number of data files is the same after normalization by a list of ROI"""
samples_path = self.data_path + '/tif/sample/' # 3 files
ob1 = self.data_path + '/tif/ob/ob001.tif'
ob2 = self.data_path + '/tif/ob/ob002.tif'
o_norm = Normalization()
o_norm.load(folder=samples_path, auto_gamma_filter=False)
o_norm.load(file=[ob1, ob2], data_type='ob', auto_gamma_filter=False)
_roi1 = ROI(x0=0, y0=0, x1=2, y1=2)
_roi2 = ROI(x0=1, y0=1, x1=3, y1=3)
_list_roi = [_roi1, _roi2]
nbr_data_before = len(o_norm.data['sample']['data'])
o_norm.normalization(roi=_list_roi)
nbr_data_after = len(o_norm.data['sample']['data'])
assert nbr_data_after == nbr_data_before
def run_normalization(self, dict_roi=None):
if dict_roi is None:
#try:
self.o_norm.df_correction()
self.o_norm.normalization(notebook=True)
self.normalized_data_array = self.o_norm.get_normalized_data()
#except:
# display(HTML('<span style="font-size: 20px; color:red">Data Size ' +
# 'do not Match (use bin_images.ipynb notebook to resize them)!</span>'))
else:
_list_roi = []
for _key in dict_roi.keys():
_roi = dict_roi[_key]
x0 = _roi['x0']
y0 = _roi['y0']
x1 = _roi['x1']
y1 = _roi['y1']
x_left = np.min([x0, x1])
y_top = np.min([y0, y1])
width_roi = np.abs(x0 - x1)
height_roi = np.abs(y0 - y1)
_roi = ROI(x0=x_left,
y0=y_top,
width=width_roi,
height=height_roi)
_list_roi.append(_roi)
# try:
self.o_norm.df_correction()
self.o_norm.normalization(roi=_list_roi, notebook=True)
self.normalized_data_array = self.o_norm.get_normalized_data()
def test_normalization_using_roi_of_sample_only(self):
"""testing features that will normalized the data according to a ROI of the sample
this feature require at least 1 ROI. the average counts of the ROI for each image will be used as
"open beam" counts and each pixel will be divided by this value
"""
sample_file = self.data_path + '/tif/special_sample/image_0001_roi_no_ob.tiff'
o_norm = Normalization()
o_norm.load(file=sample_file, auto_gamma_filter=False)
with pytest.raises(ValueError):
o_norm.normalization(use_only_sample=True)
del o_norm
o_norm = Normalization()
o_norm.load(file=sample_file, auto_gamma_filter=False)
_roi = ROI(x0=0, y0=0, x1=0, y1=0)
o_norm.normalization(roi=_roi, use_only_sample=True)
_norm_expected = np.ones((5, 5))
_norm_expected[1:, 0] = 1. / 10
_norm_expected[:, 1] = 2. / 10
_norm_expected[:, 2] = 3. / 10
_norm_expected[:, 3] = 4. / 10
_norm_expected[:-1, 4] = 5. / 10
_norm_returned = o_norm.get_normalized_data()[0]
nbr_col, nbr_row = np.shape(_norm_expected)
for _col in np.arange(nbr_col):
for _row in np.arange(nbr_row):
assert _norm_expected[_col, _row] == pytest.approx(
_norm_returned[_col, _row], 1e-5)
def test_normalization_works_with_only_1_df(self):
"""assert using 1 df in normalization works"""
samples_path = self.data_path + '/tif/sample/' # 3 files
ob1 = self.data_path + '/tif/ob/ob001.tif'
ob2 = self.data_path + '/tif/ob/ob002.tif'
df1 = self.data_path + '/tif/df/df001.tif'
o_norm = Normalization()
o_norm.load(folder=samples_path, auto_gamma_filter=False)
o_norm.load(file=[ob1, ob2], data_type='ob', auto_gamma_filter=False)
o_norm.load(file=df1, data_type='df', auto_gamma_filter=False)
o_norm.df_correction()
_roi1 = ROI(x0=0, y0=0, x1=2, y1=2)
_roi2 = ROI(x0=1, y0=1, x1=3, y1=3)
_list_roi = [_roi1, _roi2]
nbr_data_before = len(o_norm.data['sample']['data'])
o_norm.normalization(roi=_list_roi)
nbr_data_after = len(o_norm.data['sample']['data'])
assert nbr_data_after == nbr_data_before
def test_normalization_ran_twice_with_force_flag(self):
"""assert normalization can be ran twice using force flag"""
sample_tif_folder = self.data_path + '/tif/sample'
ob_tif_folder = self.data_path + '/tif/ob'
# testing sample with norm_roi
o_norm = Normalization()
o_norm.load(folder=sample_tif_folder, auto_gamma_filter=False)
o_norm.load(folder=ob_tif_folder,
data_type='ob',
auto_gamma_filter=False)
roi = ROI(x0=0, y0=0, x1=3, y1=2)
o_norm.normalization(roi=roi)
_returned_first_time = o_norm.data['sample']['data'][0]
roi = ROI(x0=0, y0=0, x1=2, y1=3)
o_norm.normalization(roi=roi, force=True)
_returned_second_time = o_norm.data['sample']['data'][0]
assert not ((_returned_first_time == _returned_second_time).all())
def test_setting_roi_x0_y0_x1_y1(self):
"""assert roi are correctly defined using x0, y0, x1 and y1"""
x0 = 1
y0 = 1
x1 = 5
y1 = 10
_roi = ROI(x0=x0, y0=y0, x1=x1, y1=y1)
_expected = [x0, y0, x1, y1]
_returned = [_roi.x0, _roi.y0, _roi.x1, _roi.y1]
self.assertTrue(_expected == _returned)
def test_setting_roi_x0_y0_width_height(self):
"""assert roi are correctly defined using x0, y0, width and height"""
x0 = 1
y0 = 1
width = 4
height = 9
_roi = ROI(x0=x0, y0=y0, width=width, height=height)
_expected = [x0, y0, x0 + width, y0 + height]
_returned = [_roi.x0, _roi.y0, _roi.x1, _roi.y1]
self.assertTrue(_expected == _returned)
def test_roi_fit_images(self):
"""assert norm roi do fit the images"""
sample_tif_file = self.data_path + '/tif/sample/image001.tif'
ob_tif_file = self.data_path + '/tif/ob/ob001.tif'
# x0 < 0 or x1 > image_width
o_norm = Normalization()
o_norm.load(file=sample_tif_file,
data_type='sample',
auto_gamma_filter=False)
o_norm.load(file=ob_tif_file, data_type='ob', auto_gamma_filter=False)
roi = ROI(x0=0, y0=0, x1=20, y1=4)
with pytest.raises(ValueError):
o_norm.normalization(roi=roi)
o_norm = Normalization()
o_norm.load(file=sample_tif_file,
data_type='sample',
auto_gamma_filter=False)
o_norm.load(file=ob_tif_file, data_type='ob', auto_gamma_filter=False)
roi = ROI(x0=-1, y0=0, x1=4, y1=4)
with pytest.raises(ValueError):
o_norm.normalization(roi=roi)
# y0 < 0 or y1 > image_height
o_norm = Normalization()
o_norm.load(file=sample_tif_file,
data_type='sample',
auto_gamma_filter=False)
o_norm.load(file=ob_tif_file, data_type='ob', auto_gamma_filter=False)
roi = ROI(x0=0, y0=-1, x1=4, y1=4)
with pytest.raises(ValueError):
o_norm.normalization(roi=roi)
# y1>image_height
o_norm = Normalization()
o_norm.load(file=sample_tif_file,
data_type='sample',
auto_gamma_filter=False)
o_norm.load(file=ob_tif_file, data_type='ob', auto_gamma_filter=False)
roi = ROI(x0=0, y0=0, x1=4, y1=20)
with pytest.raises(ValueError):
o_norm.normalization(roi=roi)
def normalization():
def remove_back_slash(list_files):
if list_files:
list_files_cleaned = []
for _file in list_files:
new_file = _file.replace("\\", "")
list_files_cleaned.append(new_file)
return list_files_cleaned
return list_files
args = parser.parse_args()
# print("output: {}".format(args.output)) # to retrieve output value
# print("sf: {}".format(args.sample_files))
# print("ob: {}".format(args.ob_files))
#
# print("x0: {}".format(args.roi_x0))
# print("y0: {}".format(args.roi_y0))
# print("x1: {}".format(args.roi_x1))
# print("y1: {}".format(args.roi_y1))
o_norm = Normalization()
list_samples = args.sample_files.split(',')
list_obs = args.ob_files.split(',')
list_samples = remove_back_slash(list_samples)
list_obs = remove_back_slash(list_obs)
# loading
o_norm.load(file=list_samples)
o_norm.load(file=list_obs, data_type='ob')
if args.df_files:
list_dfs = args.df_files.split(',')
list_dfs = remove_back_slash(list_dfs)
o_norm.load(file=list_dfs, data_type='df')
o_norm.df_correction()
if args.rois:
list_rois = args.rois.split(':')
array_roi_object = []
for _rois in list_rois:
[x0,y0,x1,y1] = _rois.split(',') #x0,y0,x1,y1
_roi = ROI(x0=np.int(x0), y0=np.int(y0), x1=np.int(x1), y1=np.int(y1))
array_roi_object.append(_roi)
o_norm.normalization(roi=array_roi_object)
else:
o_norm.normalization()
output_folder = args.output
output_folder = output_folder.replace("\\", "")
o_norm.export(folder=output_folder)
def test_full_normalization_sample_with_one_roi(self):
"""assert the full normalization works with several roi selected"""
sample_path = self.data_path + '/tif/sample'
ob_path = self.data_path + '/tif/ob'
df_path = self.data_path + '/tif/df'
# without DF
o_norm = Normalization()
o_norm.load(folder=sample_path, auto_gamma_filter=False)
o_norm.load(folder=ob_path, data_type='ob', auto_gamma_filter=False)
_roi_1 = ROI(x0=0, y0=0, x1=1, y1=1)
o_norm.normalization(roi=[_roi_1])
_norm_returned = o_norm.data['normalized'][0]
_norm_expected = np.ones((5, 5))
_norm_expected[:, 2] = 2
_norm_expected[:, 3] = 3
_norm_expected[:, 4] = 4
nbr_col, nbr_row = np.shape(_norm_expected)
for _col in np.arange(nbr_col):
for _row in np.arange(nbr_row):
assert _norm_expected[_col, _row] == _norm_returned[_col, _row]
# with DF
o_norm = Normalization()
o_norm.load(folder=sample_path, auto_gamma_filter=False)
o_norm.load(folder=ob_path, data_type='ob', auto_gamma_filter=False)
o_norm.load(folder=df_path, data_type='df', auto_gamma_filter=False)
_roi_1 = ROI(x0=0, y0=0, x1=1, y1=1)
o_norm.normalization(roi=[_roi_1])
_norm_returned = o_norm.data['normalized'][0]
_norm_expected = np.ones((5, 5))
_norm_expected[:, 2] = 2
_norm_expected[:, 3] = 3
_norm_expected[:, 4] = 4
nbr_col, nbr_row = np.shape(_norm_expected)
for _col in np.arange(nbr_col):
for _row in np.arange(nbr_row):
assert _norm_expected[_col, _row] == _norm_returned[_col, _row]
def test_full_normalization_sample_with_several_roi(self):
"""assert the full normalization works with several roi selected"""
sample_path = self.data_path + '/tif/sample'
ob_path = self.data_path + '/tif/ob'
df_path = self.data_path + '/tif/df'
# without DF
o_norm = Normalization()
o_norm.load(folder=sample_path, auto_gamma_filter=False)
o_norm.load(folder=ob_path, data_type='ob', auto_gamma_filter=False)
# o_norm.load(folder=df_path, data_type='df', auto_gamma_filter=False)
_roi_1 = ROI(x0=0, y0=0, x1=1, y1=1)
_roi_2 = ROI(x0=0, y0=0, x1=1, y1=1)
o_norm.normalization(roi=[_roi_1, _roi_2])
_norm_returned = o_norm.data['normalized'][0]
_norm_expected = np.ones((5, 5))
_norm_expected[:, 2] = 2
_norm_expected[:, 3] = 3
_norm_expected[:, 4] = 4
assert _norm_expected[0, 0] == pytest.approx(_norm_returned[0, 0],
1e-8)
# with DF
o_norm = Normalization()
o_norm.load(folder=sample_path, auto_gamma_filter=False)
o_norm.load(folder=ob_path, data_type='ob', auto_gamma_filter=False)
o_norm.load(folder=df_path, data_type='df', auto_gamma_filter=False)
_roi_1 = ROI(x0=0, y0=0, x1=1, y1=1)
_roi_2 = ROI(x0=0, y0=0, x1=1, y1=1)
o_norm.normalization(roi=[_roi_1, _roi_2])
_norm_returned = o_norm.data['normalized'][0]
_norm_expected = np.ones((5, 5))
_norm_expected[:, 2] = 2
_norm_expected[:, 3] = 3
_norm_expected[:, 4] = 4
assert _norm_expected[0, 0] == pytest.approx(_norm_returned[0, 0],
1e-8)
sample_path = self.data_path + '/tif/special_sample'
ob_path = self.data_path + '/tif/special_ob'
# without DF
o_norm = Normalization()
o_norm.load(folder=sample_path, auto_gamma_filter=False)
o_norm.load(folder=ob_path, data_type='ob', auto_gamma_filter=False)
# o_norm.load(folder=df_path, data_type='df', auto_gamma_filter=False)
_roi_1 = ROI(x0=0, y0=0, x1=0, y1=0)
_roi_2 = ROI(x0=4, y0=4, x1=4, y1=4)
o_norm.normalization(roi=[_roi_1, _roi_2])
_norm_returned = o_norm.data['normalized'][0]
_norm_expected = np.ones((5, 5))
_norm_expected[:, 2] = 2
_norm_expected[:, 3] = 3
_norm_expected[:, 4] = 4
assert _norm_expected[0, 0] == pytest.approx(_norm_returned[0, 0],
1e-8)
def normalization(self, list_rois=None):
if list_rois is None:
self.o_norm.normalization()
else:
list_o_roi = []
for key in list_rois.keys():
roi = list_rois[key]
_x0 = roi['x0']
_y0 = roi['y0']
_x1 = roi['x1']
_y1 = roi['y1']
list_o_roi.append(ROI(x0=_x0, y0=_y0, x1=_x1, y1=_y1))
self.o_norm.normalization(roi=list_o_roi, notebook=True)
def test_x_and_y_correctly_sorted(self):
"""assert x0 and y0 are always the smallest of the x and y values"""
x0 = 1
y0 = 1
x1 = 5
y1 = 10
_roi = ROI(x0=x0, y0=y0, x1=x1, y1=y1)
_x0_expected = 1
_x0_returned = _roi.x0
self.assertTrue(_x0_expected == _x0_returned)
x0 = 1
y0 = 1
width = 5
height = 10
_roi = ROI(x0=x0, y0=y0, width=width, height=height)
_x0_expected = 1
_x0_returned = _roi.x0
self.assertTrue(_x0_expected == _x0_returned)
x0 = 5
y0 = 1
x1 = 1
y1 = 10
_roi = ROI(x0=x0, y0=y0, x1=x1, y1=y1)
_x0_expected = 1
_x0_returned = _roi.x0
self.assertTrue(_x0_expected == _x0_returned)
x0 = 5
y0 = 1
x1 = 1
y1 = 10
_roi = ROI(x0=x0, y0=y0, x1=x1, y1=y1)
_y0_expected = 1
_y0_returned = _roi.y0
self.assertTrue(_y0_expected == _y0_returned)
x0 = 5
y0 = 10
x1 = 1
y1 = 1
_roi = ROI(x0=x0, y0=y0, x1=x1, y1=y1)
_y0_expected = 1
_y0_returned = _roi.y0
self.assertTrue(_y0_expected == _y0_returned)
x0 = 1
y0 = 1
width = 5
height = 10
_roi = ROI(x0=x0, y0=y0, width=width, height=height)
_y0_expected = 1
_y0_returned = _roi.y0
self.assertTrue(_y0_expected == _y0_returned)
def test_normalization_ran_only_once(self):
"""assert normalization is only once if force switch not turn on"""
sample_tif_folder = self.data_path + '/tif/sample'
ob_tif_folder = self.data_path + '/tif/ob'
# testing sample with norm_roi
o_norm = Normalization()
o_norm.load(folder=sample_tif_folder, auto_gamma_filter=False)
o_norm.load(folder=ob_tif_folder,
data_type='ob',
auto_gamma_filter=False)
roi = ROI(x0=0, y0=0, x1=3, y1=2)
o_norm.normalization(roi=roi)
_returned_first_time = o_norm.data['sample']['data'][0]
o_norm.normalization(roi=roi)
_returned_second_time = o_norm.data['sample']['data'][0]
assert (_returned_first_time == _returned_second_time).all()
def normalization(self, list_rois=None):
if list_rois is None:
self.o_norm.normalization()
else:
list_o_roi = []
for key in list_rois.keys():
roi = list_rois[key]
_x0 = roi['x0']
_y0 = roi['y0']
_x1 = roi['x1']
_y1 = roi['y1']
list_o_roi.append(ROI(x0=_x0, y0=_y0, x1=_x1, y1=_y1))
self.o_norm.normalization(roi=list_o_roi, notebook=True)
display(
HTML(
'<span style="font-size: 15px; color:green"> Normalization DONE! </span>'
))
def test_normalization_works_with_1_roi_given_as_a_list(self):
"""Make sure the normalization works when 2 ROI are used"""
sample = self.data_path + '/fits/test_roi/sample.fits'
ob = self.data_path + '/fits/test_roi/ob.fits'
_roi = ROI(x0=0, y0=0, x1=1, y1=2)
list_roi = [_roi]
o_norm = Normalization()
o_norm.load(file=sample, auto_gamma_filter=False)
o_norm.load(file=ob, data_type='ob', auto_gamma_filter=False)
o_norm.normalization(roi=list_roi)
normalized_data = o_norm.get_normalized_data()[0]
expected_normalized_data = np.ones([5, 4])
expected_normalized_data[0:2, 2:4] = .5
expected_normalized_data[3:5, 0:2] = 1.11111111
height, width = np.shape(expected_normalized_data)
for _h in np.arange(height):
for _w in np.arange(width):
assert expected_normalized_data[_h, _w] == pytest.approx(
normalized_data[_h, _w], 1e-5)
def test_full_normalization_sample_divide_by_ob_works(self):
"""assert the full normalization works (when sample is divided by ob)"""
# without normalization roi
sample_path = self.data_path + '/tif/sample'
ob_path = self.data_path + '/tif/ob'
df_path = self.data_path + '/tif/df'
o_norm = Normalization()
o_norm.load(folder=sample_path, auto_gamma_filter=False)
o_norm.load(folder=ob_path, data_type='ob', auto_gamma_filter=False)
o_norm.load(folder=df_path, data_type='df', auto_gamma_filter=False)
o_norm.normalization()
_norm_expected = np.ones((5, 5))
_norm_expected[:, 2] = 2
_norm_expected[:, 3] = 3
_norm_expected[:, 4] = 4
_norm_returned = o_norm.data['normalized']
assert (_norm_expected == _norm_returned).all()
# with normalization roi
sample_path = self.data_path + '/tif/sample'
ob_path = self.data_path + '/tif/ob'
df_path = self.data_path + '/tif/df'
o_norm = Normalization()
o_norm.load(folder=sample_path, auto_gamma_filter=False)
o_norm.load(folder=ob_path, data_type='ob', auto_gamma_filter=False)
o_norm.load(folder=df_path, data_type='df', auto_gamma_filter=False)
_roi = ROI(x0=0, y0=0, x1=2, y1=2)
o_norm.normalization(roi=_roi)
_norm_expected = np.ones((5, 5))
_norm_expected.fill(0.8125)
_norm_expected[:, 2] = 1.625
_norm_expected[:, 3] = 2.4375
_norm_expected[:, 4] = 3.25
_norm_returned = o_norm.data['normalized']
assert (_norm_expected == _norm_returned).all()
o_norm.debugging_roi. # + from NeuNorm.normalization import Normalization from NeuNorm.roi import ROI from pathlib import Path import matplotlib.pyplot as plt # %matplotlib notebook # - my_roi = ROI(x0=118, y0=18, x1=259, y1=489) o_norm.o_norm.normalization(roi=my_roi) #o_norm.o_norm.data['ob'] plt.imshow(o_norm.o_norm.data['ob']['data'][0]) # + divided = o_norm.data.sample[0] / o_norm.data.ob[0] plt.figure() plt.imshow(divided) # - import matplotlib.pyplot as plt