def load_data(directory, do_fit2D=False, do_filtering=False):
"""Loads all data from 'directory' initially to all_data (unsorted list), and then to dictionary structure dataD
folderN1 ---- shot_typeN1 ---- [list of Image_Load instances]
shot_typeN2 ---- [list of Image_Load instances]
....
folderN2 ---- shot_typeN1 ---- [list of Image_Load instances]
shot_typeN2 ---- [list of Image_Load instances]
....
By default does not fit each image 2D-gauss"""
import os, re
dirs = [
os.path.join(directory, dr) for dr in os.listdir(directory)
if re.match(r'[-+]?[0-9.]+ms', dr)
]
all_data = []
w = FloatProgress(min=0, max=len(dirs), value=0)
w.description = 'Loading in progress...'
display(w)
for dr in dirs:
w.value += 1
files = [
os.path.join(dr, fl) for fl in os.listdir(dr)
if re.match(r'.*_\d+.png', fl)
]
for url in files:
new_im = Image_Load(url, do_fit2D, do_filtering)
if new_im.isgood:
all_data.append(new_im)
w.bar_style = 'success'
w.description = 'Loading Done'
# all_data = list(flatten(map(single_directory_load, dirs ,[do_fit2D]*len(dirs), [do_filtering]*len(dirs))))
print('Total number of images: ', len(all_data))
return all_data
def normalise_individual_image(dictionary,
signal_shot,
calibration_shot,
attribute,
index=None,
do_fit2D=False):
"""normalize each image using attribute[index] value - usually 'total' or 'x_data_fit[0]'
returns constracted dictionary (like what returns 'load_data()' function"""
norm_data = dict()
w = FloatProgress(min=0, max=len(dictionary), value=0)
w.description = 'Normalizing in progress...'
display(w)
for folderN, f_dict in dictionary.items():
w.value += 1
calibrated_images = []
for s_elem in f_dict[signal_shot]:
c_elems = [
c_elem for c_elem in f_dict[calibration_shot]
if c_elem.shotN == s_elem.shotN
]
if c_elems == []:
print('s_elem.image_url has no calibration image')
continue
calibrated_images = append(
calibrated_images,
Image_Fitted(
s_elem.image / get_value(c_elems[0], attribute, index),
do_fit2D))
if calibrated_images != []:
norm_data[folderN] = dict()
norm_data[folderN][signal_shot] = calibrated_images
w.bar_style = 'success'
w.description = 'Normalizing Done'
print('Normalization is complited')
return norm_data
def load_data(directory, do_fit2D = False, do_filtering=False):
"""Loads all data from 'directory' initially to all_data (unsorted list), and then to dictionary structure dataD
folderN1 ---- shot_typeN1 ---- [list of Image_Load instances]
shot_typeN2 ---- [list of Image_Load instances]
....
folderN2 ---- shot_typeN1 ---- [list of Image_Load instances]
shot_typeN2 ---- [list of Image_Load instances]
....
By default does not fit each image 2D-gauss"""
import os, re
dirs = [os.path.join(directory,dr) for dr in os.listdir(directory) if re.match(r'[-+]?[0-9.]+ms',dr)]
all_data = []
w = FloatProgress(min=0, max=len(dirs),value=0)
w.description='Loading in progress...'
display(w)
for dr in dirs:
w.value += 1
files = [os.path.join(dr,fl) for fl in os.listdir(dr) if re.match(r'.*_\d+.png',fl)]
for url in files:
new_im = Image_Load(url, do_fit2D, do_filtering)
if new_im.isgood:
all_data.append(new_im)
w.bar_style='success'
w.description = 'Loading Done'
# all_data = list(flatten(map(single_directory_load, dirs ,[do_fit2D]*len(dirs), [do_filtering]*len(dirs))))
print('Total number of images: ', len(all_data))
return all_data
def normalise_avr_image(dictionary, signal_shot, calibration_shot, attribute, index=None, do_fit2D = True):
"""normalize image from evarage dictionary using attribute[index] value - usually 'total' or 'x_data_fit[0]'
returns constracted dictionary (like what returns 'average_data()' function"""
norm_data = dict()
w = FloatProgress(min=0, max=len(dictionary),value=0)
w.description='Normalizing in progress...'
display(w)
for folderN, f_dict in dictionary.items():
w.value += 1
norm_data[folderN] = dict()
norm_data[folderN][signal_shot] = Image_Fitted(f_dict[signal_shot].image /
get_value(f_dict[calibration_shot],attribute,index),do_fit2D)
w.bar_style='success'
w.description = 'Normalizing Done'
print('Normalization is complited')
return norm_data
def rearrange_data(all_data):
dataD = dict()
w = FloatProgress(min=0, max=len(all_data), value=0)
w.description = 'Rearranging in progress...'
display(w)
for elem in all_data:
w.value += 1
if elem.folderN not in dataD:
dataD[elem.folderN] = dict()
d = dataD[elem.folderN]
if elem.shot_typeN not in d:
d[elem.shot_typeN] = []
d[elem.shot_typeN].append(elem)
w.bar_style = 'success'
w.description = 'Rearranging Done'
print('Rearranging to dictionary is complited')
return dataD
def rearrange_data(all_data):
dataD = dict()
w = FloatProgress(min=0, max=len(all_data),value=0)
w.description='Rearranging in progress...'
display(w)
for elem in all_data:
w.value +=1
if elem.folderN not in dataD:
dataD[elem.folderN] = dict()
d = dataD[elem.folderN]
if elem.shot_typeN not in d:
d[elem.shot_typeN] = []
d[elem.shot_typeN].append(elem)
w.bar_style='success'
w.description = 'Rearranging Done'
print('Rearranging to dictionary is complited')
return dataD
def sift(dataD, confidence_interval = 0.1):
"""Sifts (filters) data on empty images by using average information and comperes centers of 1D gaussian fits.
If difference is larger the 'confidence_interval' from the average value, the image would be removed from dataD"""
w = FloatProgress(min=0, max = len(dataD), value=0)
w.description='Sifting in progress...'
display(w)
for folderN, folder_dict in dataD.items():
w.value += 1
for shot_typeN, shot_list in folder_dict.items():
#print(folderN, shot_typeN)
avr_inf = Avr_inf(shot_list, do_fit2D=False)
to_remove = []
for elem in shot_list:
if abs(elem.x_data_fit[1]-avr_inf.x_data_fit[1])/avr_inf.x_data_fit[1] > confidence_interval or abs(elem.y_data_fit[1]-avr_inf.y_data_fit[1])/avr_inf.y_data_fit[1] > confidence_interval:
to_remove.append(elem)
for elem in to_remove:
print('remove element',shot_list.index(elem), elem.image_url )
shot_list.remove(elem)
w.bar_style='success'
w.description = 'Sifting Done'
def normalise_avr_image(dictionary,
signal_shot,
calibration_shot,
attribute,
index=None,
do_fit2D=True):
"""normalize image from evarage dictionary using attribute[index] value - usually 'total' or 'x_data_fit[0]'
returns constracted dictionary (like what returns 'average_data()' function"""
norm_data = dict()
w = FloatProgress(min=0, max=len(dictionary), value=0)
w.description = 'Normalizing in progress...'
display(w)
for folderN, f_dict in dictionary.items():
w.value += 1
norm_data[folderN] = dict()
norm_data[folderN][signal_shot] = Image_Fitted(
f_dict[signal_shot].image /
get_value(f_dict[calibration_shot], attribute, index), do_fit2D)
w.bar_style = 'success'
w.description = 'Normalizing Done'
print('Normalization is complited')
return norm_data
def sift(dataD, confidence_interval=0.1):
"""Sifts (filters) data on empty images by using average information and comperes centers of 1D gaussian fits.
If difference is larger the 'confidence_interval' from the average value, the image would be removed from dataD"""
w = FloatProgress(min=0, max=len(dataD), value=0)
w.description = 'Sifting in progress...'
display(w)
for folderN, folder_dict in dataD.items():
w.value += 1
for shot_typeN, shot_list in folder_dict.items():
#print(folderN, shot_typeN)
avr_inf = Avr_inf(shot_list, do_fit2D=False)
to_remove = []
for elem in shot_list:
if abs(elem.x_data_fit[1] - avr_inf.x_data_fit[1]
) / avr_inf.x_data_fit[1] > confidence_interval or abs(
elem.y_data_fit[1] - avr_inf.y_data_fit[1]
) / avr_inf.y_data_fit[1] > confidence_interval:
to_remove.append(elem)
for elem in to_remove:
print('remove element', shot_list.index(elem), elem.image_url)
shot_list.remove(elem)
w.bar_style = 'success'
w.description = 'Sifting Done'
def status_printer(_, total=None, desc=None, ncols=None):
"""
Manage the printing of an IPython/Jupyter Notebook progress bar widget.
"""
# Fallback to text bar if there's no total
# DEPRECATED: replaced with an 'info' style bar
# if not total:
# return super(tqdm_notebook, tqdm_notebook).status_printer(file)
# fp = file
# Prepare IPython progress bar
try:
if total:
pbar = IProgress(min=0, max=total)
else: # No total? Show info style bar with no progress tqdm status
pbar = IProgress(min=0, max=1)
pbar.value = 1
pbar.bar_style = 'info'
except NameError:
# #187 #451 #558
raise ImportError(
"FloatProgress not found. Please update jupyter and ipywidgets."
" See https://ipywidgets.readthedocs.io/en/stable"
"/user_install.html")
if desc:
pbar.description = desc
if IPYW >= 7:
pbar.style.description_width = 'initial'
# Prepare status text
ptext = HTML()
# Only way to place text to the right of the bar is to use a container
container = HBox(children=[pbar, ptext])
# Prepare layout
if ncols is not None: # use default style of ipywidgets
# ncols could be 100, "100px", "100%"
ncols = str(ncols) # ipywidgets only accepts string
try:
if int(ncols) > 0: # isnumeric and positive
ncols += 'px'
except ValueError:
pass
pbar.layout.flex = '2'
container.layout.width = ncols
container.layout.display = 'inline-flex'
container.layout.flex_flow = 'row wrap'
display(container)
return container
def average_data(dataD, do_fit2D=True):
"""Averages data from dataD to dictionary structure avr_dataD
folderN1 ---- shot_typeN1 ---- Avr_inf instances
shot_typeN2 ---- Avr_inf instances
....
folderN2 ---- shot_typeN1 ---- Avr_inf instances
shot_typeN2 ---- Avr_inf instances
....
By default does fit each average image 2D-gauss"""
avr_dataD = dict()
w = FloatProgress(min=0, max=len(dataD), value=0)
w.description = 'Averaging in progress...'
display(w)
for folderN, folder_dict in dataD.items():
w.value += 1
avr_dataD[folderN] = dict()
temp_dict = avr_dataD[folderN]
for shot_typeN, shot_list in folder_dict.items():
if shot_list != []:
temp_dict[shot_typeN] = Avr_inf(shot_list, do_fit2D)
w.bar_style = 'success'
w.description = 'Averaging Done'
print('Averaging is complited')
return avr_dataD
def normalise_individual_image(dictionary, signal_shot, calibration_shot, attribute, index=None, do_fit2D = False):
"""normalize each image using attribute[index] value - usually 'total' or 'x_data_fit[0]'
returns constracted dictionary (like what returns 'load_data()' function"""
norm_data = dict()
w = FloatProgress(min=0, max=len(dictionary),value=0)
w.description='Normalizing in progress...'
display(w)
for folderN, f_dict in dictionary.items():
w.value += 1
calibrated_images = []
for s_elem in f_dict[signal_shot]:
c_elems = [c_elem for c_elem in f_dict[calibration_shot] if c_elem.shotN == s_elem.shotN]
if c_elems == []:
print('s_elem.image_url has no calibration image')
continue
calibrated_images = append(calibrated_images,
Image_Fitted(s_elem.image / get_value(c_elems[0],attribute,index), do_fit2D))
if calibrated_images != []:
norm_data[folderN] = dict()
norm_data[folderN][signal_shot] = calibrated_images
w.bar_style='success'
w.description = 'Normalizing Done'
print('Normalization is complited')
return norm_data
def average_data(dataD, do_fit2D=True):
"""Averages data from dataD to dictionary structure avr_dataD
folderN1 ---- shot_typeN1 ---- Avr_inf instances
shot_typeN2 ---- Avr_inf instances
....
folderN2 ---- shot_typeN1 ---- Avr_inf instances
shot_typeN2 ---- Avr_inf instances
....
By default does fit each average image 2D-gauss"""
avr_dataD = dict()
w = FloatProgress(min=0, max = len(dataD), value=0)
w.description='Averaging in progress...'
display(w)
for folderN, folder_dict in dataD.items():
w.value += 1
avr_dataD[folderN] = dict()
temp_dict = avr_dataD[folderN]
for shot_typeN, shot_list in folder_dict.items():
if shot_list != []:
temp_dict[shot_typeN] = Avr_inf(shot_list, do_fit2D)
w.bar_style='success'
w.description = 'Averaging Done'
print('Averaging is complited')
return avr_dataD
