예제 #1
0
def load_data(directory, do_fit2D=False, do_filtering=False):
    """Loads all data from 'directory' initially to all_data (unsorted list), and then to dictionary structure dataD
    folderN1  ----    shot_typeN1   ----  [list of Image_Load instances]
                      shot_typeN2   ----  [list of Image_Load instances]
                     ....
    folderN2  ----    shot_typeN1   ----  [list of Image_Load instances]
                      shot_typeN2   ----  [list of Image_Load instances]
                     ....
    By default does not fit each image 2D-gauss"""
    import os, re
    dirs = [
        os.path.join(directory, dr) for dr in os.listdir(directory)
        if re.match(r'[-+]?[0-9.]+ms', dr)
    ]
    all_data = []
    w = FloatProgress(min=0, max=len(dirs), value=0)
    w.description = 'Loading in progress...'
    display(w)
    for dr in dirs:
        w.value += 1
        files = [
            os.path.join(dr, fl) for fl in os.listdir(dr)
            if re.match(r'.*_\d+.png', fl)
        ]
        for url in files:
            new_im = Image_Load(url, do_fit2D, do_filtering)
            if new_im.isgood:
                all_data.append(new_im)
    w.bar_style = 'success'
    w.description = 'Loading Done'
    #     all_data = list(flatten(map(single_directory_load, dirs ,[do_fit2D]*len(dirs), [do_filtering]*len(dirs))))
    print('Total number of images: ', len(all_data))
    return all_data
예제 #2
0
def normalise_individual_image(dictionary,
                               signal_shot,
                               calibration_shot,
                               attribute,
                               index=None,
                               do_fit2D=False):
    """normalize each image using attribute[index] value - usually 'total' or 'x_data_fit[0]'
        returns constracted dictionary (like what returns 'load_data()' function"""
    norm_data = dict()
    w = FloatProgress(min=0, max=len(dictionary), value=0)
    w.description = 'Normalizing in progress...'
    display(w)
    for folderN, f_dict in dictionary.items():
        w.value += 1
        calibrated_images = []
        for s_elem in f_dict[signal_shot]:
            c_elems = [
                c_elem for c_elem in f_dict[calibration_shot]
                if c_elem.shotN == s_elem.shotN
            ]
            if c_elems == []:
                print('s_elem.image_url has no calibration image')
                continue
            calibrated_images = append(
                calibrated_images,
                Image_Fitted(
                    s_elem.image / get_value(c_elems[0], attribute, index),
                    do_fit2D))
        if calibrated_images != []:
            norm_data[folderN] = dict()
            norm_data[folderN][signal_shot] = calibrated_images
    w.bar_style = 'success'
    w.description = 'Normalizing Done'
    print('Normalization is complited')
    return norm_data
def load_data(directory, do_fit2D = False, do_filtering=False):
    """Loads all data from 'directory' initially to all_data (unsorted list), and then to dictionary structure dataD
    folderN1  ----    shot_typeN1   ----  [list of Image_Load instances]
                      shot_typeN2   ----  [list of Image_Load instances]
                     ....
    folderN2  ----    shot_typeN1   ----  [list of Image_Load instances]
                      shot_typeN2   ----  [list of Image_Load instances]
                     ....
    By default does not fit each image 2D-gauss"""
    import os, re
    dirs = [os.path.join(directory,dr) for dr in os.listdir(directory) if re.match(r'[-+]?[0-9.]+ms',dr)]
    all_data = []
    w = FloatProgress(min=0, max=len(dirs),value=0)
    w.description='Loading in progress...'
    display(w)
    for dr in dirs:
        w.value += 1
        files = [os.path.join(dr,fl) for fl in os.listdir(dr) if re.match(r'.*_\d+.png',fl)]
        for url in files:
            new_im = Image_Load(url, do_fit2D, do_filtering)
            if new_im.isgood:
                all_data.append(new_im)
    w.bar_style='success'
    w.description = 'Loading Done'
#     all_data = list(flatten(map(single_directory_load, dirs ,[do_fit2D]*len(dirs), [do_filtering]*len(dirs))))
    print('Total number of images: ', len(all_data))
    return all_data
def normalise_avr_image(dictionary, signal_shot, calibration_shot, attribute, index=None, do_fit2D = True):
    """normalize image from evarage dictionary using attribute[index] value - usually 'total' or 'x_data_fit[0]'
        returns constracted dictionary (like what returns 'average_data()' function"""
    norm_data = dict()
    w = FloatProgress(min=0, max=len(dictionary),value=0)
    w.description='Normalizing in progress...'
    display(w)
    for folderN, f_dict in dictionary.items():
        w.value += 1
        norm_data[folderN] = dict()
        norm_data[folderN][signal_shot] = Image_Fitted(f_dict[signal_shot].image / 
                                          get_value(f_dict[calibration_shot],attribute,index),do_fit2D)
    w.bar_style='success'
    w.description = 'Normalizing Done'
    print('Normalization is complited')
    return norm_data
예제 #5
0
def rearrange_data(all_data):
    dataD = dict()
    w = FloatProgress(min=0, max=len(all_data), value=0)
    w.description = 'Rearranging in progress...'
    display(w)
    for elem in all_data:
        w.value += 1
        if elem.folderN not in dataD:
            dataD[elem.folderN] = dict()
        d = dataD[elem.folderN]
        if elem.shot_typeN not in d:
            d[elem.shot_typeN] = []
        d[elem.shot_typeN].append(elem)
    w.bar_style = 'success'
    w.description = 'Rearranging Done'
    print('Rearranging to dictionary is complited')
    return dataD
def rearrange_data(all_data):
    dataD = dict()
    w = FloatProgress(min=0, max=len(all_data),value=0)
    w.description='Rearranging in progress...'
    display(w)
    for elem in all_data:
        w.value +=1
        if elem.folderN not in dataD:
            dataD[elem.folderN] = dict()
        d = dataD[elem.folderN]
        if elem.shot_typeN not in d:
            d[elem.shot_typeN] = []
        d[elem.shot_typeN].append(elem)
    w.bar_style='success'
    w.description = 'Rearranging Done'
    print('Rearranging to dictionary is complited')
    return dataD
def sift(dataD, confidence_interval = 0.1):
    """Sifts (filters) data on empty images by using average information and comperes centers  of 1D gaussian fits.
    If difference is larger the 'confidence_interval' from the average value, the image would be removed from dataD"""
    w = FloatProgress(min=0, max = len(dataD), value=0)
    w.description='Sifting in progress...'
    display(w)
    for folderN, folder_dict in dataD.items():
        w.value += 1
        for shot_typeN, shot_list in folder_dict.items():
            #print(folderN, shot_typeN)
            avr_inf = Avr_inf(shot_list, do_fit2D=False)
            to_remove = []
            for elem in shot_list:
                if abs(elem.x_data_fit[1]-avr_inf.x_data_fit[1])/avr_inf.x_data_fit[1] > confidence_interval or                     abs(elem.y_data_fit[1]-avr_inf.y_data_fit[1])/avr_inf.y_data_fit[1] > confidence_interval:
                        to_remove.append(elem)
            for elem in to_remove:
                print('remove element',shot_list.index(elem), elem.image_url )
                shot_list.remove(elem)
    w.bar_style='success'
    w.description = 'Sifting Done'
예제 #8
0
def normalise_avr_image(dictionary,
                        signal_shot,
                        calibration_shot,
                        attribute,
                        index=None,
                        do_fit2D=True):
    """normalize image from evarage dictionary using attribute[index] value - usually 'total' or 'x_data_fit[0]'
        returns constracted dictionary (like what returns 'average_data()' function"""
    norm_data = dict()
    w = FloatProgress(min=0, max=len(dictionary), value=0)
    w.description = 'Normalizing in progress...'
    display(w)
    for folderN, f_dict in dictionary.items():
        w.value += 1
        norm_data[folderN] = dict()
        norm_data[folderN][signal_shot] = Image_Fitted(
            f_dict[signal_shot].image /
            get_value(f_dict[calibration_shot], attribute, index), do_fit2D)
    w.bar_style = 'success'
    w.description = 'Normalizing Done'
    print('Normalization is complited')
    return norm_data
예제 #9
0
def sift(dataD, confidence_interval=0.1):
    """Sifts (filters) data on empty images by using average information and comperes centers  of 1D gaussian fits.
    If difference is larger the 'confidence_interval' from the average value, the image would be removed from dataD"""
    w = FloatProgress(min=0, max=len(dataD), value=0)
    w.description = 'Sifting in progress...'
    display(w)
    for folderN, folder_dict in dataD.items():
        w.value += 1
        for shot_typeN, shot_list in folder_dict.items():
            #print(folderN, shot_typeN)
            avr_inf = Avr_inf(shot_list, do_fit2D=False)
            to_remove = []
            for elem in shot_list:
                if abs(elem.x_data_fit[1] - avr_inf.x_data_fit[1]
                       ) / avr_inf.x_data_fit[1] > confidence_interval or abs(
                           elem.y_data_fit[1] - avr_inf.y_data_fit[1]
                       ) / avr_inf.y_data_fit[1] > confidence_interval:
                    to_remove.append(elem)
            for elem in to_remove:
                print('remove element', shot_list.index(elem), elem.image_url)
                shot_list.remove(elem)
    w.bar_style = 'success'
    w.description = 'Sifting Done'
예제 #10
0
    def status_printer(_, total=None, desc=None, ncols=None):
        """
        Manage the printing of an IPython/Jupyter Notebook progress bar widget.
        """
        # Fallback to text bar if there's no total
        # DEPRECATED: replaced with an 'info' style bar
        # if not total:
        #    return super(tqdm_notebook, tqdm_notebook).status_printer(file)

        # fp = file

        # Prepare IPython progress bar
        try:
            if total:
                pbar = IProgress(min=0, max=total)
            else:  # No total? Show info style bar with no progress tqdm status
                pbar = IProgress(min=0, max=1)
                pbar.value = 1
                pbar.bar_style = 'info'
        except NameError:
            # #187 #451 #558
            raise ImportError(
                "FloatProgress not found. Please update jupyter and ipywidgets."
                " See https://ipywidgets.readthedocs.io/en/stable"
                "/user_install.html")

        if desc:
            pbar.description = desc
            if IPYW >= 7:
                pbar.style.description_width = 'initial'
        # Prepare status text
        ptext = HTML()
        # Only way to place text to the right of the bar is to use a container
        container = HBox(children=[pbar, ptext])
        # Prepare layout
        if ncols is not None:  # use default style of ipywidgets
            # ncols could be 100, "100px", "100%"
            ncols = str(ncols)  # ipywidgets only accepts string
            try:
                if int(ncols) > 0:  # isnumeric and positive
                    ncols += 'px'
            except ValueError:
                pass
            pbar.layout.flex = '2'
            container.layout.width = ncols
            container.layout.display = 'inline-flex'
            container.layout.flex_flow = 'row wrap'
        display(container)

        return container
예제 #11
0
def average_data(dataD, do_fit2D=True):
    """Averages data from dataD to dictionary structure avr_dataD
    folderN1  ----    shot_typeN1   ----  Avr_inf instances
                      shot_typeN2   ----  Avr_inf instances
                     ....
    folderN2  ----    shot_typeN1   ----  Avr_inf instances
                      shot_typeN2   ----  Avr_inf instances
                     ....
    By default does fit each average image 2D-gauss"""
    avr_dataD = dict()
    w = FloatProgress(min=0, max=len(dataD), value=0)
    w.description = 'Averaging in progress...'
    display(w)
    for folderN, folder_dict in dataD.items():
        w.value += 1
        avr_dataD[folderN] = dict()
        temp_dict = avr_dataD[folderN]
        for shot_typeN, shot_list in folder_dict.items():
            if shot_list != []:
                temp_dict[shot_typeN] = Avr_inf(shot_list, do_fit2D)
    w.bar_style = 'success'
    w.description = 'Averaging Done'
    print('Averaging is complited')
    return avr_dataD
def normalise_individual_image(dictionary, signal_shot, calibration_shot, attribute, index=None, do_fit2D = False):
    """normalize each image using attribute[index] value - usually 'total' or 'x_data_fit[0]'
        returns constracted dictionary (like what returns 'load_data()' function"""
    norm_data = dict()
    w = FloatProgress(min=0, max=len(dictionary),value=0)
    w.description='Normalizing in progress...'
    display(w)
    for folderN, f_dict in dictionary.items():
        w.value += 1
        calibrated_images = []
        for s_elem in f_dict[signal_shot]:
            c_elems = [c_elem for c_elem in f_dict[calibration_shot] if c_elem.shotN == s_elem.shotN]
            if c_elems == []:
                print('s_elem.image_url has no calibration image')
                continue
            calibrated_images = append(calibrated_images, 
                                       Image_Fitted(s_elem.image / get_value(c_elems[0],attribute,index), do_fit2D))
        if calibrated_images != []:
            norm_data[folderN] = dict()
            norm_data[folderN][signal_shot] = calibrated_images
    w.bar_style='success'
    w.description = 'Normalizing Done'
    print('Normalization is complited')
    return norm_data
def average_data(dataD, do_fit2D=True):
    """Averages data from dataD to dictionary structure avr_dataD
    folderN1  ----    shot_typeN1   ----  Avr_inf instances
                      shot_typeN2   ----  Avr_inf instances
                     ....
    folderN2  ----    shot_typeN1   ----  Avr_inf instances
                      shot_typeN2   ----  Avr_inf instances
                     ....
    By default does fit each average image 2D-gauss"""
    avr_dataD = dict()
    w = FloatProgress(min=0, max = len(dataD), value=0)
    w.description='Averaging in progress...'
    display(w)
    for folderN, folder_dict in dataD.items():
        w.value += 1
        avr_dataD[folderN] = dict()
        temp_dict = avr_dataD[folderN]
        for shot_typeN, shot_list in folder_dict.items():
            if shot_list != []:
                temp_dict[shot_typeN] = Avr_inf(shot_list, do_fit2D)
    w.bar_style='success'
    w.description = 'Averaging Done'
    print('Averaging is complited')
    return avr_dataD