コード例 #1
0
def main():
    """
    Tme main function
    """
    #nx_quad = 1056 # For Tempo
    #ny_quad = 1046 # For Tempo
    #nlat = nx_quad*2
    #nspec = ny_quad*2
    file_path = r'C:\Users\nmishra\Workspace\TEMPO\Data\GroundTest\FPS\Integration_Sweep\Dark\saved_quads'
    save_dir = r'C:\Users\nmishra\Workspace\TEMPO\outlier_mask'
    data_path_all = [each for each in os.listdir(file_path) if each.endswith('.sav')]
    #names_to_check = ['SHORT', 'NOM', 'LONG']
    names_to_check = ['SHORT', 'NOM','OP', 'LONG']

    mean_mask_all_lower = []
    median_mask_all_lower = []    
    mean_mask_all_upper = []
    median_mask_all_upper = []
    saturation_mask_upper = []
    saturation_mask_lower = []
    for data_path in data_path_all:
    
    
        print(data_path)
        full_frame, collection_type, frames,int_time = \
        make_quads_from_IDL_sav_file(file_path, data_path, names_to_check)
        # all_full_frame contains all 100 frames
        
        #Let's fetch the quads now
        #quad_A = full_frame[0:ny_quad, 0:nx_quad]
         
        quad_A = full_frame[:, 0, :, :]
        active_quad_A = np.mean(quad_A[:, 4:1028, 10:1034], axis=0)        
     
        quad_B = full_frame[:, 1, :, :]
        active_quad_B = np.mean(quad_B[:, 4:1028, 10:1034], axis=0)
      
        quad_C = full_frame[:, 2, :, :]
        active_quad_C = np.mean(quad_C[:, 4:1028, 10:1034], axis=0)
        
        quad_D = full_frame[:, 3, :, :]
        active_quad_D = np.mean(quad_D[:, 4:1028, 10:1034], axis=0)
       
        lower_quads = np.concatenate((active_quad_A, active_quad_B), axis=1)
        upper_quads = np.concatenate((active_quad_D, active_quad_C), axis=1)
        active_quads =[lower_quads, upper_quads]   
        # for overclocks
        #lower_quads = np.concatenate((bias_A, bias_B), axis=1)
        #upper_quads = np.concatenate((bias_D, bias_C), axis=1)
        active_quads =[lower_quads, upper_quads]   
            
        quads = ['Lower_quads','Upper_quads']
        
        # Now let's save the full_frame images
        plot_dir = save_dir
        image_dir = 'Quad_images_'
        save_quad_images = os.path.join(plot_dir, image_dir)
        
        if not os.path.exists(save_quad_images):
            os.makedirs(save_quad_images)
        image_title = 'Full Frame Quad Image'
#        plot_full_frame_image(full_frame, image_title, collection_type, frames,
#                              int_time, save_quad_images)
#       
        # Create list of directories To save histograms before and after
        #outlier rejections
        
        for k in np.arange(0,2):
            #mean_plot = r'Outlier_mean_tsoc'+'/'+ quads[k]
            median_plot = r'Outlier_median'+'/'+ quads[k]
            folder_name_hist = 'Hist_plot'
            folder_name_mask = 'Mask_plot'
            folder_name_sat = 'Saturation_plot'
            
            
    
            save_median_hist = os.path.join(plot_dir, median_plot, folder_name_hist)
            if not os.path.exists(save_median_hist):
                os.makedirs(save_median_hist)
    
            
    
            save_median_mask = os.path.join(plot_dir, median_plot, folder_name_mask)
            if not os.path.exists(save_median_mask):
                os.makedirs(save_median_mask)
             
            save_sat_mask = os.path.join(plot_dir, median_plot, folder_name_sat)
            if not os.path.exists(save_sat_mask):
                os.makedirs(save_sat_mask)
            
            # Now let's call the outlier functions. First identify saturated
            #bits and then identify the 'hot and cold' outlier pixels
            
            if k == 0:
                title = 'Histogram of Quads A & B'
            else:
                title = 'Histogram of Quads C & D'
            
            sat_pixels = identify_saturation(active_quads[k],
                                                         save_sat_mask,
                                                         collection_type,
                                                         frames, int_time)            
            
            #outlier_filt_mean, outlier_mean = reject_outlier_mean(active_quads[k])
    
            outlier_filt_med, outlier_med = reject_outlier_median(active_quads[k])
    
               
    
            # Ok now lets' plot the histograms to understand what we see.
            # First the mean approach
            print('ok, let us plot the histograms now')
            
            #plot_hist_image(active_quads[k], title, collection_type, frames,
                            #outlier_filt_mean, outlier_mean, int_time,
                           # save_mean_hist)
            
            
            
            # Secondly the median absolute difference approach
            plot_hist_image(active_quads[k], title, collection_type, frames,
                            outlier_filt_med, outlier_med, int_time,
                            save_median_hist)
            #outlier_med= None
            # Ok, lets now create a binary mask of outlier pixels
            # First with the mean based approach
    
    
#            mean_mask = create_outlier_mask(active_quads[k], outlier_mean[0],
#                                            collection_type,
#                                            frames, save_mean_mask)
#            if k == 0:
#                mean_mask_all_lower.append(mean_mask)
#                
#            else :
#                mean_mask_all_upper.append(mean_mask)
#    
#            # Secondly, the same for median ased approach
          
            if len(outlier_med) == 1:
                outlier_med = 0
            else:
                outlier_med = outlier_med[1]
                
            if k == 0:
                title = 'Binary Outlier Mask of Quads A & B'
            else:
                title = 'Binary Outlier Mask of Quads C & D'
            median_mask = create_outlier_mask(active_quads[k], outlier_med,
                                              title, collection_type, frames,
                                              save_median_mask)
            
            
            active_quads[k] = None
            outlier_med = None
                        
                
            if k == 0:
                median_mask_all_lower.append(median_mask[:])
                median_mask=None
                saturation_mask_lower.append(sat_pixels[:])
                sat_pixels = None
            elif k==1 :
                median_mask_all_upper.append(median_mask[:])
                median_mask = None
                saturation_mask_upper.append(sat_pixels[:])
                sat_pixels = None
                   
                
   #*************************************************************************** 
    # The following function creates final mask based on 80% occurence criteria
    
    final_path = r'C:\Users\nmishra\Workspace\TEMPO\outlier_mask\Outlier_median\save_masks'
    if not os.path.exists(final_path):
                os.makedirs(final_path)
    quad_name = 'lower_quads'
    title = 'Quads A & B outlier mask (80% criteria)'
    final_mask_lower = create_final_mask(median_mask_all_lower, quad_name, title, final_path)
    quad_name = 'upper_quads'
    title = 'Quads C & D outlier mask (80% criteria)'
    final_mask_upper = create_final_mask(median_mask_all_upper, quad_name, title, final_path)
    final_outlier_mask = np.concatenate((final_mask_lower, final_mask_upper), 
                                         axis=0)
    plt.figure()
    plt.imshow(final_outlier_mask, cmap='bwr', 
               interpolation='none', origin='lower')
    #cbar = plt.colorbar(image)
    nx_quad, ny_quad = np.array(final_outlier_mask).shape
    final_outliers = np.array(np.where(np.reshape(final_outlier_mask,
                                          (nx_quad*ny_quad, 1))==0)).shape[1]     
    plt.title('TEMPO outlier Mask (outliers = '+ str(final_outliers)+')',
              fontsize=14)
    plt.xlabel('# of spatial pixels', fontsize=12)
    plt.ylabel('# of spectral pixels', fontsize=12)
    plt.grid(False)
    
    #final_path = r'C:\Users\nmishra\Workspace\TEMPO\Data\GroundTest\FPS\Integration_Sweep\Dark\saved_quads\Outlier_median\save_masks'
    if not os.path.exists(final_path):
                os.makedirs(final_path)
    plt.savefig(final_path+'/'+'final_mask.png', dpi=100, bbox_inches="tight")
   #*************************************************************************** 

   #*************************************************************************** 
    # if we want to 'OR" all the mask, use the following function
    quad_name = 'lower_quads_or'  
    title = 'Quads A & B outlier mask (Logical OR)'
    final_mask_lower_ored= create_ORed_mask(median_mask_all_lower, quad_name, title, final_path)
    quad_name = 'upper_quads_ored'
    title = 'Quads C & D outlier mask (Logical OR)'
    final_mask_upper_ored = create_ORed_mask(median_mask_all_upper, quad_name, title, final_path)
    final_outlier_mask_ored = np.concatenate((final_mask_lower_ored,
                                              final_mask_upper_ored), axis=0)
    mask_directory = r'C:\Users\nmishra\Workspace\TEMPO\outlier_mask\Outlier_median\save_masks'
    mask_name = mask_directory+'/'+'final_outlier_mask_2_sigma.csv'	
    np.savetxt(mask_name, np.array(final_outlier_mask_ored), delimiter =",")
    plt.figure()
    plt.imshow(np.invert(final_outlier_mask_ored), cmap='bwr', 
               interpolation='none', origin='lower')
    #cbar = plt.colorbar(image)
    nx_quad, ny_quad = np.array(final_outlier_mask_ored).shape
    final_outliers_ored = np.array(np.where(np.reshape(final_outlier_mask_ored,
                                          (nx_quad*ny_quad, 1))==1)).shape[1]     
    plt.title('TEMPO Outlier Mask (outliers = '+ str(final_outliers_ored)+')',
              fontsize=14)
    plt.xlabel('# of spatial pixels', fontsize=12)
    plt.ylabel('# of spectral pixels', fontsize=12)
    plt.grid(False)    
    
    plt.savefig(final_path+'/'+'final_mask_ored.png', dpi=100, bbox_inches="tight")
コード例 #2
0
def main():
    """
    Tme main function
    """
    #nx_quad = 1056 # For Tempo
    #ny_quad = 1046 # For Tempo
    #nlat = nx_quad*2
    #nspec = ny_quad*2
    file_path = r'C:\Users\nmishra\Workspace\TEMPO\Data\GroundTest\FPS\Dark\FPS_Dark\saved_quads'
    data_path_all = [
        each for each in os.listdir(file_path) if each.endswith('.sav')
    ]
    #names_to_check = ['SHORT', 'NOM', 'LONG']
    names_to_check = ['SHORT', 'NOM', 'OP', 'LONG']

    mean_mask_all_lower = []
    median_mask_all_lower = []
    mean_mask_all_upper = []
    median_mask_all_upper = []
    saturation_mask_upper = []
    saturation_mask_lower = []
    for data_path in data_path_all:

        # Now let's find the integtration type
        print(data_path)
        full_frame, collection_type, frames, int_time = \
        make_quads_from_IDL_sav_file(file_path, data_path, names_to_check)
        #Let's fetch the quads now
        #quad_A = full_frame[0:ny_quad, 0:nx_quad]
        active_quad_A = np.array(full_frame[4:1028, 10:1034])
        #quad_B = full_frame[ny_quad:, 0:nx_quad]
        active_quad_B = np.array(full_frame[4:1028, 1078:2102])
        #quad_C = full_frame[ny_quad:, nx_quad:]
        active_quad_C = np.array(full_frame[1064:2088, 1078:2102])
        #quad_D = full_frame[ny_quad:, 0:nx_quad]
        active_quad_D = np.array(full_frame[1064:2088, 10:1034])
        lower_quads = np.concatenate((active_quad_A, active_quad_B), axis=1)
        upper_quads = np.concatenate((active_quad_D, active_quad_C), axis=1)
        active_quads = [lower_quads, upper_quads]

        #quads = ['Lower_quads','Upper_quads']

        # Now let's save the full_frame images
        plot_dir = file_path
        image_dir = 'Quad_images'
        save_quad_images = os.path.join(plot_dir, image_dir)

        if not os.path.exists(save_quad_images):
            os.makedirs(save_quad_images)
        image_title = 'Full Frame Quad Image'
        plot_full_frame_image(full_frame, image_title, collection_type, frames,
                              int_time, save_quad_images)
        # Create list of directories To save histograms before and after
        #outlier rejections

        mean_plot = r'Outlier_mean' + '/' + 'Lower_quads'
        median_plot = r'Outlier_median' + '/' + 'Lower_quads'
        folder_name_hist = 'Hist_plot'
        folder_name_mask = 'Mask_plot'
        folder_name_sat = 'Saturation_plot'

        save_mean_hist = os.path.join(plot_dir, mean_plot, folder_name_hist)
        if not os.path.exists(save_mean_hist):
            os.makedirs(save_mean_hist)

        save_median_hist = os.path.join(plot_dir, median_plot,
                                        folder_name_hist)
        if not os.path.exists(save_median_hist):
            os.makedirs(save_median_hist)

        save_mean_mask = os.path.join(plot_dir, mean_plot, folder_name_mask)
        if not os.path.exists(save_mean_mask):
            os.makedirs(save_mean_mask)

        save_median_mask = os.path.join(plot_dir, median_plot,
                                        folder_name_mask)
        if not os.path.exists(save_median_mask):
            os.makedirs(save_median_mask)

        save_sat_mask = os.path.join(plot_dir, median_plot, folder_name_sat)
        if not os.path.exists(save_sat_mask):
            os.makedirs(save_sat_mask)

        # Now let's call the outlier functions. First identify saturated
        #bits and then identify the 'hot and cold' outlier pixels

        title = 'Histogram of Quads A & B'

        #title = 'Histogram of Quads C & D'

        sat_pixels = identify_saturation(lower_quads, save_sat_mask,
                                         collection_type, frames, int_time)

        #outlier_filt_mean, outlier_mean = reject_outlier_mean(active_quads[k])

        outlier_filt_med, outlier_med = reject_outlier_median(lower_quads)

        # Ok now lets' plot the histograms to understand what we see.
        # First the mean approach
        print('ok, let us plot the histograms now')

        #plot_hist_image(active_quads[k], title, collection_type, frames,
        #outlier_filt_mean, outlier_mean, int_time,
        # save_mean_hist)

        # Secondly the median absolute difference approach
        plot_hist_image(lower_quads, title, collection_type, frames,
                        outlier_filt_med, outlier_med, int_time,
                        save_median_hist)
        #outlier_med= None
        # Ok, lets now create a binary mask of outlier pixels
        # First with the mean based approach

        #            mean_mask = create_outlier_mask(active_quads[k], outlier_mean[0],
        #                                            collection_type,
        #                                            frames, save_mean_mask)
        #            if k == 0:
        #                mean_mask_all_lower.append(mean_mask)
        #
        #            else :
        #                mean_mask_all_upper.append(mean_mask)
        #
        #            # Secondly, the same for median ased approach

        #if len(outlier_med) == 1:
        #   outlier_med = 0
        #else:
        #outlier_med = outlier_med[1]
        median_mask = create_outlier_mask(lower_quads, outlier_med[1],
                                          collection_type, frames,
                                          save_median_mask)
        lower_quads = None
        outlier_med = None

        median_mask_all_lower.append(median_mask[:])
        median_mask = None
        saturation_mask_lower.append(sat_pixels[:])

    # Ok now, lets' or all the median masks

    print(np.array(median_mask_all_lower).shape)
    final_mask_median_lower = np.bitwise_or.reduce(median_mask_all_lower[:])

    plt.figure()
    plt.imshow(np.invert(final_mask_median_lower),
               cmap='bwr',
               interpolation='none',
               origin='lower')
    #cbar = plt.colorbar(image)
    plt.title('Binary outlier mask', fontsize=14)
    plt.xlabel('# of spatial pixels', fontsize=12)
    plt.ylabel('# of spectral pixels', fontsize=12)
    plt.grid(False)
    plt.savefig(r'C:\Users\nmishra\Desktop' + '/1.png',
                dpi=100,
                bbox_inches="tight")
    cc

    plt.figure()
    plt.imshow(np.invert(median_mask_all_lower[1]),
               cmap='bwr',
               interpolation='none',
               origin='lower')
    #cbar = plt.colorbar(image)
    plt.title('Binary outlier mask', fontsize=14)
    plt.xlabel('# of spatial pixels', fontsize=12)
    plt.ylabel('# of spectral pixels', fontsize=12)
    plt.grid(False)
    plt.savefig(r'C:\Users\nmishra\Desktop' + '/2.png',
                dpi=100,
                bbox_inches="tight")
コード例 #3
0
def main():
    """
    Tme main function
    """
    #nx_quad = 1056 # For Tempo
    #ny_quad = 1046 # For Tempo
    #nlat = nx_quad*2
    #nspec = ny_quad*2
    file_path = r'F:\TEMPO\Data\GroundTest\FPS\Spectrometer\2017.06.28\saved_quads'
    save_dir = r'C:\Users\nmishra\Workspace\TEMPO\outlier_mask_spectrometer'
    all_int_files = [
        each for each in os.listdir(file_path) if each.endswith('.sav')
    ]
    #data_path_all = [items for items in all_int_files if not items.endswith('_INT_0.dat.sav')]

    #names_to_check = ['SHORT', 'NOM', 'LONG']
    #names_to_check = ['SHORT', 'NOM','OP', 'LONG']
    median_mask_A = []
    median_mask_B = []
    median_mask_C = []
    median_mask_D = []
    saturation_mask_A = []
    saturation_mask_B = []
    saturation_mask_C = []
    saturation_mask_D = []
    for data_path in all_int_files:
        print(data_path)
        data_file = os.path.join(file_path, data_path)
        IDL_variable = readsav(data_file)
        full_frame = IDL_variable.quads
        active_quad_A = full_frame[0, 4:1028, 10:1034]
        active_quad_B = full_frame[1, 4:1028, 10:1034]
        active_quad_C = full_frame[2, 4:1028, 10:1034]
        active_quad_D = full_frame[3, 4:1028, 10:1034]
        active_quads = [
            active_quad_A, active_quad_B, active_quad_C, active_quad_D
        ]
        quads = ['Quad A', 'Quad B', 'Quad C', 'Quad D']
        # Now let's save the full_frame images
        int_time = 139.97
        collection_type = 'Nom Int.'
        frames = data_path.split('.')[0]

        for k in np.arange(0, 4):
            #mean_plot = r'Outlier_mean_tsoc'+'/'+ quads[k]

            median_plot = r'Outlier_median' + '/' + quads[k]
            image_dir = 'Saved_Images'
            folder_name_hist = 'Hist_plot'
            folder_name_mask = 'Mask_plot'
            folder_name_sat = 'Saturation_plot'

            save_median_hist = os.path.join(save_dir, median_plot,
                                            folder_name_hist)
            if not os.path.exists(save_median_hist):
                os.makedirs(save_median_hist)

            saved_images = os.path.join(save_dir, median_plot, image_dir)
            if not os.path.exists(saved_images):
                os.makedirs(saved_images)
            title = quads[0] + 'image (Spectrometer)'
            figure_name = saved_images + '/' + frames + '.png'
            create_image(active_quads[k], title, figure_name)

            save_median_mask = os.path.join(save_dir, median_plot,
                                            folder_name_mask)
            if not os.path.exists(save_median_mask):
                os.makedirs(save_median_mask)

            save_sat_mask = os.path.join(save_dir, median_plot,
                                         folder_name_sat)
            if not os.path.exists(save_sat_mask):
                os.makedirs(save_sat_mask)

            # Now let's call the outlier functions. First identify saturated
            #bits and then identify the 'hot and cold' outlier pixels

            title = 'Histogram of ' + quads[k]

            sat_pixels = identify_saturation(active_quads[k], save_sat_mask,
                                             collection_type, frames, int_time)

            #outlier_filt_mean, outlier_mean = reject_outlier_mean(active_quads[k])

            outlier_filt_med, outlier_med = reject_outlier_median(
                active_quads[k])

            # Ok now lets' plot the histograms to understand what we see.
            # First the mean approach
            print('ok, let us plot the histograms now')

            plot_hist_image(active_quads[k], title, collection_type, frames,
                            outlier_filt_med, outlier_med, int_time,
                            save_median_hist)

            if len(outlier_med) == 1:
                outlier_med = 0
            else:
                outlier_med = outlier_med[1]
            title = 'Binary Outlier Mask of ' + quads[k]
            median_mask = create_outlier_mask(active_quads[k], outlier_med,
                                              title, collection_type, frames,
                                              save_median_mask)
            active_quads[k] = None
            outlier_med = None
            if k == 0:
                median_mask_A.append(median_mask[:])
                median_mask = None
                saturation_mask_A.append(sat_pixels[:])
                sat_pixels = None
            elif k == 1:
                median_mask_B.append(median_mask[:])
                median_mask = None
                saturation_mask_B.append(sat_pixels[:])
                sat_pixels = None

            elif k == 2:
                median_mask_C.append(median_mask[:])
                median_mask = None
                saturation_mask_C.append(sat_pixels[:])
                sat_pixels = None

            elif k == 3:
                median_mask_D.append(median_mask[:])
                median_mask = None
                saturation_mask_D.append(sat_pixels[:])
                sat_pixels = None

#***************************************************************************
# The following function creates final mask based on 80% occurence criteria

    final_path = r'C:\Users\nmishra\Workspace\TEMPO\outlier_mask_spectrometer\Outlier_median\save_masks_80%'
    if not os.path.exists(final_path):
        os.makedirs(final_path)
    quad_name = 'Quad A'
    title = 'Quad A outlier mask (50% criteria)'
    final_mask_A = create_final_mask(median_mask_A, quad_name, title,
                                     final_path)
    mask_name_A = final_path + '/' + 'quad_A_outlier_mask.csv'
    np.savetxt(mask_name_A, np.array(final_mask_A), delimiter=",")

    quad_name = 'Quad B'
    title = 'Quad B outlier mask (50% criteria)'
    final_mask_B = create_final_mask(median_mask_B, quad_name, title,
                                     final_path)
    mask_name_B = final_path + '/' + 'quad_B_outlier_mask.csv'
    np.savetxt(mask_name_B, np.array(final_mask_B), delimiter=",")

    quad_name = 'Quad C'
    title = 'Quad C outlier mask (50% criteria)'
    final_mask_C = create_final_mask(median_mask_C, quad_name, title,
                                     final_path)
    mask_name_C = final_path + '/' + 'quad_C_outlier_mask.csv'
    np.savetxt(mask_name_C, np.array(final_mask_C), delimiter=",")

    quad_name = 'Quad D'
    title = 'Quad D outlier mask (50% criteria)'
    final_mask_D = create_final_mask(median_mask_D, quad_name, title,
                                     final_path)
    mask_name_D = final_path + '/' + 'quad_D_outlier_mask.csv'
    np.savetxt(mask_name_D, np.array(final_mask_D), delimiter=",")

    lower_quads = np.concatenate((final_mask_A, final_mask_B), axis=1)
    upper_quads = np.concatenate((final_mask_D, final_mask_C), axis=1)
    final_outlier_mask = np.concatenate((lower_quads, upper_quads), axis=0)

    mask_name = final_path + '/' + 'final_outlier_mask_80%.csv'
    np.savetxt(mask_name, np.array(final_outlier_mask), delimiter=",")

    plt.figure()
    final_outlier_mask = np.invert(final_outlier_mask)
    plt.imshow(final_outlier_mask,
               cmap='bwr',
               interpolation='none',
               origin='lower')
    #cbar = plt.colorbar(image)
    nx_quad, ny_quad = np.array(final_outlier_mask).shape
    final_outliers = np.array(
        np.where(np.reshape(final_outlier_mask, (nx_quad * ny_quad,
                                                 1)) == 0)).shape[1]
    plt.title(
        'TEMPO outlier Mask (50% Occurence Criteria)\n (Num. outliers = ' +
        str(final_outliers) + ')',
        fontsize=14)
    plt.xlabel('# of spatial pixels', fontsize=12)
    plt.ylabel('# of spectral pixels', fontsize=12)
    plt.grid(False)
    plt.savefig(final_path + '/' + 'final_mask_50%.png',
                dpi=200,
                bbox_inches="tight")

    #***************************************************************************

    #***************************************************************************
    # if we want to 'OR" all the mask, use the following function

    final_path = r'C:\Users\nmishra\Workspace\TEMPO\outlier_mask_spectrometer\Outlier_median\save_masks_logical_OR'
    if not os.path.exists(final_path):
        os.makedirs(final_path)
    quad_name = 'Quad A'
    title = 'Quads A outlier mask (Logical OR)'
    final_mask_A = create_ORed_mask(median_mask_A, quad_name, title,
                                    final_path)
    mask_name_A = final_path + '/' + 'quad_A_outlier_mask_ored.csv'
    np.savetxt(mask_name_A, np.array(final_mask_A), delimiter=",")

    quad_name = 'Quad B'
    title = 'Quads B outlier mask (Logical OR)'
    final_mask_B = create_ORed_mask(median_mask_B, quad_name, title,
                                    final_path)
    mask_name_B = final_path + '/' + 'quad_B_outlier_mask_ored.csv'
    np.savetxt(mask_name_B, np.array(final_mask_B), delimiter=",")

    quad_name = 'Quad C'
    title = 'Quads C outlier mask (Logical OR)'
    final_mask_C = create_ORed_mask(median_mask_C, quad_name, title,
                                    final_path)
    mask_name_C = final_path + '/' + 'quad_C_outlier_mask_ored.csv'
    np.savetxt(mask_name_C, np.array(final_mask_C), delimiter=",")

    quad_name = 'Quad D'
    title = 'Quads D outlier mask (Logical OR)'
    final_mask_D = create_ORed_mask(median_mask_D, quad_name, title,
                                    final_path)
    mask_name_D = final_path + '/' + 'quad_D_outlier_mask_ored.csv'
    np.savetxt(mask_name_D, np.array(final_mask_D), delimiter=",")

    quad_name = 'upper_quads_ored'
    title = 'Quads C & D outlier mask (Logical OR)'

    lower_quads = np.concatenate((final_mask_A, final_mask_B), axis=1)
    upper_quads = np.concatenate((final_mask_D, final_mask_C), axis=1)
    final_outlier_mask_ored = np.concatenate((lower_quads, upper_quads),
                                             axis=0)
    mask_name = final_path + '/' + 'final_outlier_mask_ored.csv'
    np.savetxt(mask_name, np.array(final_outlier_mask_ored), delimiter=",")

    plt.figure()
    plt.imshow(np.invert(final_outlier_mask_ored),
               cmap='bwr',
               interpolation='none',
               origin='lower')
    nx_quad, ny_quad = np.array(final_outlier_mask_ored).shape
    final_outliers_ored = np.array(
        np.where(
            np.reshape(final_outlier_mask_ored, (nx_quad * ny_quad,
                                                 1)) == 1)).shape[1]
    plt.title('TEMPO Outlier Mask (Logical OR)\n (Num. outliers = ' +
              str(final_outliers_ored) + ')',
              fontsize=14)
    plt.xlabel('# of spatial pixels', fontsize=12)
    plt.ylabel('# of spectral pixels', fontsize=12)
    plt.grid(False)
    plt.savefig(final_path + '/' + 'final_mask_ored.png',
                dpi=200,
                bbox_inches="tight")
コード例 #4
0
def main():
    """
    Tme main function
    """
    #nx_quad = 1056 # For Tempo
    #ny_quad = 1046 # For Tempo
    #nlat = nx_quad*2
    #nspec = ny_quad*2

    file_path1 = r'F:\TEMPO\Data\GroundTest\FPS\Integration_Sweep\Light\Saved_quads'
    save_dir = r'C:\Users\nmishra\Workspace\TEMPO\Data\GroundTest\FPS\Signal_dependent_offset\Light_Data'
    all_int_files = [each for each in os.listdir(file_path1) \
                         if each.endswith('.dat.sav')]
    if 'Integration_Sweep' in file_path1:
        saturated_collects = [
            'FT6_LONG_INT_130018.dat.sav',  #'FT6_SHORT_INT_0.dat.sav',
            'FT6_LONG_INT_134990.dat.sav',
            'FT6_LONG_INT_139961.dat.sav',
            'FT6_LONG_INT_145028.dat.sav',
            'FT6_LONG_INT_149999.dat.sav',
            'FT6_LONG_INT_154970.dat.sav',
            'FT6_LONG_INT_160037.dat.sav',
            'FT6_LONG_INT_165008.dat.sav',
            'FT6_LONG_INT_169980.dat.sav',
            'FT6_LONG_INT_175047.dat.sav',
            'FT6_LONG_INT_180018.dat.sav',
            'FT6_LONG_INT_184989.dat.sav',
            'FT6_LONG_INT_189960.dat.sav',
            'FT6_LONG_INT_195027.dat.sav',
            'FT6_LONG_INT_199999.dat.sav'
        ]
    elif 'Intensity_Sweep' in file_path1:
        saturated_collects = [
            '162_OP_INT_118000.dat.sav', '164_OP_INT_118000.dat.sav',
            '166_OP_INT_118000.dat.sav', '168_OP_INT_118000.dat.sav',
            '170_OP_INT_118000.dat.sav', '172_OP_INT_118000.dat.sav',
            '174_OP_INT_118000.dat.sav', '176_OP_INT_118000.dat.sav',
            '178_OP_INT_118000.dat.sav', '180_OP_INT_118000.dat.sav',
            '182_OP_INT_118000.dat.sav', '184_OP_INT_118000.dat.sav',
            '186_OP_INT_118000.dat.sav', '188_OP_INT_118000.dat.sav',
            '190_OP_INT_118000.dat.sav', '192_OP_INT_118000.dat.sav',
            '194_OP_INT_118000.dat.sav', '196_OP_INT_118000.dat.sav',
            '198_OP_INT_118000.dat.sav', '200_OP_INT_118000.dat.sav',
            '202_OP_INT_118000.dat.sav'
        ]

    nominal_int_files = [
        items for items in all_int_files
        if not items.endswith(tuple(saturated_collects))
        if items in all_int_files
    ]

    for i in range(0, 4):

        dframe1 = pd.DataFrame()
        dframe2 = pd.DataFrame()
        dframe3 = pd.DataFrame()
        all_int_time = []
        active_quad_even_all = []
        active_quad_odd_all = []
        active_quad_even_all_outlier_filt = []
        active_quad_odd_all_outlier_filt = []
        tsoc_even_all = []
        tsoc_odd_all = []
        tsoc_even_all_outlier_filt = []
        tsoc_odd_all_outlier_filt = []
        unct_spectral_even = []
        unct_spectral_odd = []
        #nominal_int_files = [nominal_int_files[0], nominal_int_files[1]]

        for data_files in nominal_int_files:
            data_path_name_split = data_files.split('_')
            data_file = os.path.join(file_path1, data_files)
            print(data_file)

            IDL_variable = readsav(data_file)
            if 'Intensity_Sweep' in file_path1:
                int_time = data_path_name_split[0]
                string1 = 'VA_'
                string2 = 'VA Setting = '
            else:
                int_time = round(int(data_path_name_split[-1].split('.')[0]))
                string1 = 'Integ_time_'
                string2 = 'Int.time = '
            #print(int_time)
            all_int_time.append(int_time)

            quads = ['Quad A', 'Quad B', 'Quad C', 'Quad D']
            all_full_frame = IDL_variable.q
            quads = ['Quad A', 'Quad B', 'Quad C', 'Quad D']
            all_full_frame = IDL_variable.q
            quad = all_full_frame[:, i, :, :]
            tsoc_all = quad[:, 4:1028, 1034:1056]
            active_quad = np.mean(quad[:, 4:1028, 10:1034], axis=0)
            active_quad, smear = perform_smear_subtraction(
                active_quad, int_time)
            active_quad[active_quad == 16383] = 'nan'
            active_quad_even = active_quad[:, ::2]
            active_quad_even = np.nanmean(active_quad_even, axis=1)
            active_quad_odd = np.nanmean((active_quad[:, 1::2]), axis=1)
            #active_quad_even_outlier_filt = np.mean((active_quad[:, ::2]), axis=1)
            #active_quad_odd_outlier_filt = np.mean((active_quad[:, 1::2]), axis=1)
            active_quad_even_all.append(active_quad_even)
            active_quad_odd_all.append(active_quad_odd)
            # active_quad_even_all_outlier_filt.append(active_quad_even_outlier_filt)
            #active_quad_odd_all_outlier_filt.append(active_quad_odd_outlier_filt)

            quad_dir = quads[i]

            #----------------------------------------------------------------#
            # Ok, let's plot the histogram of saved quads

            all_frames_hist = 'all_frames_hist'
            save_dir_image = os.path.join(save_dir, quad_dir, all_frames_hist)
            if not os.path.exists(save_dir_image):
                os.makedirs(save_dir_image)

            # separate out even and odd lines

            even_samples_all = tsoc_all[:, :, ::2]
            odd_samples_all = tsoc_all[:, :, 1::2]

            even_samples_avg = np.mean(even_samples_all, axis=0)
            odd_samples_avg = np.mean(odd_samples_all, axis=0)

            #            tsoc_even_all.append(np.mean((even_samples_avg)))
            #            tsoc_even_all_outlier_filt.append(np.mean(filter_outlier_median(even_samples_avg)))
            #            tsoc_odd_all.append(np.mean((odd_samples_avg)))
            #            tsoc_odd_all_outlier_filt.append(np.mean(filter_outlier_median(odd_samples_avg)))

            title = 'Histogram of Serial Overclocks (All 100 Frames)\n ' + quads[
                i] + ', ' + string2 + str(int_time)  #+ r" $\mu$" +'secs'
            figure_name = save_dir_image + '/' + string1 + str(
                int_time) + '_image.png'
            plot_hist(even_samples_all, odd_samples_all, title, figure_name)

            avg_frames_hist = 'avg_frames_hist'
            save_dir_image = os.path.join(save_dir, quad_dir, avg_frames_hist)
            if not os.path.exists(save_dir_image):
                os.makedirs(save_dir_image)
            title = 'Histogram of Serial Overclocks (Avg. of 100 Frames)\n ' + quads[
                i] + ', ' + string2 + str(int_time)  #+ r" $\mu$" +'secs'
            figure_name = save_dir_image + '/' + string1 + str(
                int_time) + '_image.png'
            plot_hist(even_samples_avg, odd_samples_avg, title, figure_name)

            final_two_lines = 'final_two_lines'
            save_dir_image = os.path.join(save_dir, quad_dir, final_two_lines)
            if not os.path.exists(save_dir_image):
                os.makedirs(save_dir_image)

            even_samples_used = np.mean(even_samples_avg, axis=1)
            unct_spectral_even.append(
                np.std(even_samples_used) / np.mean(even_samples_used))

            odd_samples_used = np.mean(odd_samples_avg, axis=1)
            unct_spectral_odd.append(
                np.std(odd_samples_used) / np.mean(odd_samples_used))

            title = 'Histogram of Serial Overclocks (Avg  for even and odd lines)\n ' + quads[
                i] + ', ' + string2 + str(int_time)  #+ r" $\mu$" +'secs'
            figure_name = save_dir_image + '/' + string1 + str(
                int_time) + '_image.png'
            #plot_hist(even_samples_used, odd_samples_used,  title, figure_name)

            tsoc_profile = 'tsoc_plot'
            save_tsoc_profile = os.path.join(save_dir, quad_dir, tsoc_profile)
            if not os.path.exists(save_tsoc_profile):
                os.makedirs(save_tsoc_profile)
            figure_name = save_tsoc_profile + '/' + string1 + str(
                int_time) + '_image.png'
            title = 'Profile of Serial Overclocks ' + quads[
                i] + ', ' + string2 + str(int_time)  #+ r" $\mu$" +'secs'
            even_samples_mean, odd_samples_mean = plot_few_tsocs(
                even_samples_avg, odd_samples_avg, figure_name, title)
            # save average in spatial direction
            tsoc_even_all.append(even_samples_mean)
            tsoc_odd_all.append(odd_samples_mean)

            odd_even_lines = [even_samples_avg, odd_samples_avg]
            odd_even_lines_name = ['Even Lines', 'Odd Lines']
            for k in np.arange(0, 2):
                # Lets make directory of
                #k=1
                median_plot = r'Outlier_median_tsoc' + '/' + odd_even_lines_name[
                    k]
                folder_name_hist = 'Hist_plot'
                folder_name_mask = 'Mask_plot'
                folder_name_sat = 'Saturation_plot'
                save_median_hist = os.path.join(save_dir, quad_dir,
                                                median_plot, folder_name_hist)
                if not os.path.exists(save_median_hist):
                    os.makedirs(save_median_hist)
                save_median_mask = os.path.join(save_dir, quad_dir,
                                                median_plot, folder_name_mask)
                if not os.path.exists(save_median_mask):
                    os.makedirs(save_median_mask)
                save_sat_mask = os.path.join(save_dir, quad_dir, median_plot,
                                             folder_name_sat)
                if not os.path.exists(save_sat_mask):
                    os.makedirs(save_sat_mask)

                figure_name = save_sat_mask + '/' + string1 + str(
                    int_time) + '_image.png'
                sat_pixels = identify_saturation(odd_even_lines[k], int_time,
                                                 figure_name)
                outlier_filt_med, outlier_med = reject_outlier_median(
                    odd_even_lines[k])
                if len(outlier_filt_med) == 1:
                    outlier_med = 0
                else:
                    outlier_med = outlier_med[1]

                title = 'Binary Outlier Mask of Trailing Overclocks (' + quads[
                    i] + ', ' + odd_even_lines_name[k] + ', ' + string2 + str(
                        int_time)  #+' micro secs)'

                figure_name = save_median_mask + '/' + string1 + str(
                    int_time) + '_image.png'
                median_mask = create_outlier_mask(odd_even_lines[k],
                                                  outlier_med, title,
                                                  figure_name)

                title = 'Histogram of Trailing Overclocks (' + quads[
                    i] + ', ' + odd_even_lines_name[k] + ', ' + string2 + str(
                        int_time) + ')'
                title1 = 'outliers = ' + str(outlier_med.shape[0])
                figure_name = save_median_hist + '/' + string1 + str(
                    int_time) + '_image.png'
                xlim_low = [800, 825]
                xlim_high = [815, 840]

                #plot_hist_image(odd_even_lines[k], title, title1, outlier_filt_med, outlier_med, figure_name, xlim_low[k], xlim_high[k])


#        print(all_int_time)
#        print(active_quad_even_all_outlier_filt)
#        print(active_quad_odd_all_outlier_filt)
#        print(tsoc_even_all_outlier_filt)
#        print(tsoc_odd_all_outlier_filt)
#
#dframe1 = pd.DataFrame(
#                  {'Avg_Active_Quad_even' : active_quad_even_all,
#                   'Avg_Active_Quad_odd' : active_quad_odd_all,
#                   'Avg_tsoc_Quad_even' : tsoc_even_all,
#                   'Avg_tsoc_Quad_odd': tsoc_odd_all
#                  })
#        dframe2 = pd.DataFrame(
#                  {'Int_time.' : all_int_time,
#                   'Avg_Active_Quad_even' : active_quad_even_all_outlier_filt,
#                   'Avg_Active_Quad_odd' : active_quad_odd_all_outlier_filt,
#                   'Avg_tsoc_Quad_even' : tsoc_even_all_outlier_filt,
#                   'Avg_tsoc_Quad_odd': tsoc_odd_all_outlier_filt
#                    })
#        dframe3 = pd.DataFrame(
#                  {'Int_time.' : all_int_time,
#                   'Unct_tsoc_Quad_even' : unct_spectral_even,
#                   'Unct_tsoc_Quad_odd': unct_spectral_odd
#                    })
        data_to_be_saved = np.concatenate(
            (active_quad_even_all, active_quad_odd_all, tsoc_even_all,
             tsoc_odd_all),
            axis=0)
        csv_save_dir = os.path.join(save_dir, quad_dir)
        if not os.path.exists(csv_save_dir):
            os.makedirs(csv_save_dir)
        csv_file_name1 = csv_save_dir + '/' + quads[
            i] + '_Signal_dependent_offset_more_outliers.csv'
        np.savetxt(csv_file_name1,
                   np.array(data_to_be_saved).T,
                   delimiter=',',
                   fmt='%1.2f')
コード例 #5
0
def main():
    """
    Tme main function
    """
    #nx_quad = 1056 # For Tempo
    #ny_quad = 1046 # For Tempo
    #nlat = nx_quad*2
    #nspec = ny_quad*2
    file_path = r'C:\Users\nmishra\Workspace\TEMPO\Data\GroundTest\FPS\Integration_Sweep\Dark'
    save_dir = r'C:\Users\nmishra\Workspace\TEMPO\outlier_mask_3_sigma'
    data_path_all = [
        each for each in os.listdir(file_path) if each.endswith('.sav')
    ]
    #data_path_all = [items for items in all_int_files if not items.endswith('.dat.sav')]

    #names_to_check = ['SHORT', 'NOM', 'LONG']
    names_to_check = ['SHORT', 'NOM', 'OP', 'LONG']
    median_mask_A = []
    median_mask_B = []
    median_mask_C = []
    median_mask_D = []
    saturation_mask_A = []
    saturation_mask_B = []
    saturation_mask_C = []
    saturation_mask_D = []
    for data_path in data_path_all:
        print(data_path)
        full_frame, collection_type, frames,int_time = \
        make_quads_from_IDL_sav_file(file_path, data_path, names_to_check)
        # all_full_frame contains all 100 frames

        #Let's fetch the quads now
        #quad_A = full_frame[0:ny_quad, 0:nx_quad]
        quad_A = full_frame[:, 0, :, :]
        active_quad_A = np.mean(quad_A[:, 2:1030, 10:1034], axis=0)

        quad_B = full_frame[:, 1, :, :]
        active_quad_B = np.mean(quad_B[:, 2:1030, 10:1034], axis=0)

        quad_C = full_frame[:, 2, :, :]
        active_quad_C = np.mean(quad_C[:, 2:1030, 10:1034], axis=0)

        quad_D = full_frame[:, 3, :, :]
        active_quad_D = np.mean(quad_D[:, 2:1030, 10:1034], axis=0)

        #lower_quads = np.concatenate((active_quad_A, active_quad_B), axis=1)
        #upper_quads = np.concatenate((active_quad_D, active_quad_C), axis=1)
        #active_quads =[lower_quads, upper_quads]
        # for overclocks
        #lower_quads = np.concatenate((bias_A, bias_B), axis=1)
        #upper_quads = np.concatenate((bias_D, bias_C), axis=1)
        active_quads = [
            active_quad_A, active_quad_B, active_quad_C, active_quad_D
        ]
        quads = ['Quad A', 'Quad B', 'Quad C', 'Quad D']
        # Now let's save the full_frame images
        plot_dir = save_dir
        image_dir = 'Quad_images_'
        save_quad_images = os.path.join(plot_dir, image_dir)

        #        if not os.path.exists(save_quad_images):
        #            os.makedirs(save_quad_images)
        #        image_title = 'Full Frame Quad Image'
        #        plot_full_frame_image(full_frame, image_title, collection_type, frames,
        #                              int_time, save_quad_images)
        #
        # Create list of directories To save histograms before and after
        #outlier rejections

        for k in np.arange(0, 4):
            #mean_plot = r'Outlier_mean_tsoc'+'/'+ quads[k]
            median_plot = r'Outlier_median' + '/' + quads[k]
            folder_name_hist = 'Hist_plot'
            folder_name_mask = 'Mask_plot'
            folder_name_sat = 'Saturation_plot'

            save_median_hist = os.path.join(plot_dir, median_plot,
                                            folder_name_hist)
            if not os.path.exists(save_median_hist):
                os.makedirs(save_median_hist)

            save_median_mask = os.path.join(plot_dir, median_plot,
                                            folder_name_mask)
            if not os.path.exists(save_median_mask):
                os.makedirs(save_median_mask)

            save_sat_mask = os.path.join(plot_dir, median_plot,
                                         folder_name_sat)
            if not os.path.exists(save_sat_mask):
                os.makedirs(save_sat_mask)

            # Now let's call the outlier functions. First identify saturated
            #bits and then identify the 'hot and cold' outlier pixels

            title = 'Histogram of ' + quads[k]

            sat_pixels = identify_saturation(active_quads[k], save_sat_mask,
                                             collection_type, frames, int_time)

            #outlier_filt_mean, outlier_mean = reject_outlier_mean(active_quads[k])

            outlier_filt_med, outlier_med = reject_outlier_median(
                active_quads[k])

            # Ok now lets' plot the histograms to understand what we see.
            # First the mean approach
            #print('ok, let us plot the histograms now')

            plot_hist_image(active_quads[k], title, collection_type, frames,
                            outlier_filt_med, outlier_med, int_time,
                            save_median_hist)

            if len(outlier_med) == 1:
                outlier_med = 0
            else:
                outlier_med = outlier_med[1]
            title = 'Binary Outlier Mask of ' + quads[k]
            median_mask = create_outlier_mask(active_quads[k], outlier_med,
                                              title, collection_type, frames,
                                              save_median_mask)
            #print(median_mask.max(), median_mask.min())
            #cc
            active_quads[k] = None
            outlier_med = None
            if k == 0:
                median_mask_A.append(median_mask[:])
                median_mask = None
                saturation_mask_A.append(sat_pixels[:])
                sat_pixels = None
            elif k == 1:
                median_mask_B.append(median_mask[:])
                median_mask = None
                saturation_mask_B.append(sat_pixels[:])
                sat_pixels = None

            elif k == 2:
                median_mask_C.append(median_mask[:])
                median_mask = None
                saturation_mask_C.append(sat_pixels[:])
                sat_pixels = None

            elif k == 3:
                median_mask_D.append(median_mask[:])
                median_mask = None
                saturation_mask_D.append(sat_pixels[:])
                sat_pixels = None

#***************************************************************************
# The following function creates final mask based on 80% occurence criteria

    final_path = r'C:\Users\nmishra\Workspace\TEMPO\outlier_mask\Outlier_median\save_masks_80\to_SAO%'
    if not os.path.exists(final_path):
        os.makedirs(final_path)
    quad_name = 'Quad A'
    title = 'Quad A outlier mask (50% criteria)'
    final_mask_A = create_final_mask(median_mask_A, quad_name, title,
                                     final_path)
    #print(np.array(final_mask_A).max(), np.array( final_mask_A).min())

    mask_name_A = final_path + '/' + 'quad_A_outlier_mask.csv'
    np.savetxt(mask_name_A, np.array(final_mask_A), delimiter=",")
    #cc
    quad_name = 'Quad B'
    title = 'Quad B outlier mask (50% criteria)'
    final_mask_B = create_final_mask(median_mask_B, quad_name, title,
                                     final_path)
    mask_name_B = final_path + '/' + 'quad_B_outlier_mask.csv'
    np.savetxt(mask_name_B, np.array(final_mask_B), delimiter=",")

    quad_name = 'Quad C'
    title = 'Quad C outlier mask (50% criteria)'
    final_mask_C = create_final_mask(median_mask_C, quad_name, title,
                                     final_path)
    mask_name_C = final_path + '/' + 'quad_C_outlier_mask.csv'
    np.savetxt(mask_name_C, np.array(final_mask_C), delimiter=",")

    quad_name = 'Quad D'
    title = 'Quad D outlier mask (50% criteria)'
    final_mask_D = create_final_mask(median_mask_D, quad_name, title,
                                     final_path)
    mask_name_D = final_path + '/' + 'quad_D_outlier_mask.csv'
    np.savetxt(mask_name_D, np.array(final_mask_D), delimiter=",")

    lower_quads = np.concatenate((final_mask_A, final_mask_B), axis=1)
    upper_quads = np.concatenate((final_mask_D, final_mask_C), axis=1)
    final_outlier_mask = np.concatenate((lower_quads, upper_quads), axis=0)

    mask_name = final_path + '/' + 'final_outlier_mask_80%.csv'
    np.savetxt(mask_name, np.array(final_outlier_mask), delimiter=",")

    plt.figure()
    final_outlier_mask = np.invert(final_outlier_mask)
    plt.imshow(final_outlier_mask,
               cmap='bwr',
               interpolation='none',
               origin='lower')
    #cbar = plt.colorbar(image)
    nx_quad, ny_quad = np.array(final_outlier_mask).shape
    final_outliers = np.array(
        np.where(np.reshape(final_outlier_mask, (nx_quad * ny_quad,
                                                 1)) == 0)).shape[1]
    plt.title(
        'TEMPO outlier Mask (50% Occurence Criteria)\n (Num. outliers = ' +
        str(final_outliers) + ')',
        fontsize=14)
    plt.xlabel('# of spatial pixels', fontsize=12)
    plt.ylabel('# of spectral pixels', fontsize=12)
    plt.grid(False)
    plt.savefig(final_path + '/' + 'final_mask_50%.png',
                dpi=200,
                bbox_inches="tight")

    #***************************************************************************

    #***************************************************************************
    # if we want to 'OR" all the mask, use the following function

    final_path = r'C:\Users\nmishra\Workspace\TEMPO\outlier_mask\Outlier_median\save_masks_logical_OR\to_SAO'
    if not os.path.exists(final_path):
        os.makedirs(final_path)
    quad_name = 'Quad A'
    title = 'Quads A outlier mask (Logical OR)'
    final_mask_A = create_ORed_mask(median_mask_A, quad_name, title,
                                    final_path)
    mask_name_A = final_path + '/' + 'quad_A_outlier_mask_ored.csv'
    np.savetxt(mask_name_A, np.array(final_mask_A), delimiter=",")

    quad_name = 'Quad B'
    title = 'Quads B outlier mask (Logical OR)'
    final_mask_B = create_ORed_mask(median_mask_B, quad_name, title,
                                    final_path)
    mask_name_B = final_path + '/' + 'quad_B_outlier_mask_ored.csv'
    np.savetxt(mask_name_B, np.array(final_mask_B), delimiter=",")

    quad_name = 'Quad C'
    title = 'Quads C outlier mask (Logical OR)'
    final_mask_C = create_ORed_mask(median_mask_C, quad_name, title,
                                    final_path)
    mask_name_C = final_path + '/' + 'quad_C_outlier_mask_ored.csv'
    np.savetxt(mask_name_C, np.array(final_mask_C), delimiter=",")

    quad_name = 'Quad D'
    title = 'Quads D outlier mask (Logical OR)'
    final_mask_D = create_ORed_mask(median_mask_D, quad_name, title,
                                    final_path)
    mask_name_D = final_path + '/' + 'quad_D_outlier_mask_ored.csv'
    np.savetxt(mask_name_D, np.array(final_mask_D), delimiter=",")

    quad_name = 'upper_quads_ored'
    title = 'Quads C & D outlier mask (Logical OR)'

    lower_quads = np.concatenate((final_mask_A, final_mask_B), axis=1)
    upper_quads = np.concatenate((final_mask_D, final_mask_C), axis=1)
    final_outlier_mask_ored = np.concatenate((lower_quads, upper_quads),
                                             axis=0)
    mask_name = final_path + '/' + 'final_outlier_mask_ored.csv'
    np.savetxt(mask_name, np.array(final_outlier_mask_ored), delimiter=",")

    plt.figure()
    plt.imshow(np.invert(final_outlier_mask_ored),
               cmap='bwr',
               interpolation='none',
               origin='lower')
    nx_quad, ny_quad = np.array(final_outlier_mask_ored).shape
    final_outliers_ored = np.array(
        np.where(
            np.reshape(final_outlier_mask_ored, (nx_quad * ny_quad,
                                                 1)) == 1)).shape[1]
    plt.title('TEMPO Outlier Mask (Logical OR)\n (Num. outliers = ' +
              str(final_outliers_ored) + ')',
              fontsize=14)
    plt.xlabel('# of spatial pixels', fontsize=12)
    plt.ylabel('# of spectral pixels', fontsize=12)
    plt.grid(False)
    plt.savefig(final_path + '/' + 'final_mask_ored.png',
                dpi=200,
                bbox_inches="tight")
コード例 #6
0
def main():
    """
    Tme main function    """

    file_path = r'C:\Users\nmishra\Workspace\TEMPO\Data\GroundTest\FPS\Dark_Current_Radiation_testing\Pre_rad'
    data_path_all = [
        each for each in os.listdir(file_path) if each.endswith('.fits')
    ]
    collection_type = 'Pred rad Test'

    median_mask_all_quad_A = []
    median_mask_all_quad_B = []
    for data_path in data_path_all:

        full_frame, int_time, temp, frames = make_quads_from_fits_file(
            file_path, data_path)
        # Radiation testing was done only to 1 CCD ( quads A and B)
        #There are 16 frames each for Quads A and B. CLoser inspection suggested
        #that you can average these 16 frames and work on average
        # Let's fetch the quads now
        ndimes, nx_quad, ny_quad = full_frame.shape
        quad_A = np.median(full_frame[0:ndimes:2, :, :], axis=0)

        active_quad_A = np.array(quad_A[4:1028, 10:1034])

        quad_B = np.median(full_frame[1:ndimes:2, :, :], axis=0)
        active_quad_B = np.array(quad_B[4:1028, 10:1034])
        quads_name = ['Quad A', 'Quad B']
        lower_quads = [active_quad_A, active_quad_B]
        lower_quads_all = [quad_A, quad_B]
        plot_dir = file_path

        # Create list of directories To save histograms before and after
        #outlier rejections

        for k in np.arange(0, 2):
            image_dir = r'Quad_images' + '/' + quads_name[k]
            median_plot = r'Outlier_median' + '/' + quads_name[k]
            folder_name_hist = 'Hist_plot'
            folder_name_mask = 'Mask_plot'
            folder_name_sat = 'Saturation_plot'

            save_quad_images = os.path.join(plot_dir, image_dir)
            if not os.path.exists(save_quad_images):
                os.makedirs(save_quad_images)

            save_median_hist = os.path.join(plot_dir, median_plot,
                                            folder_name_hist)
            if not os.path.exists(save_median_hist):
                os.makedirs(save_median_hist)

            save_median_mask = os.path.join(plot_dir, median_plot,
                                            folder_name_mask)
            if not os.path.exists(save_median_mask):
                os.makedirs(save_median_mask)

            save_sat_mask = os.path.join(plot_dir, median_plot,
                                         folder_name_sat)
            if not os.path.exists(save_sat_mask):
                os.makedirs(save_sat_mask)

            # Lets plt the frame first
            if k == 0:
                image_title = 'Full Frame Quad A Image'
            else:
                image_title = 'Full Frame Quad B Image'

            int_time = str(int_time)

            title = image_title + ' ('+ collection_type+')' + '\n Int. time = ' + int_time + \
                                    ', Temp = ' +temp

            plt.figure()
            image = plt.imshow(lower_quads_all[k],
                               cmap='nipy_spectral',
                               interpolation='none',
                               origin='lower')
            cbar = plt.colorbar(image)
            plt.title(title, fontsize=14)
            plt.xlabel('# of spatial pixels', fontsize=12)
            plt.ylabel('# of spectral pixels', fontsize=12)
            plt.grid(False)

            plt.savefig(save_quad_images+'/'+ quads_name[k]+'_'+\
                    frames+'.png',dpi=100,bbox_inches="tight")
            plt.close('all')

            # Now let's call the outlier functions. First identify saturated
            #bits and then identify the 'hot and cold' outlier pixels

            if k == 0:
                title = 'Histogram of Quad A'
            else:
                title = 'Histogram of Quad B'

#            sat_pixels = identify_saturation(lower_quads[k],
#                                                         save_sat_mask,
#                                                         collection_type,
#                                                         frames, int_time)

            outlier_filt_med, outlier_med = reject_outlier_median(
                lower_quads[k])
            # Ok now lets' plot the histograms to understand what we see.
            # First the mean approach
            print('ok, let us plot the histograms now')
            # Secondly the median absolute difference approach
            plot_hist_image(lower_quads[k], title, collection_type, frames,
                            outlier_filt_med, outlier_med, int_time,
                            save_median_hist)

            if len(outlier_med) == 1:
                outlier_med = 0
            else:
                outlier_med = outlier_med[1]

            if k == 0:
                title = 'Binary Outlier Mask of Quad A'
            else:
                title = 'Binary Outlier Mask of Quad B'

            median_mask = create_outlier_mask(lower_quads[k], outlier_med,
                                              title, collection_type, frames,
                                              save_median_mask)
            lower_quads[k] = None
            outlier_med = None

            if k == 0:
                median_mask_all_quad_A.append(median_mask[:])
                median_mask = None
                #saturation_mask_lower.append(sat_pixels[:])
                #sat_pixels = None
            elif k == 1:
                median_mask_all_quad_B.append(median_mask[:])
                median_mask = None
                #saturation_mask_upper.append(sat_pixels[:])
                #sat_pixels = None

#***************************************************************************
# The following function creates final mask based on 80% occurence criteria

    final_path = file_path + '\Outlier_median\save_masks'
    if not os.path.exists(final_path):
        os.makedirs(final_path)
    quad_name = 'QuadA_80%'
    title = 'Quad A outlier mask (80% occurence criteria)'
    final_mask_quad_A = create_final_mask(median_mask_all_quad_A, quad_name,
                                          title, final_path)
    quad_name = 'QuadB_80%'
    title = 'Quad B outlier mask (80% ocurence criteria)'
    final_mask_quad_B = create_final_mask(median_mask_all_quad_B, quad_name,
                                          title, final_path)
    final_outlier_mask = np.concatenate((final_mask_quad_A, final_mask_quad_B),
                                        axis=1)
    plt.figure()
    plt.imshow(final_outlier_mask,
               cmap='bwr',
               interpolation='none',
               origin='lower')

    nx_quad, ny_quad = np.array(final_outlier_mask).shape
    final_outliers = np.array(
        np.where(np.reshape(final_outlier_mask, (nx_quad * ny_quad,
                                                 1)) == 0)).shape[1]
    plt.title('Quad A and B Outlier Mask' + ' (outliers = ' +
              str(final_outliers) + ')',
              fontsize=14)
    plt.xlabel('# of spatial pixels', fontsize=12)
    plt.ylabel('# of spectral pixels', fontsize=12)
    plt.grid(False)

    plt.savefig(final_path + '/' + 'final_mask.png',
                dpi=100,
                bbox_inches="tight")

    #***************************************************************************

    #***************************************************************************
    # if we want to 'OR" all the mask, use the following function

    quad_name = 'QuadA_ored'
    title = 'Quad A outlier mask (Logical OR)'
    final_mask_lower_ored = create_ORed_mask(median_mask_all_quad_A, quad_name,
                                             title, final_path)
    quad_name = 'QuadB_ored'
    title = 'Quad B outlier mask (Logical OR)'
    final_mask_upper_ored = create_ORed_mask(median_mask_all_quad_B, quad_name,
                                             title, final_path)
    final_outlier_mask_ored = np.concatenate(
        (final_mask_lower_ored, final_mask_upper_ored), axis=1)
    plt.figure()
    plt.imshow(np.invert(final_outlier_mask_ored),
               cmap='bwr',
               interpolation='none',
               origin='lower')

    nx_quad, ny_quad = np.array(final_outlier_mask_ored).shape
    final_outliers_ored = np.array(
        np.where(
            np.reshape(final_outlier_mask_ored, (nx_quad * ny_quad,
                                                 1)) == 1)).shape[1]
    plt.title('Quad A & B Outlier Mask' + ' (outliers = ' +
              str(final_outliers_ored) + ')',
              fontsize=14)
    plt.xlabel('# of spatial pixels', fontsize=12)
    plt.ylabel('# of spectral pixels', fontsize=12)
    plt.grid(False)

    plt.savefig(final_path + '/' + 'final_mask_ored.png',
                dpi=100,
                bbox_inches="tight")
コード例 #7
0
def main():
    """
    Tme main function
    """
    nx_quad = 1056  # For Tempo
    ny_quad = 1046  # For Tempo
    nlat = nx_quad * 2
    nspec = ny_quad * 2
    file_path = r'C:\Users\nmishra\Workspace\TEMPO\Data\GroundTest\FPS\Light'
    data_path = [
        each for each in os.listdir(file_path) if each.endswith('.dat')
    ]
    names_to_check = ['SHORT', 'NOM', 'LONG']
    for data_path in data_path:

        #data_path = 'SPI_20160912142518420_LIGHT_INTEGRATION_SWEEP_207_FT6_NOM_INT_99999.dat'
        # Now let's find the integtration type

        data_path_name_split = data_path.split('_')
        #print(data_path_name_split)
        integration_type = [
            x for x in names_to_check if x in data_path_name_split
        ]
        frames = data_path_name_split[-1]
        data_file = os.path.join(file_path, data_path)
        fileinfo = get_size(data_file)
        size_file = fileinfo.st_size
        #-------------Some printing to do Sanity checks----------------------------
        #print ("size of the file is ", size_file, "bytes")
        nframes = size_file / (nlat * nspec * 2)
        #print("num of frames : ", nframes)
        #dt = np.dtype(')
        result = np.fromfile(data_file, dtype='>u2', count=-1) & int(
            '3fff', 16)
        # Refer to 2450865 Ball document
        #print(result.size)
        #*Ok, let's do few steps before we move into data analysis

        # 1. Create full frame from raw FPE.
        # 2. Arrange full frame into various quads.
        full_frame = make_full_frame_from_raw_fpe(result, nx_quad, ny_quad)
        quads = full_frame_to_quads(full_frame, nx_quad, ny_quad)

        # Now let's save the full_frame images
        plot_dir = r'C:\Users\nmishra\Desktop\test'
        image_dir = 'Quad_images'
        save_quad_images = os.path.join(plot_dir, image_dir)
        if not os.path.exists(save_quad_images): os.makedirs(save_quad_images)

        aligned_quads = 'aligned_quads'
        save_aligned_quads = os.path.join(plot_dir, aligned_quads)
        if not os.path.exists(save_aligned_quads):
            os.makedirs(save_aligned_quads)

        collection_type = integration_type[0]
        image_title = 'Full Frame Quad Image'
        plot_full_frame_image(full_frame, image_title, collection_type, frames,
                              save_quad_images)
        plot_each_quad(quads, image_title, collection_type, frames,
                       save_aligned_quads)

        quads = full_frame_to_quads(full_frame, nx_quad, ny_quad)
        active_pixel_quads = find_active_region_each_quads(quads)
        title = 'Histogram of active pixels'
        # plot_hist_each_quad(active_pixel_quads,title, collection_type,frames)

        # Creste list of directories To save histograms before and after
        #outlier rejections
        mean_plot = 'Outlier_mean'
        median_plot = 'Outlier_median'
        folder_name_hist = 'Hist_plot'
        folder_name_mask = 'Mask_plot'

        save_mean_hist = os.path.join(plot_dir, mean_plot, folder_name_hist)
        if not os.path.exists(save_mean_hist): os.makedirs(save_mean_hist)

        save_median_hist = os.path.join(plot_dir, median_plot,
                                        folder_name_hist)
        if not os.path.exists(save_median_hist): os.makedirs(save_median_hist)

        save_mean_mask = os.path.join(plot_dir, mean_plot, folder_name_mask)
        if not os.path.exists(save_mean_mask): os.makedirs(save_mean_mask)

        save_median_mask = os.path.join(plot_dir, median_plot,
                                        folder_name_mask)
        if not os.path.exists(save_median_mask): os.makedirs(save_median_mask)

        # Now let's call the outlier functions
        outlier_filt_mean, outlier_mean = reject_outlier_mean(
            active_pixel_quads)
        outlier_filt_med, outlier_med = reject_outlier_median(
            active_pixel_quads)

        # Ok now lets' plot the histograms to understand what we see.
        # First the mean approach
        plot_hist_image(active_pixel_quads, title, collection_type, frames,
                        outlier_filt_mean, outlier_mean, save_mean_hist)
        # Secondly the median absolute difference approach
        plot_hist_image(active_pixel_quads, title, collection_type, frames,
                        outlier_filt_med, outlier_med, save_median_hist)

        # Ok, lets now create a binary mask of outlier pixels
        # First with the mean based approach
        create_outlier_mask(active_pixel_quads, outlier_med, collection_type,
                            frames, save_mean_mask)
        # Secondly, the same for median based approach
        create_outlier_mask(active_pixel_quads, outlier_med, collection_type,
                            frames, save_median_mask)