def main():
    """
    This is the main function that does all the processing. It calls the required
    functions to process the raw image. Raw images are saved in .sav (IDL)
    format and read through this script.
    """
    data_dir = r'F:\TEMPO\Data\GroundTest\FPS\Spectrometer\Radiance_Cal_VIS_Lamp'
    telemetry_dir = r'F:\TEMPO\Data\GroundTest\FPS\Spectrometer\2018.06.27'
    image_dir = os.path.join(data_dir, r'saved_quads')
    image_save_dir = os.path.join(image_dir, 'processed_corrected_linearity')
    telemetry_dir = os.path.join(telemetry_dir, 'processed/h5')
    linearity = 1 # option to turn on/off linearity correction
    smear = 1
    cross_talk = 1
    temp_correction = 1
#    all_image = []
#    var_all = []
#    signal_all = []
    #dark_current = 0
    if not os.path.exists(image_save_dir):
        os.makedirs(image_save_dir)
    #print(image_save_dir)
#    data_path_all = sorted([each for each in os.listdir(image_dir)
#                            if each.startswith('2017_07_30_00_24_59_38064')
#                            and  each.endswith('.sav')])
#
    data_path_all = sorted([each for each in os.listdir(image_dir)
                            if each.endswith('.sav')])

#    #dark_data = data_path_all[1:len(data_path_all):2]


    print('Total data = ', len(data_path_all))
#    resultFile = open (os.path.join(save_dir,'file_name_all.csv'),'w')
#
#    for results in data_path_all[0:15]:
#         resultFile.write(results+"\n")
#    resultFile.close()
    count = 0
    #peak_loc_all = []
    for data_path in data_path_all[19:]:
        data_file = os.path.join(image_dir, data_path)
        print(data_path)
        
#        cc
        telemetry_file_name = data_path.split('.')[0]
       #################### Telemetry Statistics###############################
        # Now lets parse the telemetry file to get the required information###
        #Note; The function can return the CCD temp and FPE temp. But these are
        #not needed on the image processing algorithm.. atleast for the time being

        telemetry_file = os.path.join(telemetry_dir, telemetry_file_name+'.h5')
        if os.path.exists(telemetry_file):
            coadds, int_time, fpe_temp, fpa_temp = parse_telemetry_file(telemetry_dir,
                                                                        telemetry_file_name)
           # print('FPE Temp. = ', round(fpe_temp, 2), 'FPA Temp = ',
                 # round(fpa_temp, 2))
            print('Integ. Time =' , int_time)
            print(coadds)
            
        else:
            print('Telemetry file missing')
            coadds = 100
            int_time = 6000
            fpe_temp = 43
            fpa_temp = -21.5
        
       ##############Image Processing begins here##############################
       
        #print(coadds)
    
        full_frame = read_idl_file(data_file)
        
        full_frame = perform_coaddition_correction(full_frame, coadds)
        
        #full_frame = full_frame/coadds
        #print(np.max(full_frame))  
        #cc 
        #check_image = create_image_active_region(full_frame)
        #raw_image = create_final_image(np.array(full_frame))
       # print('Max. Val. Raw = ', np.max(raw_image))
        quads = ['Quad A', 'Quad B', 'Quad C', 'Quad D']

        ##########################OFFSET REMOVAL###############################
        # Input : Full_Frame TEMPO IMAGE
        # Otput : Bias Subtracted Active Region. The active region dimensions are
        #now 1028*1024. 2 pixel lines from SMEAR overclocks are now used for
        #to store storage summation information
        bias_removed_quads = perform_bias_subtraction(full_frame) 
        
        #print('0', bias_removed_quads)
        

        #print('Max. Val. Offset Removed  = ', np.max(bias_removed_quads))

        text1 = telemetry_file_name+'.img' +' (Raw Data)\n Int. time:' + str(round(int_time, 3))+ \
                           'ms, Co-adds:' +str(int(coadds))+\
                           ', FPE temp:'+ str(round(fpe_temp, 1))+'C, ' + \
                           ' FPA temp: ' + str(round(fpa_temp, 1))+'C'

        # Now let us save the raw image
        raw_image_save = os.path.join(image_save_dir, 'raw_image')
        if not os.path.exists(raw_image_save):
            os.makedirs(raw_image_save)
        plot_save_dir = raw_image_save + '/'+ data_path+'.png'
        #create_image(raw_image/coadds, text1, plot_save_dir)
        #prnu_spectrometer[prnu_spectrometer < 0.9] = 0.9
        #prnu_spectrometer[prnu_spectrometer > 1.2] = 1.2
        #print(np.min(check_image))
        #create_image(check_image, text1, plot_save_dir)
        #---------------------------------------------------------------------
        #########################NON LINEARITY CORRECTION#######################
        if temp_correction:
            temp_corrected_quads =   perform_temp_correction(bias_removed_quads, fpe_temp)
        else:
            temp_corrected_quads = bias_removed_quads
        

        
       #########################NON LINEARITY CORRECTION#######################
        # Input : Bias Subtracted Active regions
        # Output : Linearized Active Region Quads
        # pass the difference instead of coadds
        if linearity:
            linearized_a = apply_linearity_correction(temp_corrected_quads [0, :, :],
                                                      quads[0])
            linearized_b = apply_linearity_correction(temp_corrected_quads [1, :, :],
                                                      quads[1])
            linearized_c = apply_linearity_correction(temp_corrected_quads [2, :, :],
                                                      quads[2])
            linearized_d = apply_linearity_correction(temp_corrected_quads[3, :, :],
                                                      quads[3])
            linearized_quads = np.array([linearized_a, linearized_b,
                                         linearized_c, linearized_d])
            #print(np.mean(linearized_quads))
        else:
            linearized_quads = temp_corrected_quads 
        #print('2', linearized_quads)
            #----------------------------------------------------------------------
            ##########################SMEAR REMOVAL################################
            # Input : linearized quads ( all quads together)
            # Output : SMEAR offset corrected Quad

        #### lets' create the masked array with outlier mask################
        # The outlier mask is array of list of 4 quads.
        #print('Max. Val. Linearized  = ', np.max(linearized_quads))
        #outlier_mask = read_outlier_mask()

        # Note : all the arrays after this step are masked arrays


        if cross_talk:
            cross_talk_removed_quads = remove_cross_talk(linearized_quads)
        else:
            cross_talk_removed_quads = linearized_quads
            
        if smear:
            smear_removed_quads = perform_smear_removal(np.array(cross_talk_removed_quads),
                                                        int_time)
        else:
            smear_removed_quads = cross_talk_removed_quads
        #print('3',smear_removed_quads)
            #----------------------------------------------------------------------

            ##########################CROSS-TALK REMOVAL###########################
            # Input : smear removed quads (all quads together)
            # Output : cross talk removed quads
        #print('Max. Val. Smeared  = ', np.max(smear_removed_quads))
        
        
        #print('4',cross_talk_removed_quads)
            #----------------------------------------------------------------------
        #print('Max. Val. Cross Talked = ', np.max(cross_talk_removed_quads))
        
        #gain_applied_quads = apply_electronics_gain(np.array(cross_talk_removed_quads), diff_tsoc_odd_even)
        processed_image = create_final_image(np.array(smear_removed_quads))
        #processed_image = processed_image - dark_current

        #prnu_map = parse_prnu_file()
        #prnu_map = read_prnu_files()
        #prnu_spectrometer = create_final_image(np.array(prnu_map))
        #prnu_spectrometer[prnu_spectrometer > 1.03] = 1.02
        #prnu_spectrometer[prnu_spectrometer < 0.97] = 0.98
        #outlier_spectrometer = create_final_image(np.array(outlier_mask))
        nx_quad, ny_quad = processed_image.shape
        #outlier_spectrometer = np.reshape(outlier_spectrometer,
        #                                  (nx_quad*ny_quad, 1))
        #outlier_detectors = np.array(np.where([outlier_spectrometer == 1]))
        #print('outliers =', outlier_detectors.shape[1])
        #outlier_spectrometer = np.reshape(outlier_spectrometer, (nx_quad, ny_quad))
        #create_image(outlier_spectrometer,'outliers = '+str(outlier_detectors.shape[1]),'b')
        #processed_image = processed_image/(prnu_spectrometer)
        #processed_image[processed_image >= 0.90*16383] = np.NAN
        #processed_image = processed_image*180/int_time
        #print(np.nanmax(processed_image))
        #processed_image[processed_image>=0.85*16383] = 16383
       
#        text1 = telemetry_file_name+'.img, ' + 'Int. time:' + str(round(int_time,2))+ \
#                           'ms\n Co-adds:' +str(int(coadds))+\
#                           ', FPE temp:'+ str(round(fpe_temp, 1))+'C, ' + \
#                           ' FPA temp: ' + str(round(fpa_temp, ))+'C'
#        processed_image_save = os.path.join(image_save_dir, 'processed_image')
#        if not os.path.exists(processed_image_save):
#            os.makedirs(processed_image_save)
#        plot_save_dir = processed_image_save + '/' + data_path+'.png'
        #processed_image = processed_image[:, 500:2000]
        #processed_image[processed_image>6000] = 6000
        #processed_image = uniform_filter(processed_image, size=(15, 15), mode='mirror')
        #create_image(processed_image, text1, plot_save_dir)        
        #processed_image = np.array(processed_image)/16.7
#        processed_image[processed_image>=0.90*16383] = 16383
        #hf_name = data_path[:-8]
#        file_name =  data_path[:-8]+'.h5'
#        hf_file_name = os.path.join(image_save_dir, file_name)
        #Write into h5file    
#        hf = h5py.File(hf_file_name,'w')
#        hf.create_dataset('Processed_data', data=np.array(processed_image))
        #cc
        processed_file_name = image_save_dir +'/'+ data_path+'.csv'
        np.savetxt(processed_file_name, processed_image,fmt='%1.3f',  delimiter=",")  
def main():
    """
    This is the main function that does all the processing. It calls the required
    functions to process the raw image. Raw images are saved in .sav (IDL)
    format and read through this script.
    """
    data_dir = r'C:\Users\nmishra\Workspace\TEMPO_Spectrometer\Spectral_Band_pass\488.2_nm'
    image_dir = os.path.join(data_dir, 'saved_quads')
    save_dir = os.path.join(image_dir, 'processed_image')
    image_save_dir = os.path.join(image_dir, 'saved_plots')
    telemetry_dir = os.path.join(data_dir, 'processed/h5')
    linearity = 0  # option to turn on/off linearity correction
    smear = 1
    cross_talk = 1
    bandpass_val = []
    bandpass_val_norm = []
    normalization_factor_all = []
    if not os.path.exists(save_dir):
        os.makedirs(save_dir)
#    data_path_all = sorted([each for each in os.listdir(image_dir)
#                            if each.startswith('2017_07_30_00_24_59_38064')
#                            and  each.endswith('.sav')])
#
    data_path_all = sorted(
        [each for each in os.listdir(image_dir) if each.endswith('.sav')])
    #
    #    #dark_data = data_path_all[1:len(data_path_all):2]

    print('Total data = ', len(data_path_all))
    resultFile = open(os.path.join(save_dir, 'file_name_all.csv'), 'w')

    for results in data_path_all:
        resultFile.write(results + "\n")
    resultFile.close()
    dark_current = calculate_dark_current(image_dir, coadds=15)

    for data_path in data_path_all:
        data_file = os.path.join(image_dir, data_path)
        print(data_path)
        #        cc
        telemetry_file_name = data_path.split('.')[0]
        #################### Telemetry Statistics###############################
        # Now lets parse the telemetry file to get the required information###
        #Note; The function can return the CCD temp and FPE temp. But these are
        #not needed on the image processing algorithm.. atleast for the time being

        telemetry_file = os.path.join(telemetry_dir,
                                      telemetry_file_name + '.h5')
        if os.path.exists(telemetry_file):
            coadds, int_time, fpe_temp, fpa_temp = parse_telemetry_file(
                telemetry_dir, telemetry_file_name)
            print('FPE Temp. = ', round(fpe_temp, 2), 'FPA Temp = ',
                  round(fpa_temp, 2))
        else:

            print('Telemetry file missing')
            coadds = 15
            int_time = 90
            fpe_temp = 42.98
            fpa_temp = -21

    ##############Image Processing begins here##############################
        full_frame = read_idl_file(data_file)
        check_image = create_image_active_region(full_frame)
        raw_image = create_final_image(np.array(full_frame))
        # print('Max. Val. Raw = ', np.max(raw_image))
        quads = ['Quad A', 'Quad B', 'Quad C', 'Quad D']

        ##########################OFFSET REMOVAL###############################
        # Input : Full_Frame TEMPO IMAGE
        # Otput : Bias Subtracted Active Region. The active region dimensions are
        #now 1028*1024. 2 pixel lines from SMEAR overclocks are now used for
        #to store storage summation information
        # For dark data, only offset removal is needed to compute the dark current
        # For light data, additional processing such as linearity, smear and
        #cross talk is needed

        bias_removed_quads = perform_bias_subtraction(full_frame)

        #print('Max. Val. Offset Removed  = ', np.max(bias_removed_quads))

        text1 = telemetry_file_name+'.img' +' (Raw Data)\n Int. time:' + str(round(int_time, 1))+ \
                           'ms, Co-adds:' +str(int(coadds))+\
                           ', FPE temp:'+ str(round(fpe_temp, 1))+'C, ' + \
                           ' FPA temp: ' + str(round(fpa_temp, 1))+'C'
        # Now let us save the raw image
        raw_image_save = os.path.join(image_save_dir, 'raw_image')
        if not os.path.exists(raw_image_save):
            os.makedirs(raw_image_save)
        plot_save_dir = raw_image_save + '/' + data_path + '.png'
        #prnu_spectrometer[prnu_spectrometer < 0.9] = 0.9
        #prnu_spectrometer[prnu_spectrometer > 1.2] = 1.2
        #create_image(check_image/coadds, text1, plot_save_dir)
        #---------------------------------------------------------------------

        #########################NON LINEARITY CORRECTION#######################
        # Input : Bias Subtracted Active regions
        # Output : Linearized Active Region Quads
        if linearity:
            linearized_a = apply_linearity_correction(
                bias_removed_quads[0, :, :], quads[0], coadds)
            linearized_b = apply_linearity_correction(
                bias_removed_quads[1, :, :], quads[1], coadds)
            linearized_c = apply_linearity_correction(
                bias_removed_quads[2, :, :], quads[2], coadds)
            linearized_d = apply_linearity_correction(
                bias_removed_quads[3, :, :], quads[3], coadds)
            linearized_quads = np.array(
                [linearized_a, linearized_b, linearized_c, linearized_d])
            #print(np.mean(linearized_quads))
        else:
            linearized_quads = bias_removed_quads
            #----------------------------------------------------------------------
            ##########################SMEAR REMOVAL################################
            # Input : linearized quads ( all quads together)
            # Output : SMEAR offset corrected Quad

        #### lets' create the masked array with outlier mask################
        # The outlier mask is array of list of 4 quads.
        #print('Max. Val. Linearized  = ', np.max(linearized_quads))
        outlier_mask = read_outlier_mask()

        # Note : all the arrays after this step are masked arrays

        if smear:
            smear_removed_quads = perform_smear_removal(
                linearized_quads, int_time, outlier_mask)
        else:
            smear_removed_quads = linearized_quads
            #----------------------------------------------------------------------

            ##########################CROSS-TALK REMOVAL###########################
            # Input : smear removed quads (all quads together)
            # Output : cross talk removed quads
        #print('Max. Val. Smeared  = ', np.max(smear_removed_quads))

        if cross_talk:
            cross_talk_removed_quads = remove_cross_talk(
                np.array(smear_removed_quads))
        else:
            cross_talk_removed_quads = smear_removed_quads
            #----------------------------------------------------------------------
        #print('Max. Val. Cross Talked = ', np.max(cross_talk_removed_quads))
        processed_image = create_final_image(
            np.array(cross_talk_removed_quads))
        processed_image = processed_image - (dark_current / coadds)
        prnu_map = parse_prnu_file()  # BATC Map
        prnu_spectrometer = prnu_map

        #Uncomment the lines below if my PRNU is to be used.
        #prnu_map = read_prnu_files() # LaRC map
        #prnu_spectrometer = create_final_image(np.array(prnu_map))
        #prnu_spectrometer[prnu_spectrometer > 1.03] = 1.02
        #prnu_spectrometer[prnu_spectrometer < 0.97] = 0.98
        #create_image(prnu_map, 'TEMPO PRNU Map', 'a')
        #create_image(prnu_spectrometer, 'TEMPO PRNU Map', 'a')
        outlier_spectrometer = create_final_image(np.array(outlier_mask))

        #create_image(outlier_spectrometer,'Outliers = 504', 'b')
        #cc
        nx_quad, ny_quad = processed_image.shape
        outlier_spectrometer = np.reshape(outlier_spectrometer,
                                          (nx_quad * ny_quad, 1))
        outlier_detectors = np.array(np.where([outlier_spectrometer == 1]))
        #print('outliers =', outlier_detectors.shape[1])
        outlier_spectrometer = np.reshape(outlier_spectrometer,
                                          (nx_quad, ny_quad))
        #create_image(outlier_spectrometer,'outliers = '+str(outlier_detectors.shape[1]),'b')
        #processed_image = processed_image/(prnu_spectrometer)
        processed_image = processed_image / (coadds * prnu_spectrometer)

        processed_image[processed_image >= 0.85 * 16383] = 16383
        text1 = telemetry_file_name+'.img' +' (Spectral Bandpass 675 nm)\n Int. time:' + str(round(int_time, 1))+ \
                           'ms, Co-adds:' +str(int(coadds))+\
                           ', FPE temp:'+ str(round(fpe_temp, 1))+'C, ' + \
                           ' FPA temp: ' + str(round(fpa_temp, ))+'C'
        processed_image_save = os.path.join(image_save_dir, 'processed_image')
        if not os.path.exists(processed_image_save):
            os.makedirs(processed_image_save)
        plot_save_dir = processed_image_save + '/' + data_path + '.png'

        #print('min_val=', np.min(processed_image))
        #processed_image = np.reshape(processed_image, (nx_quad*ny_quad, 1))
        #processed_image[outlier_spectrometer] = 16383
        #        create_image(processed_image, text1, plot_save_dir)
        #        #col_index = np.sort([randint(100, 2000) for p in range(0, 10)])
        #        row_mean = np.mean(processed_image, axis=1)
        #        max_row_each = np.where(row_mean == max(np.mean(processed_image, axis=1)))[0]
        #        max_row_each = max_row_each[0]
        #        print(max_row_each)
        #        cc

        #max_row = 29 # for 297.8 nm
        #max_row = 1546 #for 640nm
        #max_row = 1369 #for 605nm
        #max_row = 1050 #for 605nm
        #max_row = 142 # for 320 nm
        # max_row = 319  # for 355 nm
        # max_row = 496  # for 390 nm
        # max_row = 673  # for 425 nm
        #max_row = 849  # for 460 nm
        max_row = 992  # for 488.2 nm
        #max_row = 192 # for 330 nm
        #max_row = 91 # for 310 nm
        # max_row = 1724 # 675 nm
        #max_row = 2032 # for 736 nm
        #check_val = 1392

        subset_image = processed_image[max_row - 11:max_row + 11, :]
        #subset_image_normalized = processed_image[max_row_each-14 : max_row_each+14]
        #subset_image = subset_image[subset_image]
        normalization_factor = np.sum(subset_image[subset_image < 16383])
        normalization_factor_all.append(normalization_factor)
        #normalization_factor_subset = np.sum(subset_image_normalized[subset_image_normalized<16383])
        #normalization_factor_all.append(normalization_factor_subset)
        #bandpass_val.append(processed_image[max_row, :])
        bandpass_val_norm.append(
            np.true_divide(processed_image[max_row, :], normalization_factor))
        #print(processed_image.shape)
    # cc


#        plt.figure(2)
#        plt.plot(processed_image[max_row-4, :], 'purple', label='4 lines down')
#        plt.plot(processed_image[max_row-3, :], 'y-', label='3 lines down')
#        plt.plot(processed_image[max_row-2, :], 'g-', label='2 lines down')
#        plt.plot(processed_image[max_row-1, :], 'r-', label='1 line down')
#        plt.plot(processed_image[max_row, :], 'k-', label='max (297 nm), ' + 'row:' +str(max_row))
#        plt.plot(processed_image[max_row+1, :], 'm-', label='1 line up')
#        plt.plot(processed_image[max_row+2, :], 'b-', label='2 lines up')
#        plt.plot(processed_image[max_row+3, :], 'orange', label='3 lines up')
#        plt.plot(processed_image[max_row+4, :], 'cyan', label='4 lines up')
#        plt.title('Spatial Profile along illuminated rows')
#        plt.ylabel('Counts (DN)')
#        plt.xlabel('Spatial Pixel Index')
#        #plt.yscale('log')
#        #plt.xlim(0, 2000)
#        plt.show()
#        cc
#        plt.legend(loc='best')
#        #plt.ylim(2000, 40000)
#        plt.show(block=False)
#        #cc
#        plt.figure(3)
#        for i in col_index:
#
#            plt.plot(processed_image[:, i],
#                     'o--', label='Col. ' + str(i))
#            plt.legend(loc='best', ncol=4)
#        plt.grid(True, linestyle=':')
#        plt.title('Profile along Spectral direction (Spectral Bandpass 297 nm)')
#        plt.xlabel('Spectral Pixel Index')
#        plt.ylabel('Counts (DN)')
#        #plt.xlim(655, 675)
#        #plt.yscale('log')
#
#        plt.show()
#        cc

# cc

#            prnu_map = parse_prnu_file()
#            final_image = processed_image/prnu_map

######################Save the processed image variable################

# Note : The final images is saved as python variable and read when needed
# https://stackoverflow.com/questions/6568007/how-do-i-save-and-restore-multiple-variables-in-python
#
#        file_name = data_path.split('.')[0]
#        variable_name = save_dir +'/'+ file_name
#        with open(variable_name, 'wb') as data:
#            pickle.dump(processed_image, data)
#            #cc

#create_image(np.array(processed_image))
#cc
    save_bandpass = save_dir + '/' + 'bandpass_radiance.csv'
    save_bandpass_norm = save_dir + '/' + 'bandpass_radiance_normalized.csv'
    norm_factor = save_dir + '/' + 'integrated_radiance.csv'

    np.savetxt(save_bandpass, np.array(bandpass_val), delimiter=",")
    np.savetxt(save_bandpass_norm, np.array(bandpass_val_norm), delimiter=",")
    np.savetxt(norm_factor, np.array(normalization_factor_all), delimiter=",")
    cc
def main():
    """
    This is the main function that does all the processing. It calls the required
    functions to process the raw image. Raw images are saved in .sav (IDL)
    format and read through this script.
    """
    data_dir = r'F:\TEMPO\Data\GroundTest\FPS\Spectrometer\Spectral_Smile\Flight_Slight'
    image_dir = os.path.join(data_dir, 'saved_quads')
    save_dir = os.path.join(image_dir, 'processed_image')
    image_save_dir = os.path.join(image_dir, 'saved_plots')
    telemetry_dir = os.path.join(data_dir, 'processed/h5')
    linearity = 0  # option to turn on/off linearity correction
    smear = 1
    cross_talk = 1
    bandpass_val = []
    bandpass_val_norm = []
    normalization_factor_all = []
    dark_current = 0
    if not os.path.exists(save_dir):
        os.makedirs(save_dir)
#    data_path_all = sorted([each for each in os.listdir(image_dir)
#                            if each.startswith('2017_07_30_00_24_59_38064')
#                            and  each.endswith('.sav')])
#
    data_path_all = sorted(
        [each for each in os.listdir(image_dir) if each.endswith('.sav')])
    #
    #    #dark_data = data_path_all[1:len(data_path_all):2]

    print('Total data = ', len(data_path_all))
    #    resultFile = open (os.path.join(save_dir,'file_name_all.csv'),'w')
    #
    #    for results in data_path_all[0:15]:
    #         resultFile.write(results+"\n")
    #    resultFile.close()
    if dark_current:
        dark_current = calculate_dark_current(image_dir, coadds=1)
    else:
        dark_current = 0

    count = 0
    spectral_alingn_unct = []
    for data_path in data_path_all[0:16]:

        data_file = os.path.join(image_dir, data_path)
        print(data_path)
        #        cc
        telemetry_file_name = data_path.split('.')[0]
        #################### Telemetry Statistics###############################
        # Now lets parse the telemetry file to get the required information###
        #Note; The function can return the CCD temp and FPE temp. But these are
        #not needed on the image processing algorithm.. atleast for the time being

        telemetry_file = os.path.join(telemetry_dir,
                                      telemetry_file_name + '.h5')

        if os.path.exists(telemetry_file):
            coadds, int_time, fpe_temp, fpa_temp = parse_telemetry_file(
                telemetry_dir, telemetry_file_name)
            print('FPE Temp. = ', round(fpe_temp, 2), 'FPA Temp = ',
                  round(fpa_temp, 2))
        else:

            print('Telemetry file missing')
            coadds = 10
            int_time = 6000
            fpe_temp = 43
            fpa_temp = -21

    ##############Image Processing begins here##############################
        full_frame = read_idl_file(data_file)
        check_image = create_image_active_region(full_frame)
        raw_image = create_final_image(np.array(full_frame))
        # print('Max. Val. Raw = ', np.max(raw_image))
        quads = ['Quad A', 'Quad B', 'Quad C', 'Quad D']

        ##########################OFFSET REMOVAL###############################
        # Input : Full_Frame TEMPO IMAGE
        # Otput : Bias Subtracted Active Region. The active region dimensions are
        #now 1028*1024. 2 pixel lines from SMEAR overclocks are now used for
        #to store storage summation information
        # For dark data, only offset removal is needed to compute the dark current
        # For light data, additional processing such as linearity, smear and
        #cross talk is needed

        bias_removed_quads = perform_bias_subtraction(full_frame)

        #print('Max. Val. Offset Removed  = ', np.max(bias_removed_quads))

        text1 = telemetry_file_name+'.img' +' (Raw Data)\n Int. time:' + str(round(int_time, 1))+ \
                           'ms, Co-adds:' +str(int(coadds))+\
                           ', FPE temp:'+ str(round(fpe_temp, 1))+'C, ' + \
                           ' FPA temp: ' + str(round(fpa_temp, 1))+'C'

        # Now let us save the raw image
        raw_image_save = os.path.join(image_save_dir, 'raw_image')
        if not os.path.exists(raw_image_save):
            os.makedirs(raw_image_save)
        plot_save_dir = raw_image_save + '/' + data_path + '.png'
        #prnu_spectrometer[prnu_spectrometer < 0.9] = 0.9
        #prnu_spectrometer[prnu_spectrometer > 1.2] = 1.2
        #print(np.min(check_image))
        #create_image(check_image, text1, plot_save_dir)
        #---------------------------------------------------------------------

        #########################NON LINEARITY CORRECTION#######################
        # Input : Bias Subtracted Active regions
        # Output : Linearized Active Region Quads
        if linearity:
            linearized_a = apply_linearity_correction(
                bias_removed_quads[0, :, :], quads[0], coadds)
            linearized_b = apply_linearity_correction(
                bias_removed_quads[1, :, :], quads[1], coadds)
            linearized_c = apply_linearity_correction(
                bias_removed_quads[2, :, :], quads[2], coadds)
            linearized_d = apply_linearity_correction(
                bias_removed_quads[3, :, :], quads[3], coadds)
            linearized_quads = np.array(
                [linearized_a, linearized_b, linearized_c, linearized_d])
            #print(np.mean(linearized_quads))
        else:
            linearized_quads = bias_removed_quads
            #----------------------------------------------------------------------
            ##########################SMEAR REMOVAL################################
            # Input : linearized quads ( all quads together)
            # Output : SMEAR offset corrected Quad

        #### lets' create the masked array with outlier mask################
        # The outlier mask is array of list of 4 quads.
        #print('Max. Val. Linearized  = ', np.max(linearized_quads))
        outlier_mask = read_outlier_mask()

        # Note : all the arrays after this step are masked arrays

        if smear:
            smear_removed_quads = perform_smear_removal(
                linearized_quads, int_time, outlier_mask)
        else:
            smear_removed_quads = linearized_quads
            #----------------------------------------------------------------------

            ##########################CROSS-TALK REMOVAL###########################
            # Input : smear removed quads (all quads together)
            # Output : cross talk removed quads
        #print('Max. Val. Smeared  = ', np.max(smear_removed_quads))

        if cross_talk:
            cross_talk_removed_quads = remove_cross_talk(
                np.array(smear_removed_quads))
        else:
            cross_talk_removed_quads = smear_removed_quads
            #----------------------------------------------------------------------
        #print('Max. Val. Cross Talked = ', np.max(cross_talk_removed_quads))
        processed_image = create_final_image(
            np.array(cross_talk_removed_quads))
        processed_image = processed_image - dark_current

        #prnu_map = parse_prnu_file()
        prnu_map = read_prnu_files()
        prnu_spectrometer = create_final_image(np.array(prnu_map))
        prnu_spectrometer[prnu_spectrometer > 1.03] = 1.02
        prnu_spectrometer[prnu_spectrometer < 0.97] = 0.98

        outlier_spectrometer = create_final_image(np.array(outlier_mask))

        nx_quad, ny_quad = processed_image.shape
        outlier_spectrometer = np.reshape(outlier_spectrometer,
                                          (nx_quad * ny_quad, 1))
        outlier_detectors = np.array(np.where([outlier_spectrometer == 1]))
        #print('outliers =', outlier_detectors.shape[1])
        outlier_spectrometer = np.reshape(outlier_spectrometer,
                                          (nx_quad, ny_quad))
        #create_image(outlier_spectrometer,'outliers = '+str(outlier_detectors.shape[1]),'b')
        processed_image = processed_image / (coadds * prnu_spectrometer)
        #print(np.min(processed_image))
        wavelen = np.loadtxt(os.path.join(image_dir, 'Wavelength.csv'),
                             delimiter=',')
        wavelen = wavelen[count]

        #processed_image[processed_image>=0.85*16383] = 16383
        text1 = telemetry_file_name+'.img' +' (Laser WL:' + str(wavelen)+' nm) \n Int. time:' + '250 '+ \
                           'ms, Co-adds:' +str(int(coadds))+\
                           ', FPE temp:'+ str(round(fpe_temp, 1))+'C, ' + \
                           ' FPA temp: ' + str(round(fpa_temp, ))+'C'
        processed_image_save = os.path.join(image_save_dir, 'processed_image')
        if not os.path.exists(processed_image_save):
            os.makedirs(processed_image_save)
        plot_save_dir = processed_image_save + '/' + data_path + '.png'
        #processed_image = uniform_filter(processed_image, size=(5, 3), mode='mirror')
        create_image(processed_image, text1, plot_save_dir)

        row_mean = np.mean(processed_image, axis=1)
        max_row_each = np.where(
            row_mean == max(np.mean(processed_image, axis=1)))[0]
        max_row_each = max_row_each[0]
        max_row = max_row_each
        #print(max_row_each)

        #        processed_image = processed_image[max_row-12: max_row+12, :]
        #        processed_image = uniform_filter(processed_image, size=(10, 4), mode='mirror')

        # print(wavelen)
        #        #plt.plot(processed_image[max_row-2, :], 'purple', label='2 lines up')
        #        plt.plot(processed_image[max_row-1, :], 'y-', label='1 lines up')
        #        #plt.plot(processed_image[max_row+2, :], 'g-', label='2 lines down')
        #        plt.plot(processed_image[max_row+1, :], 'r-', label='1 line down')
        #        plt.plot(processed_image[max_row, :], 'k-', label='Brightest')
        #        plt.grid(True, linestyle  = ':')
        #        plt.title('Profile along Spectral direction (Laser Wavelength ='  + str(wavelen) + 'nm)')
        #        plt.xlabel('Spatial Pixel Index')
        #        plt.ylabel('Counts (DN)')
        #        plt.show()
        #        col_index = np.sort([randint(100, 2000) for p in range(0, 10)])
        #        col_index = [480, 485, 490, 500, 510]
        #        for i in col_index:
        #                plt.plot(processed_image[:, i]/np.max(processed_image[:, i]),
        #                         'o--', label='Col. ' + str(i))
        #        plt.legend(loc='best', ncol=4)
        #        plt.grid(True, linestyle=':')
        #        plt.title('Profile along Spectral direction (Laswer WL:' + str(wavelen)+' nm)')
        #        plt.xlabel('Spectral Pixel Index')
        #        plt.ylabel('Normalized Counts')
        #        plt.show()
        #        cc

        #        cc
        wavelengths_all = []
        #print(count)
        smile_all = []
        FWHM_all = []

        for i in range(15, 2035):
            wavelengths = [wavelen] * 2020
            wavelengths_all.append(wavelengths)
            #wavelengths_all.append(wavelengths)

            smile, FWHM = fit_gaussian_func(max_row_each,
                                            processed_image[:, i], wavelen, i)
            smile_all.append(smile)
            FWHM_all.append(FWHM)
        #print(FWHM)
        mean_val = np.mean(smile_all)
        smile_all = moving_average(smile_all)
        #smile_all = uniform_filter(smile_all, size=6, mode='mirror')
        #print(np.array(smile_all).shape)
        band_center = pd.Series(smile_all)
        # Spectral alignment is calculated by comuting the difference between max and min spectral centers within 4 pixel moving window
        max_val = band_center.rolling(4).max()
        min_val = band_center.rolling(4).min()
        spectral_alignment = max_val - min_val
        spectral_alingn_unct.append(spectral_alignment)
        #plt.plot(max_val-min_val, label=str(wavelen)+ 'nm' )
        count = count + 1

        #plt.plot(wavelengths[1400:], FWHM_all[1400:], 'o', label=str(wavelen)+ 'nm')
        #plt.plot(uniform_filter(FWHM_all, size=12,mode='mirror'), label=str(wavelen)+ 'nm')
        #plt.show()

        #fit_gaussian_func(x_data, y_data, wavelen)
#
#
##        #max_row = 29 # for 297.8 nm
##        #max_row = 1546 #for 640nm
##        #max_row = 1369 #for 605nm
##        #max_row = 1050 #for 605nm
##        #max_row = 142 # for 320 nm
##
##        #max_row = 91 # for 310 nm
##        max_row = 15 # 675 nm
##        #max_row = 2032 # for 736 nm
##        #check_val = 1392
#        col_index = np.sort([randint(100, 2000) for p in range(0, 2)])
#        for i in col_index:
#
#            plt.plot(processed_image[:, i],
#                     'o--', label='Spatial Index ' + str(i))
#            plt.legend(loc='best', ncol=4)
#        plt.grid(True, linestyle=':')
#        plt.title('Profile along Spectral direction (Laser Wavelength ='  + str(wavelen) + 'nm')
#        plt.xlabel('Spectral Pixel Index')
#        plt.ylabel('Counts (DN)')
#        #plt.xlim(655, 675)
#        #plt.yscale('log')
#        plt.xlim(max_row_each-10, max_row_each+10)
#
#plt.show()

#
#plt.figure()
    spectral_alignment_unct = np.std(np.array(spectral_alingn_unct), axis=0)
    print(spectral_alignment_unct.shape)
    plt.plot(spectral_alignment_unct, 'k', label='1-Sigma Spectral. Alignment')
    plt.axhline(y=0.01,
                color='red',
                linestyle='--',
                linewidth=2,
                label='TEMPO Requirement')
    plt.title(
        'TEMPO Number of Samples per FWHM Vs. Wavelengths\n (*** Note: 2 pixel smoothing in spectral,12 pixel in spatial direction)',
        fontsize=12)
    plt.grid(True, linestyle=':')
    plt.xlabel('Wavelength (nm)', fontsize=12)
    plt.ylabel('Samples/FWHM', fontsize=12)
    #plt.ylim(2.6, 3.3)
    #plt.ylim(2.4, 3)

    #plt.ylim(-0.3, 0.3)
    # plt.axhline(y= 2.7, color='red', linestyle='--',  linewidth=2)
    #plt.axhline(y= -0.2, color='red', linestyle='--', linewidth=2)
    #plt.axhline(y= 0.2, color='red', linestyle='--', linewidth=2)

    plt.xlim(0, 2500)
    plt.legend(loc='best')
    #plt.text(500, 0.25, '----TEMPO Requirement',  color='red', fontsize=12)
    #plt.text(500, 2.72, '----TEMPO Requirement',  color='red', fontsize=12)
    plt.show()
Example #4
0
def main():
    """
    This is the main function that does all the processing. It calls the required
    functions to process the raw image. Raw images are saved in .sav (IDL)
    format and read through this script.
    """
    data_dir = r'F:\TEMPO\Data\GroundTest\FPS\Spectrometer\Spectral_Smile\Flight_Slight'
    image_dir = os.path.join(data_dir, 'saved_quads')
    save_dir = os.path.join(image_dir, 'processed_image')
    image_save_dir = os.path.join(image_dir, 'saved_plots')
    telemetry_dir = os.path.join(data_dir, 'processed/h5')
    linearity = 0  # option to turn on/off linearity correction
    smear = 1
    cross_talk = 1
    bandpass_val = []
    bandpass_val_norm = []
    normalization_factor_all = []
    dark_current = 0
    if not os.path.exists(save_dir):
        os.makedirs(save_dir)
#    data_path_all = sorted([each for each in os.listdir(image_dir)
#                            if each.startswith('2017_07_30_00_24_59_38064')
#                            and  each.endswith('.sav')])
#
    data_path_all = sorted(
        [each for each in os.listdir(image_dir) if each.endswith('.sav')])
    #
    #    #dark_data = data_path_all[1:len(data_path_all):2]

    print('Total data = ', len(data_path_all))
    #    resultFile = open (os.path.join(save_dir,'file_name_all.csv'),'w')
    #
    #    for results in data_path_all[0:15]:
    #         resultFile.write(results+"\n")
    #    resultFile.close()
    if dark_current:
        dark_current = calculate_dark_current(image_dir, coadds=1)
    else:
        dark_current = 0

    for data_path in data_path_all[7:]:

        data_file = os.path.join(image_dir, data_path)
        print(data_path)
        #        cc
        telemetry_file_name = data_path.split('.')[0]
        #################### Telemetry Statistics###############################
        # Now lets parse the telemetry file to get the required information###
        #Note; The function can return the CCD temp and FPE temp. But these are
        #not needed on the image processing algorithm.. atleast for the time being

        telemetry_file = os.path.join(telemetry_dir,
                                      telemetry_file_name + '.h5')

        if os.path.exists(telemetry_file):
            coadds, int_time, fpe_temp, fpa_temp = parse_telemetry_file(
                telemetry_dir, telemetry_file_name)
            print('FPE Temp. = ', round(fpe_temp, 2), 'FPA Temp = ',
                  round(fpa_temp, 2))
        else:

            print('Telemetry file missing')
            coadds = 10
            int_time = 6000
            fpe_temp = 43
            fpa_temp = -21

    ##############Image Processing begins here##############################
        full_frame = read_idl_file(data_file)
        check_image = create_image_active_region(full_frame)
        raw_image = create_final_image(np.array(full_frame))
        # print('Max. Val. Raw = ', np.max(raw_image))
        quads = ['Quad A', 'Quad B', 'Quad C', 'Quad D']

        ##########################OFFSET REMOVAL###############################
        # Input : Full_Frame TEMPO IMAGE
        # Otput : Bias Subtracted Active Region. The active region dimensions are
        #now 1028*1024. 2 pixel lines from SMEAR overclocks are now used for
        #to store storage summation information
        # For dark data, only offset removal is needed to compute the dark current
        # For light data, additional processing such as linearity, smear and
        #cross talk is needed

        bias_removed_quads = perform_bias_subtraction(full_frame)

        #print('Max. Val. Offset Removed  = ', np.max(bias_removed_quads))

        text1 = telemetry_file_name+'.img' +' (Raw Data)\n Int. time:' + str(round(int_time, 1))+ \
                           'ms, Co-adds:' +str(int(coadds))+\
                           ', FPE temp:'+ str(round(fpe_temp, 1))+'C, ' + \
                           ' FPA temp: ' + str(round(fpa_temp, 1))+'C'

        # Now let us save the raw image
        raw_image_save = os.path.join(image_save_dir, 'raw_image')
        if not os.path.exists(raw_image_save):
            os.makedirs(raw_image_save)
        plot_save_dir = raw_image_save + '/' + data_path + '.png'
        #prnu_spectrometer[prnu_spectrometer < 0.9] = 0.9
        #prnu_spectrometer[prnu_spectrometer > 1.2] = 1.2
        #print(np.min(check_image))
        #create_image(check_image, text1, plot_save_dir)
        #---------------------------------------------------------------------

        #########################NON LINEARITY CORRECTION#######################
        # Input : Bias Subtracted Active regions
        # Output : Linearized Active Region Quads
        if linearity:
            linearized_a = apply_linearity_correction(
                bias_removed_quads[0, :, :], quads[0], coadds)
            linearized_b = apply_linearity_correction(
                bias_removed_quads[1, :, :], quads[1], coadds)
            linearized_c = apply_linearity_correction(
                bias_removed_quads[2, :, :], quads[2], coadds)
            linearized_d = apply_linearity_correction(
                bias_removed_quads[3, :, :], quads[3], coadds)
            linearized_quads = np.array(
                [linearized_a, linearized_b, linearized_c, linearized_d])
            #print(np.mean(linearized_quads))
        else:
            linearized_quads = bias_removed_quads
            #----------------------------------------------------------------------
            ##########################SMEAR REMOVAL################################
            # Input : linearized quads ( all quads together)
            # Output : SMEAR offset corrected Quad

        #### lets' create the masked array with outlier mask################
        # The outlier mask is array of list of 4 quads.
        #print('Max. Val. Linearized  = ', np.max(linearized_quads))
        outlier_mask = read_outlier_mask()

        # Note : all the arrays after this step are masked arrays

        if smear:
            smear_removed_quads = perform_smear_removal(
                linearized_quads, int_time, outlier_mask)
        else:
            smear_removed_quads = linearized_quads
            #----------------------------------------------------------------------

            ##########################CROSS-TALK REMOVAL###########################
            # Input : smear removed quads (all quads together)
            # Output : cross talk removed quads
        #print('Max. Val. Smeared  = ', np.max(smear_removed_quads))

        if cross_talk:
            cross_talk_removed_quads = remove_cross_talk(
                np.array(smear_removed_quads))
        else:
            cross_talk_removed_quads = smear_removed_quads
            #----------------------------------------------------------------------
        #print('Max. Val. Cross Talked = ', np.max(cross_talk_removed_quads))
        processed_image = create_final_image(
            np.array(cross_talk_removed_quads))
        processed_image = processed_image - dark_current

        #prnu_map = parse_prnu_file()
        prnu_map = read_prnu_files()
        prnu_spectrometer = create_final_image(np.array(prnu_map))
        prnu_spectrometer[prnu_spectrometer > 1.03] = 1.02
        prnu_spectrometer[prnu_spectrometer < 0.97] = 0.98

        outlier_spectrometer = create_final_image(np.array(outlier_mask))

        nx_quad, ny_quad = processed_image.shape
        outlier_spectrometer = np.reshape(outlier_spectrometer,
                                          (nx_quad * ny_quad, 1))
        outlier_detectors = np.array(np.where([outlier_spectrometer == 1]))
        #print('outliers =', outlier_detectors.shape[1])
        outlier_spectrometer = np.reshape(outlier_spectrometer,
                                          (nx_quad, ny_quad))
        #create_image(outlier_spectrometer,'outliers = '+str(outlier_detectors.shape[1]),'b')
        processed_image = processed_image / (prnu_spectrometer)
        #print(np.min(processed_image))

        #processed_image[processed_image>=0.85*16383] = 16383
        text1 = telemetry_file_name+'.img' +' (Spectral Bandpass 675 nm)\n Int. time:' + str(round(int_time, 1))+ \
                           'ms, Co-adds:' +str(int(coadds))+\
                           ', FPE temp:'+ str(round(fpe_temp, 1))+'C, ' + \
                           ' FPA temp: ' + str(round(fpa_temp, ))+'C'
        processed_image_save = os.path.join(image_save_dir, 'processed_image')
        if not os.path.exists(processed_image_save):
            os.makedirs(processed_image_save)
        plot_save_dir = processed_image_save + '/' + data_path + '.png'
        processed_image = uniform_filter(processed_image,
                                         size=3,
                                         mode='constant')

        #create_image(processed_image, text1, plot_save_dir)

        row_mean = np.mean(processed_image, axis=1)
        max_row_each = np.where(
            row_mean == max(np.mean(processed_image, axis=1)))[0]
        max_row_each = max_row_each[0]
        print(max_row_each)
        #
        #
        #        #max_row = 29 # for 297.8 nm
        #        #max_row = 1546 #for 640nm
        #        #max_row = 1369 #for 605nm
        #        #max_row = 1050 #for 605nm
        #        #max_row = 142 # for 320 nm
        #
        #        #max_row = 91 # for 310 nm
        #        max_row = 15 # 675 nm
        #        #max_row = 2032 # for 736 nm
        #        #check_val = 1392
        col_index = np.sort([randint(100, 2000) for p in range(0, 10)])
        for i in col_index:

            plt.plot(processed_image[:, i], 'o--', label='Col. ' + str(i))
            plt.legend(loc='best', ncol=4)
        plt.grid(True, linestyle=':')
        plt.title(
            'Profile along Spectral direction (Spectral Bandpass 297 nm)')
        plt.xlabel('Spectral Pixel Index')
        plt.ylabel('Counts (DN)')
        #plt.xlim(655, 675)
        #plt.yscale('log')
        plt.xlim(max_row_each - 10, max_row_each + 10)

        plt.show()
        cc
Example #5
0
def main():
    """
    This is the main function that does all the processing. It calls the required
    functions to process the raw image. Raw images are saved in .sav (IDL)
    format and read through this script.
    """
    data_dir = r'F:\TEMPO\Data\GroundTest\FPS\Spectrometer\Photon_Transfer_TVAC'
    telemetry_dir = r'F:\TEMPO\Data\GroundTest\FPS\Spectrometer\2018.06.28'
    image_dir = os.path.join(data_dir, r'saved_quads')
    save_dir = os.path.join(image_dir, 'processed_image')
    image_save_dir = os.path.join(image_dir, 'saved_plots_straylight')
    telemetry_dir = os.path.join(telemetry_dir, 'processed/h5')
    linearity = 0  # option to turn on/off linearity correction
    smear = 1
    cross_talk = 1
    #    all_image = []
    #    var_all = []
    #    signal_all = []
    #dark_current = 0
    if not os.path.exists(save_dir):
        os.makedirs(save_dir)


#    data_path_all = sorted([each for each in os.listdir(image_dir)
#                            if each.startswith('2017_07_30_00_24_59_38064')
#                            and  each.endswith('.sav')])
#
    data_path_all = sorted(
        [each for each in os.listdir(image_dir) if each.endswith('.sav')])

    #    #dark_data = data_path_all[1:len(data_path_all):2]

    print('Total data = ', len(data_path_all))
    #    resultFile = open (os.path.join(save_dir,'file_name_all.csv'),'w')
    #
    #    for results in data_path_all[0:15]:
    #         resultFile.write(results+"\n")
    #    resultFile.close()
    #count = 0
    #peak_loc_all = []
    for data_path in data_path_all:
        data_file = os.path.join(image_dir, data_path)
        print(data_path)

        #        cc
        telemetry_file_name = data_path.split('.')[0]
        #################### Telemetry Statistics###############################
        # Now lets parse the telemetry file to get the required information###
        #Note; The function can return the CCD temp and FPE temp. But these are
        #not needed on the image processing algorithm.. atleast for the time being

        telemetry_file = os.path.join(telemetry_dir,
                                      telemetry_file_name + '.h5')
        if os.path.exists(telemetry_file):
            coadds, int_time, fpe_temp, fpa_temp = parse_telemetry_file(
                telemetry_dir, telemetry_file_name)
        # print('FPE Temp. = ', round(fpe_temp, 2), 'FPA Temp = ',
        # round(fpa_temp, 2))
        #print('Integ. Time =' , int_time)
        #print(coadds)

        else:
            #print('Telemetry file missing')
            coadds = 10
            int_time = 6000
            fpe_temp = 43
            fpa_temp = -21.5

    ##############Image Processing begins here##############################

    #print(coadds)

        full_frame = read_idl_file(data_file)

        full_frame = perform_coaddition_correction(full_frame, coadds)

        #full_frame = full_frame/coadds
        #print(np.max(full_frame))
        #cc
        #check_image = create_image_active_region(full_frame)
        #raw_image = create_final_image(np.array(full_frame))
        # print('Max. Val. Raw = ', np.max(raw_image))
        quads = ['Quad A', 'Quad B', 'Quad C', 'Quad D']

        ##########################OFFSET REMOVAL###############################
        # Input : Full_Frame TEMPO IMAGE
        # Otput : Bias Subtracted Active Region. The active region dimensions are
        #now 1028*1024. 2 pixel lines from SMEAR overclocks are now used for
        #to store storage summation information
        # For dark data, only offset removal is needed to compute the dark current
        # For light data, additional processing such as linearity, smear and
        #cross talk is needed

        bias_removed_quads = perform_bias_subtraction(full_frame)

        #print('Max. Val. Offset Removed  = ', np.max(bias_removed_quads))

        #create_image(raw_image/coadds, text1, plot_save_dir)
        #prnu_spectrometer[prnu_spectrometer < 0.9] = 0.9
        #prnu_spectrometer[prnu_spectrometer > 1.2] = 1.2
        #print(np.min(check_image))
        #create_image(check_image, text1, plot_save_dir)
        #---------------------------------------------------------------------

        #########################NON LINEARITY CORRECTION#######################
        # Input : Bias Subtracted Active regions
        # Output : Linearized Active Region Quads
        # pass the difference instead of coadds
        if linearity:
            linearized_a = apply_linearity_correction(
                bias_removed_quads[0, :, :], quads[0], coadds)
            linearized_b = apply_linearity_correction(
                bias_removed_quads[1, :, :], quads[1], coadds)
            linearized_c = apply_linearity_correction(
                bias_removed_quads[2, :, :], quads[2], coadds)
            linearized_d = apply_linearity_correction(
                bias_removed_quads[3, :, :], quads[3], coadds)
            linearized_quads = np.array(
                [linearized_a, linearized_b, linearized_c, linearized_d])
            #print(np.mean(linearized_quads))
        else:
            linearized_quads = bias_removed_quads
            #----------------------------------------------------------------------
            ##########################SMEAR REMOVAL################################
            # Input : linearized quads ( all quads together)
            # Output : SMEAR offset corrected Quad

        #### lets' create the masked array with outlier mask################
        # The outlier mask is array of list of 4 quads.
        #print('Max. Val. Linearized  = ', np.max(linearized_quads))

        if smear:
            smear_removed_quads = perform_smear_removal(
                linearized_quads, int_time)

        else:
            smear_removed_quads = linearized_quads
            #----------------------------------------------------------------------

            ##########################CROSS-TALK REMOVAL###########################
            # Input : smear removed quads (all quads together)
            # Output : cross talk removed quads
        #print('Max. Val. Smeared  = ', np.max(smear_removed_quads))

        if cross_talk:
            cross_talk_removed_quads = remove_cross_talk(
                np.array(smear_removed_quads))
        else:
            cross_talk_removed_quads = smear_removed_quads
            #----------------------------------------------------------------------

        processed_image = create_final_image(
            np.array(cross_talk_removed_quads))
        straylight_image = processed_image[:, 0:2]
        straylight_data = np.mean(straylight_image, axis=1)

        # Gain is 17.2
        processed_file_name = image_save_dir + '/' + data_path + '.csv'
        np.savetxt(processed_file_name,
                   straylight_data,
                   fmt='%1.2f',
                   delimiter=",")
def main():
    """
    This is the main function that does all the processing. It calls the required
    functions to process the raw image. Raw images are saved in .sav (IDL)
    format and read through this script.
    """
    data_dir = r'F:\TEMPO\Data\GroundTest\FPS\FPA_Gain_vs_Temp'
    telemetry_dir = r'F:\TEMPO\Data\GroundTest\FPS\Spectrometer\2016.09.19'
    image_save_dir = r'C:\Users\nmishra\Workspace\TEMPO\TEMPO_DC_FPA_TEMP_SENSTIVITY_NEW'
    telemetry_dir = os.path.join(telemetry_dir, 'processed/h5')
    linearity = 1  # option to turn on/off linearity correction
    smear = 1
    cross_talk = 1
    temp_correction = 1

    if not os.path.exists(image_save_dir):
        os.makedirs(image_save_dir)

    temperature_files = [each for each in os.listdir(data_dir) \
                        if each.endswith('_PT_Dark')]
    for k in range(1, len(temperature_files)):

        image_data_files = os.path.join(data_dir, temperature_files[k],
                                        'Script_Data', 'saved_quads')

        data_path_all = [
            each for each in os.listdir(image_data_files)
            if not each.endswith('_118000.dat.sav')
            if each.endswith('.dat.sav')
        ]

        data_path_op_int = [each for each in os.listdir(image_data_files) \
                         if each.endswith('118000.dat.sav')][-1]

        data_path_all.append(data_path_op_int)
        save_dir = os.path.join(image_save_dir, temperature_files[k])
        all_quad_D_odd = []
        all_quad_D_even = []
        all_quad_C_odd = []
        all_quad_C_even = []
        all_quad_B_odd = []
        all_quad_B_even = []
        all_quad_A_odd = []
        all_quad_A_even = []
        all_int_time = []

        for data_path in data_path_all:
            data_path_name_split = data_path.split('_')
            int_time = round(int(data_path_name_split[-1].split('.')[0]))
            int_time = int(int_time) / 1000
            #print(int_time)
            all_int_time.append(int_time)

            data_file = os.path.join(image_data_files, data_path)
            print(data_path)

            #        cc
            telemetry_file_name = data_path.split('.')[0]
            #################### Telemetry Statistics###############################
            # Now lets parse the telemetry file to get the required information###
            #Note; The function can return the CCD temp and FPE temp. But these are
            #not needed on the image processing algorithm.. atleast for the time being

            telemetry_file = os.path.join(telemetry_dir,
                                          telemetry_file_name + '.h5')
            if os.path.exists(telemetry_file):
                coadds, int_time, fpe_temp, fpa_temp = parse_telemetry_file(
                    telemetry_dir, telemetry_file_name)
                # print('FPE Temp. = ', round(fpe_temp, 2), 'FPA Temp = ',
                # round(fpa_temp, 2))
                print('Integ. Time =', int_time)
                print(coadds)

            else:
                #print('Telemetry file missing')
                coadds = 100
                #int_time = 6000
                fpe_temp = 25
                fpa_temp = 25

        ##############Image Processing begins here##############################

            full_frame = read_idl_file(data_file)

            if np.array(full_frame).ndim == 4:
                full_frame = np.mean(full_frame, axis=0)
            else:
                full_frame = perform_coaddition_correction(full_frame, coadds)

            quads = ['Quad A', 'Quad B', 'Quad C', 'Quad D']

            ##########################OFFSET REMOVAL###############################
            # Input : Full_Frame TEMPO IMAGE
            # Otput : Bias Subtracted Active Region. The active region dimensions are
            #now 1028*1024. 2 pixel lines from SMEAR overclocks are now used for
            #to store storage summation information
            bias_removed_quads = perform_bias_subtraction(full_frame)
            #---------------------------------------------------------------------
            #########################NON LINEARITY CORRECTION#######################
            if temp_correction:
                temp_corrected_quads = perform_temp_correction(
                    bias_removed_quads, fpe_temp)
            else:
                temp_corrected_quads = bias_removed_quads

        #########################NON LINEARITY CORRECTION#######################
        # Input : Bias Subtracted Active regions
        # Output : Linearized Active Region Quads
        # pass the difference instead of coadds
            if linearity:
                linearized_a = apply_linearity_correction(
                    temp_corrected_quads[0, :, :], quads[0])
                linearized_b = apply_linearity_correction(
                    temp_corrected_quads[1, :, :], quads[1])
                linearized_c = apply_linearity_correction(
                    temp_corrected_quads[2, :, :], quads[2])
                linearized_d = apply_linearity_correction(
                    temp_corrected_quads[3, :, :], quads[3])
                linearized_quads = np.array(
                    [linearized_a, linearized_b, linearized_c, linearized_d])
                #print(np.mean(linearized_quads))
            else:
                linearized_quads = temp_corrected_quads

            if cross_talk:
                cross_talk_removed_quads = remove_cross_talk(linearized_quads)
            else:
                cross_talk_removed_quads = linearized_quads

            if smear:
                smear_removed_quads = perform_smear_removal(
                    np.array(cross_talk_removed_quads), int_time)
            else:
                smear_removed_quads = cross_talk_removed_quads

            processed_image = create_final_image(np.array(smear_removed_quads))
            quad_d = processed_image[0:1028, 0:1024]
            quad_d_odd = quad_d[:, ::2]
            quad_d_even = quad_d[:, 1::2]
            all_quad_D_odd.append(
                np.mean(filter_outlier_median(quad_d_odd[300:900, 200:400])))
            all_quad_D_even.append(
                np.mean(filter_outlier_median(quad_d_even[300:900, 200:400])))

            quad_c = processed_image[0:1028, 1024:]
            quad_c_odd = quad_c[:, ::2]
            quad_c_even = quad_c[:, 1::2]
            all_quad_C_odd.append(
                np.mean(filter_outlier_median(quad_c_odd[300:900, 200:400])))
            all_quad_C_even.append(
                np.mean(filter_outlier_median(quad_c_even[300:900, 200:400])))

            quad_a = processed_image[1028:, 0:1024]
            quad_a_odd = quad_a[:, ::2]

            quad_a_even = quad_a[:, 1::2]
            all_quad_A_odd.append(
                np.mean(filter_outlier_median(quad_a_odd[300:900, 200:400])))
            all_quad_A_even.append(
                np.mean(filter_outlier_median(quad_a_even[300:900, 200:400])))

            quad_b = processed_image[1028:, 1024:]
            quad_b_odd = quad_b[:, ::2]
            quad_b_even = quad_b[:, 1::2]
            all_quad_B_odd.append(
                np.mean(filter_outlier_median(quad_b_odd[300:900, 200:400])))
            all_quad_B_even.append(
                np.mean(filter_outlier_median(quad_b_even[300:900, 200:400])))

        dframe1 = pd.DataFrame({
            'Int_time.': all_int_time,
            'Avg_Quad_A_odd': all_quad_A_odd,
            'Avg_Quad_A_even': all_quad_A_even,
            'Avg_Quad_B_odd': all_quad_B_odd,
            'Avg_Quad_B_even': all_quad_B_even,
            'Avg_Quad_C_odd': all_quad_C_odd,
            'Avg_Quad_C_even': all_quad_C_even,
            'Avg_Quad_D_odd': all_quad_D_odd,
            'Avg_Quad_D_even': all_quad_D_even
        })
        processed_file_dir = os.path.join(save_dir, temperature_files[k])
        if not os.path.exists(processed_file_dir):
            os.makedirs(processed_file_dir)

        processed_file_name = processed_file_dir + '/' + temperature_files[
            k] + '_Photon_transfer_data_all_FPA.csv'
        dframe1.to_csv(processed_file_name)
def main():
    """
    This is the main function that does all the processing. It calls the required
    functions to process the raw image. Raw images are saved in .sav (IDL)
    format and read through this script.
    """
    data_dir = r'F:\TEMPO\Data\GroundTest\FPS\Spectrometer\Dark_data\2017.07.10'
    plot_dir = os.path.join(data_dir, 'Dark_Current_Image')
    image_dir = os.path.join(data_dir, 'saved_quads')    
    image_save_dir = os.path.join(image_dir, 'saved_plots')
    telemetry_dir = r'E:\Image Data\2018.07.10\processed\h5'
    linearity = 0 # option to turn on/off linearity correction
    smear = 1
    cross_talk = 1 
    dark_current = 1
    if not os.path.exists(plot_dir):
        os.makedirs(plot_dir)
#    data_path_all = sorted([each for each in os.listdir(image_dir)
#                            if each.startswith('2017_07_30_00_24_59_38064')
#                            and  each.endswith('.sav')])
#
    data_path_all = sorted([each for each in os.listdir(image_dir)
                            if each.endswith('.sav')])
    print(len(data_path_all))
    
    
    print('Total data = ', len(data_path_all))   
    count = 0 
    all_dark_current=[]
   
    for data_path in data_path_all[1:]:
        data_file = os.path.join(image_dir, data_path)
        print(data_path)
#        cc
        telemetry_file_name = data_path.split('.')[0]
       #################### Telemetry Statistics###############################
        # Now lets parse the telemetry file to get the required information###
        #Note; The function can return the CCD temp and FPE temp. But these are
        #not needed on the image processing algorithm.. atleast for the time being

        telemetry_file = os.path.join(telemetry_dir, telemetry_file_name+'.h5')
       
        
        if os.path.exists(telemetry_file):
            coadds, int_time, fpe_temp, fpa_temp = parse_telemetry_file(telemetry_dir,
                                                                        telemetry_file_name)
            print('FPE Temp. = ', round(fpe_temp, 2), 'FPA Temp = ',
                  round(fpa_temp, 2))
            print(int_time)
            print(coadds)
        else:

            print('Telemetry file missing')
            coadds = 10
            int_time = 160
            fpe_temp = 35.8
            fpa_temp = 11.9
           
       ##############Image Processing begins here##############################
        #CC 
        full_frame = read_idl_file(data_file)
        #check_image = create_image_active_region(full_frame)
        #raw_image = create_final_image(np.array(full_frame))
        #create_image(raw_image, 'a', 'b')
       # print('Max. Val. Raw = ', np.max(raw_image))
        quads = ['Quad A', 'Quad B', 'Quad C', 'Quad D']

        ##########################OFFSET REMOVAL###############################
        # Input : Full_Frame TEMPO IMAGE
        # Otput : Bias Subtracted Active Region. The active region dimensions are
        #now 1028*1024. 2 pixel lines from SMEAR overclocks are now used for
        #to store storage summation information
        # For dark data, only offset removal is needed to compute the dark current
        # For light data, additional processing such as linearity, smear and
        #cross talk is needed


        bias_removed_quads = perform_bias_subtraction(full_frame)
        #raw_image = create_final_image(np.array(bias_removed_quads))
        #create_image(raw_image/coadds, 'a', 'b')
        

        #print('Max. Val. Offset Removed  = ', np.max(bias_removed_quads))

        text1 = telemetry_file_name + '.img' + ' (Raw Data)\n Int. time:' + str(round(int_time, 1)) + \
                           'ms, Co-adds:' + str(int(coadds)) + \
                           ', FPE temp:' + str(round(fpe_temp, 1)) + 'C, ' + \
                           ' FPA temp: ' + str(round(fpa_temp, 1)) + 'C'
                          
                             

        # Now let us save the raw image
        raw_image_save = os.path.join(image_save_dir, 'raw_image')
        if not os.path.exists(raw_image_save):
            os.makedirs(raw_image_save)
        plot_save_dir = raw_image_save + '/'+ data_path+'.png'
        #prnu_spectrometer[prnu_spectrometer < 0.9] = 0.9
        #prnu_spectrometer[prnu_spectrometer > 1.2] = 1.2
        #print(np.min(check_image))
        #create_image(check_image, text1, plot_save_dir)
        #---------------------------------------------------------------------

       #########################NON LINEARITY CORRECTION#######################
        # Input : Bias Subtracted Active regions
        # Output : Linearized Active Region Quads
        if linearity:
            linearized_a = apply_linearity_correction(bias_removed_quads[0, :, :],
                                                      quads[0], coadds)
            linearized_b = apply_linearity_correction(bias_removed_quads[1, :, :],
                                                      quads[1], coadds)
            linearized_c = apply_linearity_correction(bias_removed_quads[2, :, :],
                                                      quads[2], coadds)
            linearized_d = apply_linearity_correction(bias_removed_quads[3, :, :],
                                                      quads[3], coadds)
            linearized_quads = np.array([linearized_a, linearized_b,
                                         linearized_c, linearized_d])
            #print(np.mean(linearized_quads))
        else:
            linearized_quads = bias_removed_quads
            #----------------------------------------------------------------------
            ##########################SMEAR REMOVAL################################
            # Input : linearized quads ( all quads together)
            # Output : SMEAR offset corrected Quad

        #### lets' create the masked array with outlier mask################
        # The outlier mask is array of list of 4 quads.
        #print('Max. Val. Linearized  = ', np.max(linearized_quads))
        outlier_mask = read_outlier_mask()        

        # Note : all the arrays after this step are masked arrays

        if smear:
            smear_removed_quads = perform_smear_removal(linearized_quads, int_time,
                                                        outlier_mask)
        else:
            smear_removed_quads = linearized_quads
            #----------------------------------------------------------------------

            ##########################CROSS-TALK REMOVAL###########################
            # Input : smear removed quads (all quads together)
            # Output : cross talk removed quads
        #print('Max. Val. Smeared  = ', np.max(smear_removed_quads))

        if cross_talk:
            cross_talk_removed_quads = remove_cross_talk(np.array(smear_removed_quads))
        else:
            cross_talk_removed_quads = smear_removed_quads
            #----------------------------------------------------------------------
        #print('Max. Val. Cross Talked = ', np.max(cross_talk_removed_quads))
        processed_image = create_final_image(np.array(cross_talk_removed_quads))
        processed_image = processed_image 
  
        #prnu_map = parse_prnu_file()
        prnu_map = read_prnu_files()
        prnu_spectrometer = create_final_image(np.array(prnu_map))
        prnu_spectrometer[prnu_spectrometer > 1.03] = 1.02
        prnu_spectrometer[prnu_spectrometer < 0.97] = 0.98
        outlier_spectrometer = create_final_image(np.array(outlier_mask))
        nx_quad, ny_quad = processed_image.shape
        outlier_spectrometer = np.reshape(outlier_spectrometer,
                                          (nx_quad*ny_quad, 1))
        #outlier_detectors = np.array(np.where([outlier_spectrometer == 1]))
        #print('outliers =', outlier_detectors.shape[1])
        outlier_spectrometer = np.reshape(outlier_spectrometer, (nx_quad, ny_quad))
        #create_image(outlier_spectrometer,'outliers = '+str(outlier_detectors.shape[1]),'b')
        processed_image = processed_image/(coadds*int_time)
        processed_image[processed_image >= 0.8*16383] = 0
        dark_current_each = np.median(processed_image, axis=1)
        all_dark_current.append(dark_current_each)
        
           
        #processed_image[processed_image==16383] = np.min(processed_image)
        #processed_image[processed_image<0] = 0
        #processed_image[processed_image==16383] = np.min(processed_image)
        #print(np.min(processed_image))
        
        text1 = telemetry_file_name+'.img' +', UV CCD \n' + \
                           '(Int. time:' + str(round(int_time,2))+ \
                           'ms, Co-adds:' +str(int(coadds))+\
                           ', FPE temp:'+ str(round(fpe_temp, 2))+'C, ' + \
                           ' FPA temp: ' + str(round(fpa_temp, 2))+'C)\n'+ \
                         'Mean DC = '+ str(round(np.mean(processed_image[0:1028:, :]),3)) +' DN/msec'
                           
                            
        processed_image_save = os.path.join(image_save_dir, 'processed_image')
        if not os.path.exists(processed_image_save):
            os.makedirs(processed_image_save)
        plot_save_dir = plot_dir + '/' + data_path + '.png'
        
        #processed_image = uniform_filter(processed_image, size=(5, 3), mode='mirror')
        dark_current = processed_image
        med_val = np.median(dark_current)
        dark_current[dark_current>0.15] = med_val
        dark_current[dark_current<-0.1]= med_val
       
        create_image(dark_current, text1, plot_save_dir)


       
    Dark_Current_name = plot_dir +'/'+ 'dark_current.csv'
    np.savetxt(Dark_Current_name, np.array(all_dark_current), delimiter=",")
    print('DONE!!')
def main():
    """
    This is the main function that does all the processing. It calls the required
    functions to process the raw image. Raw images are saved in .sav (IDL)
    format and read through this script.
    """
    data_dir = r'F:\TEMPO\Data\GroundTest\FPS\Spectrometer\Spectral_Range\740_nm_2'
    image_dir = os.path.join(data_dir, 'saved_quads')
    save_dir = os.path.join(image_dir, 'processed_image')
    image_save_dir = os.path.join(image_dir, 'saved_plots')
    telemetry_dir = os.path.join(data_dir, 'processed/h5')
    wavelen_dir = r'F:\TEMPO\Data\GroundTest\FPS\Spectrometer\Spectral_Range'
    wavelen_val = data_dir.strip('_')[-8:]
    wavelen_val = wavelen_val[0:6]

    # Standar image processing step. If value is assigned 1, the routine is called.
    # If value is zero, routine is skipped.
    linearity = 1
    smear = 1
    cross_talk = 1
    bandpass_val = []
    bandpass_val_norm = []
    normalization_factor_all = []
    if not os.path.exists(save_dir):
        os.makedirs(save_dir)

    data_path_all = sorted(
        [each for each in os.listdir(image_dir) if each.endswith('.sav')])

    wavelength = read_wavelength_file(wavelen_dir, wavelen_val)

    print('Total data = ', len(data_path_all))
    #    resultFile = open (os.path.join(save_dir,'file_name_all.csv'),'w')
    #
    #    for results in data_path_all:
    #         resultFile.write(results+"\n")
    #    resultFile.close()
    dark_current = calculate_dark_current(image_dir, coadds=1)
    count = 252
    for data_path in data_path_all[252:]:
        data_file = os.path.join(image_dir, data_path)
        print(data_path)
        #        cc
        telemetry_file_name = data_path.split('.')[0]
        #################### Telemetry Statistics###############################
        # Now lets parse the telemetry file to get the required information###
        #Note; The function can return the CCD temp and FPE temp. But these are
        #not needed on the image processing algorithm.. atleast for the time being

        telemetry_file = os.path.join(telemetry_dir,
                                      telemetry_file_name + '.h5')
        if os.path.exists(telemetry_file):
            coadds, int_time, fpe_temp, fpa_temp = parse_telemetry_file(
                telemetry_dir, telemetry_file_name)
            print('FPE Temp. = ', round(fpe_temp, 2), 'FPA Temp = ',
                  round(fpa_temp, 2))
        else:

            print('Telemetry file missing')
            coadds = 1
            int_time = 3500
            fpe_temp = 43
            fpa_temp = -21

    ##############Image Processing begins here##############################
        full_frame = read_idl_file(data_file)
        #check_image = create_image_active_region(full_frame)
        # raw_image = create_final_image(np.array(full_frame))
        # print('Max. Val. Raw = ', np.max(raw_image))
        quads = ['Quad A', 'Quad B', 'Quad C', 'Quad D']

        ##########################OFFSET REMOVAL###############################
        # Input : Full_Frame TEMPO IMAGE
        # Otput : Bias Subtracted Active Region. The active region dimensions are
        #now 1028*1024. 2 pixel lines from SMEAR overclocks are now used for
        #to store storage summation information
        # For dark data, only offset removal is needed to compute the dark current
        # For light data, additional processing such as linearity, smear and
        #cross talk is needed

        bias_removed_quads = perform_bias_subtraction(full_frame)

        #print('Max. Val. Offset Removed  = ', np.max(bias_removed_quads))

        text1 = telemetry_file_name+'.img' +' (Raw Data)\n Int. time:' + str(round(int_time, 1))+ \
                           'ms, Co-adds:' +str(int(coadds))+\
                           ', FPE temp:'+ str(round(fpe_temp, 1))+'C, ' + \
                           ' FPA temp: ' + str(round(fpa_temp, 1))+'C'
        # Now let us save the raw image
        raw_image_save = os.path.join(image_save_dir, 'raw_image')
        if not os.path.exists(raw_image_save):
            os.makedirs(raw_image_save)
        plot_save_dir = raw_image_save + '/' + data_path + '.png'
        #prnu_spectrometer[prnu_spectrometer < 0.9] = 0.9
        #prnu_spectrometer[prnu_spectrometer > 1.2] = 1.2
        #create_image(check_image/coadds, text1, plot_save_dir)
        #---------------------------------------------------------------------

        #########################NON LINEARITY CORRECTION#######################
        # Input : Bias Subtracted Active regions
        # Output : Linearized Active Region Quads
        if linearity:
            linearized_a = apply_linearity_correction(
                bias_removed_quads[0, :, :], quads[0], coadds)
            linearized_b = apply_linearity_correction(
                bias_removed_quads[1, :, :], quads[1], coadds)
            linearized_c = apply_linearity_correction(
                bias_removed_quads[2, :, :], quads[2], coadds)
            linearized_d = apply_linearity_correction(
                bias_removed_quads[3, :, :], quads[3], coadds)
            linearized_quads = np.array(
                [linearized_a, linearized_b, linearized_c, linearized_d])
            #print(np.mean(linearized_quads))
        else:
            linearized_quads = bias_removed_quads
            #----------------------------------------------------------------------
            ##########################SMEAR REMOVAL################################
            # Input : linearized quads ( all quads together)
            # Output : SMEAR offset corrected Quad

        #### lets' create the masked array with outlier mask################
        # The outlier mask is array of list of 4 quads.
        #print('Max. Val. Linearized  = ', np.max(linearized_quads))
        outlier_mask = read_outlier_mask()

        # Note : all the arrays after this step are masked arrays

        if smear:
            smear_removed_quads = perform_smear_removal(
                linearized_quads, int_time, outlier_mask)
        else:
            smear_removed_quads = linearized_quads
            #----------------------------------------------------------------------

            ##########################CROSS-TALK REMOVAL###########################
            # Input : smear removed quads (all quads together)
            # Output : cross talk removed quads
        #print('Max. Val. Smeared  = ', np.max(smear_removed_quads))

        if cross_talk:
            cross_talk_removed_quads = remove_cross_talk(
                np.array(smear_removed_quads))
        else:
            cross_talk_removed_quads = smear_removed_quads
            #----------------------------------------------------------------------
        #print('Max. Val. Cross Talked = ', np.max(cross_talk_removed_quads))
        processed_image = create_final_image(
            np.array(cross_talk_removed_quads))
        processed_image = processed_image - (dark_current)
        prnu_map = parse_prnu_file()  # BATC Map
        prnu_spectrometer = prnu_map

        #Uncomment the lines below if my PRNU is to be used.
        #prnu_map = read_prnu_files() # LaRC map
        #prnu_spectrometer = create_final_image(np.array(prnu_map))
        #prnu_spectrometer[prnu_spectrometer > 1.03] = 1.02
        #prnu_spectrometer[prnu_spectrometer < 0.97] = 0.98
        #create_image(prnu_map, 'TEMPO PRNU Map', 'a')
        #create_image(prnu_spectrometer, 'TEMPO PRNU Map', 'a')
        outlier_spectrometer = create_final_image(np.array(outlier_mask))

        #create_image(outlier_spectrometer,'Outliers = 504', 'b')
        #cc
        nx_quad, ny_quad = processed_image.shape
        outlier_spectrometer = np.reshape(outlier_spectrometer,
                                          (nx_quad * ny_quad, 1))
        #outlier_detectors = np.array(np.where([outlier_spectrometer == 1]))
        #print('outliers =', outlier_detectors.shape[1])
        outlier_spectrometer = np.reshape(outlier_spectrometer,
                                          (nx_quad, ny_quad))
        #create_image(outlier_spectrometer,'outliers = '+str(outlier_detectors.shape[1]),'b')
        #processed_image = processed_image/(prnu_spectrometer)
        processed_image = processed_image / (coadds * prnu_spectrometer)

        processed_image[processed_image >= 0.9 * 16383] = 16383

        processed_image_save = os.path.join(image_save_dir, 'processed_image')
        if not os.path.exists(processed_image_save):
            os.makedirs(processed_image_save)
        plot_save_dir = processed_image_save + '/' +str(wavelength[count])+ \
                       '_' + str(count)+'_nm' +'.png'

        #print('min_val=', np.min(processed_image))
        #processed_image = np.reshape(processed_image, (nx_quad*ny_quad, 1))
        #processed_image[outlier_spectrometer] = 16383

        #col_index = np.sort([randint(100, 2000) for p in range(0, 10)])
        row_mean = np.mean(processed_image, axis=1)
        max_row_each = np.where(
            row_mean == max(np.mean(processed_image, axis=1)))[0]
        #       max_row_each = max_row_each[0]
        max_row = max_row_each
        text1 = 'WL = ' + str(wavelength[count]) + 'nm'
        create_image(processed_image, max_row, text1, plot_save_dir)
        #        plt.plot(processed_image[2055, :],'r')
        #        plt.show()
        #        cc
        wavelen = wavelength[count]
        dframe_gauss = fit_gaussian_func(processed_image, wavelen)
        #max_row = 29 # for 297.8 nm
        #max_row = 1546 #for 640nm
        #max_row = 1369 #for 605nm
        #max_row = 1050 #for 605nm
        # max_row = 142 # for 320 nm
        # max_row = 319  # for 355 nm
        # max_row = 496  # for 390 nm
        # max_row = 673  # for 425 nm
        #max_row = 849  # for 460 nm
        #max_row = 992  # for 488.2 nm
        #max_row = 192 # for 330 nm
        #max_row = 91 # for 310 nm
        #max_row = 1724 # 675 nm
        #max_row = 2032 # for 736 nm
        max_row = 1989  # for 727.5 nm
        #check_val = 1392

        #subset_image = processed_image[max_row,  :]
        subset_image_normalized = processed_image[max_row - 18:max_row + 18, :]
        normalization_factor = np.sum(
            subset_image_normalized[subset_image_normalized < 16383])
        normalization_factor_all.append(normalization_factor)
        #normalization_factor_subset = np.sum(subset_image_normalized[subset_image_normalized<16383])
        #normalization_factor_all.append(normalization_factor_subset)
        bandpass_val.append(processed_image[max_row, :])
        bandpass_val_norm.append(
            (processed_image[max_row, :] / normalization_factor))
        count = count + 1
        #print(processed_image.shape)
    # cc


#        plt.plot(processed_image[max_row-4, :], 'purple', label='4 lines down')
#        plt.plot(processed_image[max_row-3, :], 'y-', label='3 lines down')
#        plt.plot(processed_image[max_row-2, :], 'g-', label='2 lines down')
#        plt.plot(processed_image[max_row-1, :], 'r-', label='1 line down')
#        plt.plot(processed_image[max_row, :], 'k-', label='max (296.54 nm), ' + 'row:' +str(max_row))
#        plt.plot(processed_image[max_row+1, :], 'm-', label='1 line up')
##        plt.plot(processed_image[max_row+2, :], 'b-', label='2 lines up')
##        plt.plot(processed_image[max_row+3, :], 'orange', label='3 lines up')
##        plt.plot(processed_image[max_row+4, :], 'cyan', label='4 lines up')
#        plt.title('Spatial Profile along illuminated rows (Laser WL: 727.5 nm)')
#        plt.ylabel('Counts (DN)')
#        plt.xlabel('Spatial Pixel Index')
#        plt.grid(True, linestyle=':')
#
#        #plt.yscale('log')
#        #plt.xlim(0, 2000)
#
#
#        #plt.legend(loc='best')
#        plt.show()
#        cc
#        #plt.ylim(2000, 40000)
#        plt.show(block=False)
#        #cc
#        plt.figure(3)
#        for i in col_index:
#
#            plt.plot(processed_image[:, i],
#                     'o--', label='Col. ' + str(i))
#            plt.legend(loc='best', ncol=4)
#        plt.grid(True, linestyle=':')
#        plt.title('Profile along Spectral direction (Spectral Bandpass 297 nm)')
#        plt.xlabel('Spectral Pixel Index')
#        plt.ylabel('Counts (DN)')
#        #plt.xlim(655, 675)
#        #plt.yscale('log')
#
#        plt.show()
#        cc

    save_bandpass = save_dir + '/' + 'bandpass_radiance_second_iteration.csv'
    save_bandpass_norm = save_dir + '/' + 'bandpass_radiance_normalized_second_iteration.csv'
    norm_factor = save_dir + '/' + 'integrated_radiance_second_iteration.csv'

    np.savetxt(save_bandpass, np.array(bandpass_val), delimiter=",")
    np.savetxt(save_bandpass_norm, np.array(bandpass_val_norm), delimiter=",")
    np.savetxt(norm_factor, np.array(normalization_factor_all), delimiter=",")
def main():
    """
    This is the main function that does all the processing. It calls the required
    functions to process the raw image. Raw images are saved in .sav (IDL)
    format and read through this script.
    """
    data_dir = r'F:\TEMPO\Data\GroundTest\FPS\Spectrometer\Spectral_Co_registration'
    image_dir = os.path.join(data_dir, 'saved_quads')
    save_dir = os.path.join(image_dir, 'processed_image')
    image_save_dir = os.path.join(image_dir, 'saved_plots')
    telemetry_dir = os.path.join(data_dir, 'processed/h5')
    linearity = 0  # option to turn on/off linearity correction
    smear = 1
    cross_talk = 1
    bandpass_val = []
    bandpass_val_norm = []
    normalization_factor_all = []
    dark_current = 0
    if not os.path.exists(save_dir):
        os.makedirs(save_dir)
#    data_path_all = sorted([each for each in os.listdir(image_dir)
#                            if each.startswith('2017_07_30_00_24_59_38064')
#                            and  each.endswith('.sav')])
#
    data_path_all = sorted(
        [each for each in os.listdir(image_dir) if each.endswith('.sav')])

    #    #dark_data = data_path_all[1:len(data_path_all):2]

    print('Total data = ', len(data_path_all))
    #    resultFile = open (os.path.join(save_dir,'file_name_all.csv'),'w')
    #
    #    for results in data_path_all[0:15]:
    #         resultFile.write(results+"\n")
    #    resultFile.close()
    if dark_current:
        dark_current = calculate_dark_current(image_dir, coadds=1)
    else:
        dark_current = 0

    count = 8
    peak_loc_all = []
    for data_path in data_path_all[8:]:
        data_file = os.path.join(image_dir, data_path)
        print(data_path)
        #        cc
        telemetry_file_name = data_path.split('.')[0]
        #################### Telemetry Statistics###############################
        # Now lets parse the telemetry file to get the required information###
        #Note; The function can return the CCD temp and FPE temp. But these are
        #not needed on the image processing algorithm.. atleast for the time being

        telemetry_file = os.path.join(telemetry_dir,
                                      telemetry_file_name + '.h5')

        if os.path.exists(telemetry_file):
            coadds, int_time, fpe_temp, fpa_temp = parse_telemetry_file(
                telemetry_dir, telemetry_file_name)
            print('FPE Temp. = ', round(fpe_temp, 2), 'FPA Temp = ',
                  round(fpa_temp, 2))
        else:

            print('Telemetry file missing')
            coadds = 10
            int_time = 6000
            fpe_temp = 43
            fpa_temp = -21

    ##############Image Processing begins here##############################
        full_frame = read_idl_file(data_file)
        check_image = create_image_active_region(full_frame)
        raw_image = create_final_image(np.array(full_frame))
        # print('Max. Val. Raw = ', np.max(raw_image))
        quads = ['Quad A', 'Quad B', 'Quad C', 'Quad D']

        ##########################OFFSET REMOVAL###############################
        # Input : Full_Frame TEMPO IMAGE
        # Otput : Bias Subtracted Active Region. The active region dimensions are
        #now 1028*1024. 2 pixel lines from SMEAR overclocks are now used for
        #to store storage summation information
        # For dark data, only offset removal is needed to compute the dark current
        # For light data, additional processing such as linearity, smear and
        #cross talk is needed

        bias_removed_quads = perform_bias_subtraction(full_frame)

        #print('Max. Val. Offset Removed  = ', np.max(bias_removed_quads))

        text1 = telemetry_file_name+'.img' +' (Raw Data)\n Int. time:' + str(round(int_time, 1))+ \
                           'ms, Co-adds:' +str(int(coadds))+\
                           ', FPE temp:'+ str(round(fpe_temp, 1))+'C, ' + \
                           ' FPA temp: ' + str(round(fpa_temp, 1))+'C'

        # Now let us save the raw image
        raw_image_save = os.path.join(image_save_dir, 'raw_image')
        if not os.path.exists(raw_image_save):
            os.makedirs(raw_image_save)
        plot_save_dir = raw_image_save + '/' + data_path + '.png'
        #prnu_spectrometer[prnu_spectrometer < 0.9] = 0.9
        #prnu_spectrometer[prnu_spectrometer > 1.2] = 1.2
        #print(np.min(check_image))
        #create_image(check_image, text1, plot_save_dir)
        #---------------------------------------------------------------------

        #########################NON LINEARITY CORRECTION#######################
        # Input : Bias Subtracted Active regions
        # Output : Linearized Active Region Quads
        if linearity:
            linearized_a = apply_linearity_correction(
                bias_removed_quads[0, :, :], quads[0], coadds)
            linearized_b = apply_linearity_correction(
                bias_removed_quads[1, :, :], quads[1], coadds)
            linearized_c = apply_linearity_correction(
                bias_removed_quads[2, :, :], quads[2], coadds)
            linearized_d = apply_linearity_correction(
                bias_removed_quads[3, :, :], quads[3], coadds)
            linearized_quads = np.array(
                [linearized_a, linearized_b, linearized_c, linearized_d])
            #print(np.mean(linearized_quads))
        else:
            linearized_quads = bias_removed_quads
            #----------------------------------------------------------------------
            ##########################SMEAR REMOVAL################################
            # Input : linearized quads ( all quads together)
            # Output : SMEAR offset corrected Quad

        #### lets' create the masked array with outlier mask################
        # The outlier mask is array of list of 4 quads.
        #print('Max. Val. Linearized  = ', np.max(linearized_quads))
        outlier_mask = read_outlier_mask()

        # Note : all the arrays after this step are masked arrays

        if smear:
            smear_removed_quads = perform_smear_removal(
                linearized_quads, int_time, outlier_mask)
        else:
            smear_removed_quads = linearized_quads
            #----------------------------------------------------------------------

            ##########################CROSS-TALK REMOVAL###########################
            # Input : smear removed quads (all quads together)
            # Output : cross talk removed quads
        #print('Max. Val. Smeared  = ', np.max(smear_removed_quads))

        if cross_talk:
            cross_talk_removed_quads = remove_cross_talk(
                np.array(smear_removed_quads))
        else:
            cross_talk_removed_quads = smear_removed_quads
            #----------------------------------------------------------------------
        #print('Max. Val. Cross Talked = ', np.max(cross_talk_removed_quads))
        processed_image = create_final_image(
            np.array(cross_talk_removed_quads))
        processed_image = processed_image - dark_current

        #prnu_map = parse_prnu_file()
        prnu_map = read_prnu_files()
        prnu_spectrometer = create_final_image(np.array(prnu_map))
        prnu_spectrometer[prnu_spectrometer > 1.03] = 1.02
        prnu_spectrometer[prnu_spectrometer < 0.97] = 0.98

        outlier_spectrometer = create_final_image(np.array(outlier_mask))

        nx_quad, ny_quad = processed_image.shape
        outlier_spectrometer = np.reshape(outlier_spectrometer,
                                          (nx_quad * ny_quad, 1))
        outlier_detectors = np.array(np.where([outlier_spectrometer == 1]))
        #print('outliers =', outlier_detectors.shape[1])
        outlier_spectrometer = np.reshape(outlier_spectrometer,
                                          (nx_quad, ny_quad))
        #create_image(outlier_spectrometer,'outliers = '+str(outlier_detectors.shape[1]),'b')
        processed_image = processed_image / (coadds * prnu_spectrometer)
        #print(np.min(processed_image))

        # Read the laser wavelength
        wavelen = np.loadtxt(os.path.join(image_dir, 'Wavelength.csv'),
                             delimiter=',')

        # Read the spectral index
        spec_index = np.loadtxt(os.path.join(image_dir, 'Spectral_column.csv'),
                                delimiter=',')
        wavelen = wavelen[count]
        spec_pix = int(spec_index[count])
        #print('Test Data= ',  wavelen, spec_pix)

        #processed_image[processed_image>=0.85*16383] = 16383
        text1 = telemetry_file_name+'.img' +' (Laser WL:' +str(wavelen) +' nm)\n Int. time:' + str(coadds)+ \
                           'ms, Co-adds:' +str(int(coadds))+\
                           ', FPE temp:'+ str(round(fpe_temp, 1))+'C, ' + \
                           ' FPA temp: ' + str(round(fpa_temp, ))+'C'
        processed_image_save = os.path.join(image_save_dir, 'processed_image')
        if not os.path.exists(processed_image_save):
            os.makedirs(processed_image_save)
        plot_save_dir = processed_image_save + '/' + data_path + '.png'
        #processed_image = uniform_filter(processed_image, size=(5, 3), mode='mirror')
        create_image(processed_image, text1, plot_save_dir)

        #        processed_image = processed_image[max_row-12: max_row+12, :]
        #        processed_image = uniform_filter(processed_image, size=(10, 4), mode='mirror')
        #        plt.plot(processed_image[:, 7]/np.max(processed_image[:, 7]),'r', label = 'Pin Hole 1(Sp. Index : 7)')
        #        plt.plot(processed_image[:, 514]/np.max(processed_image[:, 514]),'b', label = 'Pin Hole 2(Sp. Index : 514)')
        #        plt.plot(processed_image[:, 1023]/np.max(processed_image[:, 1023]),'g', label = 'Pin Hole 3(Sp. Index : 1023)')
        #        plt.plot(processed_image[:, 1532]/np.max(processed_image[:, 1532]),'m', label = 'Pin Hole 4(Sp. Index : 1532)')
        #        plt.plot(processed_image[:, 2041]/np.max(processed_image[:, 2041]),'k', label = 'Pin Hole 5(Sp. Index : 2041)')
        #        plt.title('Profile of a pinhole illumiation along spectral direction')
        #        plt.xlabel('Spectral Pixel Index')
        #        plt.ylabel('Normalized Counts')
        #        plt.legend(loc='best')
        #        plt.grid(True,linestyle=':')
        #        plt.show()
        #       # print(wavelen)

        peak_loc = fit_gaussian_func(processed_image, wavelen, spec_pix)
        peak_loc_all.append(peak_loc)
        count = count + 1


#        plt.plot(processed_image[spec_pix, :], 'purple', label='2 lines up')
#        plt.show()
#        p1 = processed_image[spec_pix, 1:9 ]
#        p2 = processed_image[spec_pix, 510:518]
#        p3 = processed_image[spec_pix, 1019:1027]
#        p4 = processed_image[spec_pix, 1528:1536]
#        p5 = processed_image[spec_pix, 2037:2045]
#        plt.plot(p1/np.max(p1), 'ro-', label='Pin Hole 1')
#        plt.plot(p2/np.max(p2), 'bo-', label='Pin Hole 1')
#        plt.plot(p3/np.max(p3), 'go-', label='Pin Hole 1')
#        plt.plot(p4/np.max(p4), 'mo-', label='Pin Hole 1')
#        plt.plot(p5/np.max(p5), 'ko-', label='Pin Hole 1')
#        plt.grid(True, linestyle  = ':')
#        plt.legend(loc='best')
#        plt.title('Spatial Profile of 5 pin hole')
#        plt.xlabel('Spatial Pixel Index')
#        plt.ylabel('Counts (DN)')
#        plt.show()
#

    final_data = os.path.join(image_dir, 'peak_loc_all.csv')

    np.savetxt(final_data, np.array(peak_loc_all), delimiter=",")
def main():
    """
    This is the main function that does all the processing. It calls the required
    functions to process the raw image. Raw images are saved in .sav (IDL)
    format and read through this script.
    """
    data_dir = r'F:\TEMPO\Data\GroundTest\FPS\Spectrometer\Photon_Transfer_TVAC'
    telemetry_dir = r'F:\TEMPO\Data\GroundTest\FPS\Spectrometer\2018.06.28'
    image_dir = os.path.join(data_dir, 'saved_quads')
    save_dir = os.path.join(image_dir, 'processed_image')
    image_save_dir = os.path.join(image_dir, 'saved_plots')
    telemetry_dir = os.path.join(telemetry_dir, 'processed/h5')
    linearity = 0  # option to turn on/off linearity correction
    smear = 1
    cross_talk = 1
    var_all = []
    signal_all = []
    all_image = []
    dark_current = 0
    if not os.path.exists(save_dir):
        os.makedirs(save_dir)


#    data_path_all = sorted([each for each in os.listdir(image_dir)
#                            if each.startswith('2017_07_30_00_24_59_38064')
#                            and  each.endswith('.sav')])
#
    data_path_all = sorted([
        each for each in os.listdir(image_dir)
        if each.endswith('2018_06_28_20_41_18_042102.img.sav')
    ])

    #    #dark_data = data_path_all[1:len(data_path_all):2]

    print('Total data = ', len(data_path_all))
    #    resultFile = open (os.path.join(save_dir,'file_name_all.csv'),'w')
    #
    #    for results in data_path_all[0:15]:
    #         resultFile.write(results+"\n")
    #    resultFile.close()
    if dark_current:
        dark_current = calculate_dark_current(image_dir, coadds=1)
    else:
        dark_current = 0

    count = 0
    #peak_loc_all = []
    for data_path in data_path_all:
        data_file = os.path.join(image_dir, data_path)
        print(data_path)
        #        cc
        telemetry_file_name = data_path.split('.')[0]
        #################### Telemetry Statistics###############################
        # Now lets parse the telemetry file to get the required information###
        #Note; The function can return the CCD temp and FPE temp. But these are
        #not needed on the image processing algorithm.. atleast for the time being

        telemetry_file = os.path.join(telemetry_dir,
                                      telemetry_file_name + '.h5')

        if os.path.exists(telemetry_file):
            coadds, int_time, fpe_temp, fpa_temp = parse_telemetry_file(
                telemetry_dir, telemetry_file_name)
            print('FPE Temp. = ', round(fpe_temp, 2), 'FPA Temp = ',
                  round(fpa_temp, 2))
            print('Integ. Time =', int_time)

        else:

            print('Telemetry file missing')
            coadds = 10
            int_time = 6000
            fpe_temp = 43
            fpa_temp = -21.5

    ##############Image Processing begins here##############################
        full_frame = read_idl_file(data_file)
        #check_image = create_image_active_region(full_frame)
        #raw_image = create_final_image(np.array(full_frame))
        # print('Max. Val. Raw = ', np.max(raw_image))
        quads = ['Quad A', 'Quad B', 'Quad C', 'Quad D']

        ##########################OFFSET REMOVAL###############################
        # Input : Full_Frame TEMPO IMAGE
        # Otput : Bias Subtracted Active Region. The active region dimensions are
        #now 1028*1024. 2 pixel lines from SMEAR overclocks are now used for
        #to store storage summation information
        # For dark data, only offset removal is needed to compute the dark current
        # For light data, additional processing such as linearity, smear and
        #cross talk is needed

        bias_removed_quads = perform_bias_subtraction(full_frame)

        #print('Max. Val. Offset Removed  = ', np.max(bias_removed_quads))

        text1 = telemetry_file_name+'.img' +' (Raw Data)\n Int. time:' + str(round(int_time, 3))+ \
                           'ms, Co-adds:' +str(int(coadds))+\
                           ', FPE temp:'+ str(round(fpe_temp, 1))+'C, ' + \
                           ' FPA temp: ' + str(round(fpa_temp, 1))+'C'

        # Now let us save the raw image
        raw_image_save = os.path.join(image_save_dir, 'raw_image')
        if not os.path.exists(raw_image_save):
            os.makedirs(raw_image_save)
        plot_save_dir = raw_image_save + '/' + data_path + '.png'
        #prnu_spectrometer[prnu_spectrometer < 0.9] = 0.9
        #prnu_spectrometer[prnu_spectrometer > 1.2] = 1.2
        #print(np.min(check_image))
        #create_image(check_image, text1, plot_save_dir)
        #---------------------------------------------------------------------

        #########################NON LINEARITY CORRECTION#######################
        # Input : Bias Subtracted Active regions
        # Output : Linearized Active Region Quads
        if linearity:
            linearized_a = apply_linearity_correction(
                bias_removed_quads[0, :, :], quads[0], coadds)
            linearized_b = apply_linearity_correction(
                bias_removed_quads[1, :, :], quads[1], coadds)
            linearized_c = apply_linearity_correction(
                bias_removed_quads[2, :, :], quads[2], coadds)
            linearized_d = apply_linearity_correction(
                bias_removed_quads[3, :, :], quads[3], coadds)
            linearized_quads = np.array(
                [linearized_a, linearized_b, linearized_c, linearized_d])
            #print(np.mean(linearized_quads))
        else:
            linearized_quads = bias_removed_quads
            #----------------------------------------------------------------------
            ##########################SMEAR REMOVAL################################
            # Input : linearized quads ( all quads together)
            # Output : SMEAR offset corrected Quad

        #### lets' create the masked array with outlier mask################
        # The outlier mask is array of list of 4 quads.
        #print('Max. Val. Linearized  = ', np.max(linearized_quads))
        outlier_mask = read_outlier_mask()

        # Note : all the arrays after this step are masked arrays

        if smear:
            smear_removed_quads = perform_smear_removal(
                linearized_quads, int_time, outlier_mask)
        else:
            smear_removed_quads = linearized_quads
            #----------------------------------------------------------------------

            ##########################CROSS-TALK REMOVAL###########################
            # Input : smear removed quads (all quads together)
            # Output : cross talk removed quads
        #print('Max. Val. Smeared  = ', np.max(smear_removed_quads))

        if cross_talk:
            cross_talk_removed_quads = remove_cross_talk(
                np.array(smear_removed_quads))
        else:
            cross_talk_removed_quads = smear_removed_quads
            #----------------------------------------------------------------------
        #print('Max. Val. Cross Talked = ', np.max(cross_talk_removed_quads))
        processed_image = create_final_image(
            np.array(cross_talk_removed_quads))
        processed_image = processed_image - dark_current

        #prnu_map = parse_prnu_file()
        prnu_map = read_prnu_files()
        prnu_spectrometer = create_final_image(np.array(prnu_map))
        prnu_spectrometer[prnu_spectrometer > 1.03] = 1.02
        prnu_spectrometer[prnu_spectrometer < 0.97] = 0.98

        outlier_spectrometer = create_final_image(np.array(outlier_mask))

        nx_quad, ny_quad = processed_image.shape
        outlier_spectrometer = np.reshape(outlier_spectrometer,
                                          (nx_quad * ny_quad, 1))
        #outlier_detectors = np.array(np.where([outlier_spectrometer == 1]))
        #print('outliers =', outlier_detectors.shape[1])
        outlier_spectrometer = np.reshape(outlier_spectrometer,
                                          (nx_quad, ny_quad))
        #create_image(outlier_spectrometer,'outliers = '+str(outlier_detectors.shape[1]),'b')
        processed_image = processed_image / (coadds * prnu_spectrometer)
        processed_image[processed_image >= 0.85 * 16383] = 16383
        #print(np.min(processed_image))

        #processed_image[processed_image>=0.85*16383] = 16383
        text1 = telemetry_file_name+'.img, ' + 'Int. time:' + str(round(int_time,2))+ \
                           'ms\n Co-adds:' +str(int(coadds))+\
                           ', FPE temp:'+ str(round(fpe_temp, 1))+'C, ' + \
                           ' FPA temp: ' + str(round(fpa_temp, ))+'C'
        processed_image_save = os.path.join(image_save_dir, 'processed_image')
        if not os.path.exists(processed_image_save):
            os.makedirs(processed_image_save)
        plot_save_dir = processed_image_save + '/' + data_path + '.png'
        #processed_image = uniform_filter(processed_image, size=(5, 3), mode='mirror')
        #create_image(processed_image, text1, plot_save_dir)
        all_image.append(processed_image)
        count = count + 1

    var_all = np.var(all_image, axis=0)
    signal_all = np.median(all_image, axis=0)
Example #11
0
def main():
    """
    This is the main function that does all the processing. It calls the required
    functions to process the raw image. Raw images are saved in .sav (IDL)
    format and read through this script.
    """
    data_dir = r'F:\TEMPO\Data\GroundTest\FPS\Spectrometer\Read_Noise'
    plot_dir = os.path.join(data_dir, 'Read_Noise_image')
    image_dir = os.path.join(data_dir, 'saved_quads')
    image_save_dir = os.path.join(image_dir, 'saved_plots')
    telemetry_dir = r'F:\TEMPO\Data\GroundTest\FPS\Spectrometer\2018.06.28'
    telemetry_dir = os.path.join(telemetry_dir, 'processed/h5')
    linearity = 0  # option to turn on/off linearity correction
    smear = 1
    cross_talk = 1
    if not os.path.exists(plot_dir):
        os.makedirs(plot_dir)


#    data_path_all = sorted([each for each in os.listdir(image_dir)
#                            if each.startswith('2017_07_30_00_24_59_38064')
#                            and  each.endswith('.sav')])
#
    data_path_all = sorted(
        [each for each in os.listdir(image_dir) if each.endswith('.sav')])
    print(len(data_path_all))

    print('Total data = ', len(data_path_all))
    count = 0
    all_dark_current = []

    for data_path in data_path_all:
        data_file = os.path.join(image_dir, data_path)
        print(data_path)
        #        cc
        telemetry_file_name = data_path.split('.')[0]
        #################### Telemetry Statistics###############################
        # Now lets parse the telemetry file to get the required information###
        #Note; The function can return the CCD temp and FPE temp. But these are
        #not needed on the image processing algorithm.. atleast for the time being

        telemetry_file = os.path.join(telemetry_dir,
                                      telemetry_file_name + '.h5')

        if os.path.exists(telemetry_file):
            coadds, int_time, fpe_temp, fpa_temp = parse_telemetry_file(
                telemetry_dir, telemetry_file_name)
            print('FPE Temp. = ', round(fpe_temp, 2), 'FPA Temp = ',
                  round(fpa_temp, 2))
            print(int_time)
            print(coadds)
        else:

            print('Telemetry file missing')
            coadds = 10
            int_time = 160
            fpe_temp = 35.8
            fpa_temp = 11.9

    ##############Image Processing begins here##############################
    #CC
        full_frame = read_idl_file(data_file)
        #check_image = create_image_active_region(full_frame)
        #raw_image = create_final_image(np.array(full_frame))
        #create_image(raw_image, 'a', 'b')
        # print('Max. Val. Raw = ', np.max(raw_image))
        quads = ['Quad A', 'Quad B', 'Quad C', 'Quad D']

        ##########################OFFSET REMOVAL###############################
        # Input : Full_Frame TEMPO IMAGE
        # Otput : Bias Subtracted Active Region. The active region dimensions are
        #now 1028*1024. 2 pixel lines from SMEAR overclocks are now used for
        #to store storage summation information
        # For dark data, only offset removal is needed to compute the dark current
        # For light data, additional processing such as linearity, smear and
        #cross talk is needed

        bias_removed_quads = perform_bias_subtraction(full_frame)
        #raw_image = create_final_image(np.array(bias_removed_quads))
        #create_image(raw_image/coadds, 'a', 'b')

        #print('Max. Val. Offset Removed  = ', np.max(bias_removed_quads))

        text1 = telemetry_file_name + '.img' + ' (Raw Data)\n Int. time:' + str(round(int_time, 1)) + \
                           'ms, Co-adds:' + str(int(coadds)) + \
                           ', FPE temp:' + str(round(fpe_temp, 1)) + 'C, ' + \
                           ' FPA temp: ' + str(round(fpa_temp, 1)) + 'C'

        # Now let us save the raw image
        raw_image_save = os.path.join(image_save_dir, 'raw_image')
        if not os.path.exists(raw_image_save):
            os.makedirs(raw_image_save)
        plot_save_dir = raw_image_save + '/' + data_path + '.png'
        #prnu_spectrometer[prnu_spectrometer < 0.9] = 0.9
        #prnu_spectrometer[prnu_spectrometer > 1.2] = 1.2
        #print(np.min(check_image))
        #create_image(check_image, text1, plot_save_dir)
        #---------------------------------------------------------------------

        #########################NON LINEARITY CORRECTION#######################
        # Input : Bias Subtracted Active regions
        # Output : Linearized Active Region Quads
        if linearity:
            linearized_a = apply_linearity_correction(
                bias_removed_quads[0, :, :], quads[0], coadds)
            linearized_b = apply_linearity_correction(
                bias_removed_quads[1, :, :], quads[1], coadds)
            linearized_c = apply_linearity_correction(
                bias_removed_quads[2, :, :], quads[2], coadds)
            linearized_d = apply_linearity_correction(
                bias_removed_quads[3, :, :], quads[3], coadds)
            linearized_quads = np.array(
                [linearized_a, linearized_b, linearized_c, linearized_d])
            #print(np.mean(linearized_quads))
        else:
            linearized_quads = bias_removed_quads
            #----------------------------------------------------------------------
            ##########################SMEAR REMOVAL################################
            # Input : linearized quads ( all quads together)
            # Output : SMEAR offset corrected Quad

        #### lets' create the masked array with outlier mask################
        # The outlier mask is array of list of 4 quads.
        #print('Max. Val. Linearized  = ', np.max(linearized_quads))
        outlier_mask = read_outlier_mask()

        # Note : all the arrays after this step are masked arrays

        if smear:
            smear_removed_quads = perform_smear_removal(
                linearized_quads, int_time, outlier_mask)
        else:
            smear_removed_quads = linearized_quads
            #----------------------------------------------------------------------

            ##########################CROSS-TALK REMOVAL###########################
            # Input : smear removed quads (all quads together)
            # Output : cross talk removed quads
        #print('Max. Val. Smeared  = ', np.max(smear_removed_quads))

        if cross_talk:
            cross_talk_removed_quads = remove_cross_talk(
                np.array(smear_removed_quads))
        else:
            cross_talk_removed_quads = smear_removed_quads
            #----------------------------------------------------------------------
        #print('Max. Val. Cross Talked = ', np.max(cross_talk_removed_quads))
        processed_image = create_final_image(
            np.array(cross_talk_removed_quads))
        #processed_image = processed_image

        #prnu_map = parse_prnu_file()
        prnu_map = read_prnu_files()
        prnu_spectrometer = create_final_image(np.array(prnu_map))
        prnu_spectrometer[prnu_spectrometer > 1.03] = 1.02
        prnu_spectrometer[prnu_spectrometer < 0.97] = 0.98
        outlier_spectrometer = create_final_image(np.array(outlier_mask))
        nx_quad, ny_quad = processed_image.shape
        outlier_spectrometer = np.reshape(outlier_spectrometer,
                                          (nx_quad * ny_quad, 1))
        #outlier_detectors = np.array(np.where([outlier_spectrometer == 1]))
        #print('outliers =', outlier_detectors.shape[1])
        outlier_spectrometer = np.reshape(outlier_spectrometer,
                                          (nx_quad, ny_quad))
        #create_image(outlier_spectrometer,'outliers = '+str(outlier_detectors.shape[1]),'b')
        processed_image = processed_image
        processed_image[processed_image >= 0.8 * 16383] = 0
        dark_current_each = processed_image
        #print(dark_current_each)

        all_dark_current.append(dark_current_each)

        #processed_image[processed_image==16383] = np.min(processed_image)
        #processed_image[processed_image<0] = 0
        #processed_image[processed_image==16383] = np.min(processed_image)
        #print(np.min(processed_image))

        text1 = telemetry_file_name+'.img' +', UV CCD \n' + \
                           '(Int. time:' + str(round(int_time,2))+ \
                           'ms, Co-adds:' +str(int(coadds))+\
                           ', FPE temp:'+ str(round(fpe_temp, 2))+'C, ' + \
                           ' FPA temp: ' + str(round(fpa_temp, 2))+'C)'

        processed_image_save = os.path.join(image_save_dir, 'processed_image')
        if not os.path.exists(processed_image_save):
            os.makedirs(processed_image_save)
        plot_save_dir = plot_dir + '/' + data_path + '.png'
        #create_image(processed_image, text1, plot_save_dir)
        #processed_image = uniform_filter(processed_image, size=(5, 3), mode='mirror')
        #dark_current = processed_image
        #med_val = np.median(dark_current)
        #dark_current[dark_current>0.15] = med_val
        #dark_current[dark_current<-0.1]= med_val

    all_dark_current = np.array(all_dark_current)
    Dark_Current_name = plot_dir + '/' + 'read_noise.csv'
    np.savetxt(Dark_Current_name, np.array(all_dark_current), delimiter=",")

    #print(all_dark_current.shape)

    read_Noise = np.var(all_dark_current, axis=0)
    # print('read_noise_dimes = ', read_Noise.shape)
    create_image(read_Noise, text1, plot_save_dir)

    read_Noise = filter_outlier_median(read_Noise)
    label2 = 'mean variance = ' + str(round(np.mean(read_Noise), 3))
    #nx_quad, ny_quad = read_Noise.shape
    plt.figure(figsize=(10, 8))
    plt.hist(read_Noise, 60, normed=0, facecolor='red', alpha=1, label=label2)
    #title = 'Histogram of Variance of photon transfer sequence at 0 ms'
    plt.title(
        'Histogram of variance of processed Laser Image\n Int., time = 0, Num. images= 100',
        fontsize=14,
        fontweight="bold")
    plt.grid(True, linestyle=':')
    plt.xlabel('Temporal Noise (Variance)', fontsize=14, fontweight="bold")
    plt.ylabel('Frequency', fontsize=14, fontweight="bold")
    plt.legend(loc='best')
    ax = plt.gca()
    #plt.xlim(0, 300)
    ax.yaxis.set_major_formatter(FormatStrFormatter('%.0f'))
    plt.show()

    plt.savefig(plot_dir + '/' + '_Read_Noise.png', dpi=100)
    plt.close('all')

    print('DONE!!')