def main():
"""
This is the main function that does all the processing. It calls the required
functions to process the raw image. Raw images are saved in .sav (IDL)
format and read through this script.
"""
data_dir = r'F:\TEMPO\Data\GroundTest\FPS\Spectrometer\Spectral_Smile\Flight_Slight'
image_dir = os.path.join(data_dir, 'saved_quads')
save_dir = os.path.join(image_dir, 'processed_image')
image_save_dir = os.path.join(image_dir, 'saved_plots')
telemetry_dir = os.path.join(data_dir, 'processed/h5')
linearity = 0 # option to turn on/off linearity correction
smear = 1
cross_talk = 1
bandpass_val = []
bandpass_val_norm = []
normalization_factor_all = []
dark_current = 0
if not os.path.exists(save_dir):
os.makedirs(save_dir)
# data_path_all = sorted([each for each in os.listdir(image_dir)
# if each.startswith('2017_07_30_00_24_59_38064')
# and each.endswith('.sav')])
#
data_path_all = sorted(
[each for each in os.listdir(image_dir) if each.endswith('.sav')])
#
# #dark_data = data_path_all[1:len(data_path_all):2]
print('Total data = ', len(data_path_all))
# resultFile = open (os.path.join(save_dir,'file_name_all.csv'),'w')
#
# for results in data_path_all[0:15]:
# resultFile.write(results+"\n")
# resultFile.close()
if dark_current:
dark_current = calculate_dark_current(image_dir, coadds=1)
else:
dark_current = 0
for data_path in data_path_all[7:]:
data_file = os.path.join(image_dir, data_path)
print(data_path)
# cc
telemetry_file_name = data_path.split('.')[0]
#################### Telemetry Statistics###############################
# Now lets parse the telemetry file to get the required information###
#Note; The function can return the CCD temp and FPE temp. But these are
#not needed on the image processing algorithm.. atleast for the time being
telemetry_file = os.path.join(telemetry_dir,
telemetry_file_name + '.h5')
if os.path.exists(telemetry_file):
coadds, int_time, fpe_temp, fpa_temp = parse_telemetry_file(
telemetry_dir, telemetry_file_name)
print('FPE Temp. = ', round(fpe_temp, 2), 'FPA Temp = ',
round(fpa_temp, 2))
else:
print('Telemetry file missing')
coadds = 10
int_time = 6000
fpe_temp = 43
fpa_temp = -21
##############Image Processing begins here##############################
full_frame = read_idl_file(data_file)
check_image = create_image_active_region(full_frame)
raw_image = create_final_image(np.array(full_frame))
# print('Max. Val. Raw = ', np.max(raw_image))
quads = ['Quad A', 'Quad B', 'Quad C', 'Quad D']
##########################OFFSET REMOVAL###############################
# Input : Full_Frame TEMPO IMAGE
# Otput : Bias Subtracted Active Region. The active region dimensions are
#now 1028*1024. 2 pixel lines from SMEAR overclocks are now used for
#to store storage summation information
# For dark data, only offset removal is needed to compute the dark current
# For light data, additional processing such as linearity, smear and
#cross talk is needed
bias_removed_quads = perform_bias_subtraction(full_frame)
#print('Max. Val. Offset Removed = ', np.max(bias_removed_quads))
text1 = telemetry_file_name+'.img' +' (Raw Data)\n Int. time:' + str(round(int_time, 1))+ \
'ms, Co-adds:' +str(int(coadds))+\
', FPE temp:'+ str(round(fpe_temp, 1))+'C, ' + \
' FPA temp: ' + str(round(fpa_temp, 1))+'C'
# Now let us save the raw image
raw_image_save = os.path.join(image_save_dir, 'raw_image')
if not os.path.exists(raw_image_save):
os.makedirs(raw_image_save)
plot_save_dir = raw_image_save + '/' + data_path + '.png'
#prnu_spectrometer[prnu_spectrometer < 0.9] = 0.9
#prnu_spectrometer[prnu_spectrometer > 1.2] = 1.2
#print(np.min(check_image))
#create_image(check_image, text1, plot_save_dir)
#---------------------------------------------------------------------
#########################NON LINEARITY CORRECTION#######################
# Input : Bias Subtracted Active regions
# Output : Linearized Active Region Quads
if linearity:
linearized_a = apply_linearity_correction(
bias_removed_quads[0, :, :], quads[0], coadds)
linearized_b = apply_linearity_correction(
bias_removed_quads[1, :, :], quads[1], coadds)
linearized_c = apply_linearity_correction(
bias_removed_quads[2, :, :], quads[2], coadds)
linearized_d = apply_linearity_correction(
bias_removed_quads[3, :, :], quads[3], coadds)
linearized_quads = np.array(
[linearized_a, linearized_b, linearized_c, linearized_d])
#print(np.mean(linearized_quads))
else:
linearized_quads = bias_removed_quads
#----------------------------------------------------------------------
##########################SMEAR REMOVAL################################
# Input : linearized quads ( all quads together)
# Output : SMEAR offset corrected Quad
#### lets' create the masked array with outlier mask################
# The outlier mask is array of list of 4 quads.
#print('Max. Val. Linearized = ', np.max(linearized_quads))
outlier_mask = read_outlier_mask()
# Note : all the arrays after this step are masked arrays
if smear:
smear_removed_quads = perform_smear_removal(
linearized_quads, int_time, outlier_mask)
else:
smear_removed_quads = linearized_quads
#----------------------------------------------------------------------
##########################CROSS-TALK REMOVAL###########################
# Input : smear removed quads (all quads together)
# Output : cross talk removed quads
#print('Max. Val. Smeared = ', np.max(smear_removed_quads))
if cross_talk:
cross_talk_removed_quads = remove_cross_talk(
np.array(smear_removed_quads))
else:
cross_talk_removed_quads = smear_removed_quads
#----------------------------------------------------------------------
#print('Max. Val. Cross Talked = ', np.max(cross_talk_removed_quads))
processed_image = create_final_image(
np.array(cross_talk_removed_quads))
processed_image = processed_image - dark_current
#prnu_map = parse_prnu_file()
prnu_map = read_prnu_files()
prnu_spectrometer = create_final_image(np.array(prnu_map))
prnu_spectrometer[prnu_spectrometer > 1.03] = 1.02
prnu_spectrometer[prnu_spectrometer < 0.97] = 0.98
outlier_spectrometer = create_final_image(np.array(outlier_mask))
nx_quad, ny_quad = processed_image.shape
outlier_spectrometer = np.reshape(outlier_spectrometer,
(nx_quad * ny_quad, 1))
outlier_detectors = np.array(np.where([outlier_spectrometer == 1]))
#print('outliers =', outlier_detectors.shape[1])
outlier_spectrometer = np.reshape(outlier_spectrometer,
(nx_quad, ny_quad))
#create_image(outlier_spectrometer,'outliers = '+str(outlier_detectors.shape[1]),'b')
processed_image = processed_image / (prnu_spectrometer)
#print(np.min(processed_image))
#processed_image[processed_image>=0.85*16383] = 16383
text1 = telemetry_file_name+'.img' +' (Spectral Bandpass 675 nm)\n Int. time:' + str(round(int_time, 1))+ \
'ms, Co-adds:' +str(int(coadds))+\
', FPE temp:'+ str(round(fpe_temp, 1))+'C, ' + \
' FPA temp: ' + str(round(fpa_temp, ))+'C'
processed_image_save = os.path.join(image_save_dir, 'processed_image')
if not os.path.exists(processed_image_save):
os.makedirs(processed_image_save)
plot_save_dir = processed_image_save + '/' + data_path + '.png'
processed_image = uniform_filter(processed_image,
size=3,
mode='constant')
#create_image(processed_image, text1, plot_save_dir)
row_mean = np.mean(processed_image, axis=1)
max_row_each = np.where(
row_mean == max(np.mean(processed_image, axis=1)))[0]
max_row_each = max_row_each[0]
print(max_row_each)
#
#
# #max_row = 29 # for 297.8 nm
# #max_row = 1546 #for 640nm
# #max_row = 1369 #for 605nm
# #max_row = 1050 #for 605nm
# #max_row = 142 # for 320 nm
#
# #max_row = 91 # for 310 nm
# max_row = 15 # 675 nm
# #max_row = 2032 # for 736 nm
# #check_val = 1392
col_index = np.sort([randint(100, 2000) for p in range(0, 10)])
for i in col_index:
plt.plot(processed_image[:, i], 'o--', label='Col. ' + str(i))
plt.legend(loc='best', ncol=4)
plt.grid(True, linestyle=':')
plt.title(
'Profile along Spectral direction (Spectral Bandpass 297 nm)')
plt.xlabel('Spectral Pixel Index')
plt.ylabel('Counts (DN)')
#plt.xlim(655, 675)
#plt.yscale('log')
plt.xlim(max_row_each - 10, max_row_each + 10)
plt.show()
cc
def main():
"""
This is the main function that does all the processing. It calls the required
functions to process the raw image. Raw images are saved in .sav (IDL)
format and read through this script.
"""
data_dir = r'F:\TEMPO\Data\GroundTest\FPS\Spectrometer\Spectral_Smile\Flight_Slight'
image_dir = os.path.join(data_dir, 'saved_quads')
save_dir = os.path.join(image_dir, 'processed_image')
image_save_dir = os.path.join(image_dir, 'saved_plots')
telemetry_dir = os.path.join(data_dir, 'processed/h5')
linearity = 0 # option to turn on/off linearity correction
smear = 1
cross_talk = 1
bandpass_val = []
bandpass_val_norm = []
normalization_factor_all = []
dark_current = 0
if not os.path.exists(save_dir):
os.makedirs(save_dir)
# data_path_all = sorted([each for each in os.listdir(image_dir)
# if each.startswith('2017_07_30_00_24_59_38064')
# and each.endswith('.sav')])
#
data_path_all = sorted(
[each for each in os.listdir(image_dir) if each.endswith('.sav')])
#
# #dark_data = data_path_all[1:len(data_path_all):2]
print('Total data = ', len(data_path_all))
# resultFile = open (os.path.join(save_dir,'file_name_all.csv'),'w')
#
# for results in data_path_all[0:15]:
# resultFile.write(results+"\n")
# resultFile.close()
if dark_current:
dark_current = calculate_dark_current(image_dir, coadds=1)
else:
dark_current = 0
count = 0
spectral_alingn_unct = []
for data_path in data_path_all[0:16]:
data_file = os.path.join(image_dir, data_path)
print(data_path)
# cc
telemetry_file_name = data_path.split('.')[0]
#################### Telemetry Statistics###############################
# Now lets parse the telemetry file to get the required information###
#Note; The function can return the CCD temp and FPE temp. But these are
#not needed on the image processing algorithm.. atleast for the time being
telemetry_file = os.path.join(telemetry_dir,
telemetry_file_name + '.h5')
if os.path.exists(telemetry_file):
coadds, int_time, fpe_temp, fpa_temp = parse_telemetry_file(
telemetry_dir, telemetry_file_name)
print('FPE Temp. = ', round(fpe_temp, 2), 'FPA Temp = ',
round(fpa_temp, 2))
else:
print('Telemetry file missing')
coadds = 10
int_time = 6000
fpe_temp = 43
fpa_temp = -21
##############Image Processing begins here##############################
full_frame = read_idl_file(data_file)
check_image = create_image_active_region(full_frame)
raw_image = create_final_image(np.array(full_frame))
# print('Max. Val. Raw = ', np.max(raw_image))
quads = ['Quad A', 'Quad B', 'Quad C', 'Quad D']
##########################OFFSET REMOVAL###############################
# Input : Full_Frame TEMPO IMAGE
# Otput : Bias Subtracted Active Region. The active region dimensions are
#now 1028*1024. 2 pixel lines from SMEAR overclocks are now used for
#to store storage summation information
# For dark data, only offset removal is needed to compute the dark current
# For light data, additional processing such as linearity, smear and
#cross talk is needed
bias_removed_quads = perform_bias_subtraction(full_frame)
#print('Max. Val. Offset Removed = ', np.max(bias_removed_quads))
text1 = telemetry_file_name+'.img' +' (Raw Data)\n Int. time:' + str(round(int_time, 1))+ \
'ms, Co-adds:' +str(int(coadds))+\
', FPE temp:'+ str(round(fpe_temp, 1))+'C, ' + \
' FPA temp: ' + str(round(fpa_temp, 1))+'C'
# Now let us save the raw image
raw_image_save = os.path.join(image_save_dir, 'raw_image')
if not os.path.exists(raw_image_save):
os.makedirs(raw_image_save)
plot_save_dir = raw_image_save + '/' + data_path + '.png'
#prnu_spectrometer[prnu_spectrometer < 0.9] = 0.9
#prnu_spectrometer[prnu_spectrometer > 1.2] = 1.2
#print(np.min(check_image))
#create_image(check_image, text1, plot_save_dir)
#---------------------------------------------------------------------
#########################NON LINEARITY CORRECTION#######################
# Input : Bias Subtracted Active regions
# Output : Linearized Active Region Quads
if linearity:
linearized_a = apply_linearity_correction(
bias_removed_quads[0, :, :], quads[0], coadds)
linearized_b = apply_linearity_correction(
bias_removed_quads[1, :, :], quads[1], coadds)
linearized_c = apply_linearity_correction(
bias_removed_quads[2, :, :], quads[2], coadds)
linearized_d = apply_linearity_correction(
bias_removed_quads[3, :, :], quads[3], coadds)
linearized_quads = np.array(
[linearized_a, linearized_b, linearized_c, linearized_d])
#print(np.mean(linearized_quads))
else:
linearized_quads = bias_removed_quads
#----------------------------------------------------------------------
##########################SMEAR REMOVAL################################
# Input : linearized quads ( all quads together)
# Output : SMEAR offset corrected Quad
#### lets' create the masked array with outlier mask################
# The outlier mask is array of list of 4 quads.
#print('Max. Val. Linearized = ', np.max(linearized_quads))
outlier_mask = read_outlier_mask()
# Note : all the arrays after this step are masked arrays
if smear:
smear_removed_quads = perform_smear_removal(
linearized_quads, int_time, outlier_mask)
else:
smear_removed_quads = linearized_quads
#----------------------------------------------------------------------
##########################CROSS-TALK REMOVAL###########################
# Input : smear removed quads (all quads together)
# Output : cross talk removed quads
#print('Max. Val. Smeared = ', np.max(smear_removed_quads))
if cross_talk:
cross_talk_removed_quads = remove_cross_talk(
np.array(smear_removed_quads))
else:
cross_talk_removed_quads = smear_removed_quads
#----------------------------------------------------------------------
#print('Max. Val. Cross Talked = ', np.max(cross_talk_removed_quads))
processed_image = create_final_image(
np.array(cross_talk_removed_quads))
processed_image = processed_image - dark_current
#prnu_map = parse_prnu_file()
prnu_map = read_prnu_files()
prnu_spectrometer = create_final_image(np.array(prnu_map))
prnu_spectrometer[prnu_spectrometer > 1.03] = 1.02
prnu_spectrometer[prnu_spectrometer < 0.97] = 0.98
outlier_spectrometer = create_final_image(np.array(outlier_mask))
nx_quad, ny_quad = processed_image.shape
outlier_spectrometer = np.reshape(outlier_spectrometer,
(nx_quad * ny_quad, 1))
outlier_detectors = np.array(np.where([outlier_spectrometer == 1]))
#print('outliers =', outlier_detectors.shape[1])
outlier_spectrometer = np.reshape(outlier_spectrometer,
(nx_quad, ny_quad))
#create_image(outlier_spectrometer,'outliers = '+str(outlier_detectors.shape[1]),'b')
processed_image = processed_image / (coadds * prnu_spectrometer)
#print(np.min(processed_image))
wavelen = np.loadtxt(os.path.join(image_dir, 'Wavelength.csv'),
delimiter=',')
wavelen = wavelen[count]
#processed_image[processed_image>=0.85*16383] = 16383
text1 = telemetry_file_name+'.img' +' (Laser WL:' + str(wavelen)+' nm) \n Int. time:' + '250 '+ \
'ms, Co-adds:' +str(int(coadds))+\
', FPE temp:'+ str(round(fpe_temp, 1))+'C, ' + \
' FPA temp: ' + str(round(fpa_temp, ))+'C'
processed_image_save = os.path.join(image_save_dir, 'processed_image')
if not os.path.exists(processed_image_save):
os.makedirs(processed_image_save)
plot_save_dir = processed_image_save + '/' + data_path + '.png'
#processed_image = uniform_filter(processed_image, size=(5, 3), mode='mirror')
create_image(processed_image, text1, plot_save_dir)
row_mean = np.mean(processed_image, axis=1)
max_row_each = np.where(
row_mean == max(np.mean(processed_image, axis=1)))[0]
max_row_each = max_row_each[0]
max_row = max_row_each
#print(max_row_each)
# processed_image = processed_image[max_row-12: max_row+12, :]
# processed_image = uniform_filter(processed_image, size=(10, 4), mode='mirror')
# print(wavelen)
# #plt.plot(processed_image[max_row-2, :], 'purple', label='2 lines up')
# plt.plot(processed_image[max_row-1, :], 'y-', label='1 lines up')
# #plt.plot(processed_image[max_row+2, :], 'g-', label='2 lines down')
# plt.plot(processed_image[max_row+1, :], 'r-', label='1 line down')
# plt.plot(processed_image[max_row, :], 'k-', label='Brightest')
# plt.grid(True, linestyle = ':')
# plt.title('Profile along Spectral direction (Laser Wavelength =' + str(wavelen) + 'nm)')
# plt.xlabel('Spatial Pixel Index')
# plt.ylabel('Counts (DN)')
# plt.show()
# col_index = np.sort([randint(100, 2000) for p in range(0, 10)])
# col_index = [480, 485, 490, 500, 510]
# for i in col_index:
# plt.plot(processed_image[:, i]/np.max(processed_image[:, i]),
# 'o--', label='Col. ' + str(i))
# plt.legend(loc='best', ncol=4)
# plt.grid(True, linestyle=':')
# plt.title('Profile along Spectral direction (Laswer WL:' + str(wavelen)+' nm)')
# plt.xlabel('Spectral Pixel Index')
# plt.ylabel('Normalized Counts')
# plt.show()
# cc
# cc
wavelengths_all = []
#print(count)
smile_all = []
FWHM_all = []
for i in range(15, 2035):
wavelengths = [wavelen] * 2020
wavelengths_all.append(wavelengths)
#wavelengths_all.append(wavelengths)
smile, FWHM = fit_gaussian_func(max_row_each,
processed_image[:, i], wavelen, i)
smile_all.append(smile)
FWHM_all.append(FWHM)
#print(FWHM)
mean_val = np.mean(smile_all)
smile_all = moving_average(smile_all)
#smile_all = uniform_filter(smile_all, size=6, mode='mirror')
#print(np.array(smile_all).shape)
band_center = pd.Series(smile_all)
# Spectral alignment is calculated by comuting the difference between max and min spectral centers within 4 pixel moving window
max_val = band_center.rolling(4).max()
min_val = band_center.rolling(4).min()
spectral_alignment = max_val - min_val
spectral_alingn_unct.append(spectral_alignment)
#plt.plot(max_val-min_val, label=str(wavelen)+ 'nm' )
count = count + 1
#plt.plot(wavelengths[1400:], FWHM_all[1400:], 'o', label=str(wavelen)+ 'nm')
#plt.plot(uniform_filter(FWHM_all, size=12,mode='mirror'), label=str(wavelen)+ 'nm')
#plt.show()
#fit_gaussian_func(x_data, y_data, wavelen)
#
#
## #max_row = 29 # for 297.8 nm
## #max_row = 1546 #for 640nm
## #max_row = 1369 #for 605nm
## #max_row = 1050 #for 605nm
## #max_row = 142 # for 320 nm
##
## #max_row = 91 # for 310 nm
## max_row = 15 # 675 nm
## #max_row = 2032 # for 736 nm
## #check_val = 1392
# col_index = np.sort([randint(100, 2000) for p in range(0, 2)])
# for i in col_index:
#
# plt.plot(processed_image[:, i],
# 'o--', label='Spatial Index ' + str(i))
# plt.legend(loc='best', ncol=4)
# plt.grid(True, linestyle=':')
# plt.title('Profile along Spectral direction (Laser Wavelength =' + str(wavelen) + 'nm')
# plt.xlabel('Spectral Pixel Index')
# plt.ylabel('Counts (DN)')
# #plt.xlim(655, 675)
# #plt.yscale('log')
# plt.xlim(max_row_each-10, max_row_each+10)
#
#plt.show()
#
#plt.figure()
spectral_alignment_unct = np.std(np.array(spectral_alingn_unct), axis=0)
print(spectral_alignment_unct.shape)
plt.plot(spectral_alignment_unct, 'k', label='1-Sigma Spectral. Alignment')
plt.axhline(y=0.01,
color='red',
linestyle='--',
linewidth=2,
label='TEMPO Requirement')
plt.title(
'TEMPO Number of Samples per FWHM Vs. Wavelengths\n (*** Note: 2 pixel smoothing in spectral,12 pixel in spatial direction)',
fontsize=12)
plt.grid(True, linestyle=':')
plt.xlabel('Wavelength (nm)', fontsize=12)
plt.ylabel('Samples/FWHM', fontsize=12)
#plt.ylim(2.6, 3.3)
#plt.ylim(2.4, 3)
#plt.ylim(-0.3, 0.3)
# plt.axhline(y= 2.7, color='red', linestyle='--', linewidth=2)
#plt.axhline(y= -0.2, color='red', linestyle='--', linewidth=2)
#plt.axhline(y= 0.2, color='red', linestyle='--', linewidth=2)
plt.xlim(0, 2500)
plt.legend(loc='best')
#plt.text(500, 0.25, '----TEMPO Requirement', color='red', fontsize=12)
#plt.text(500, 2.72, '----TEMPO Requirement', color='red', fontsize=12)
plt.show()
def main():
"""
This is the main function that does all the processing. It calls the required
functions to process the raw image. Raw images are saved in .sav (IDL)
format and read through this script.
"""
data_dir = r'F:\TEMPO\Data\GroundTest\FPS\Spectrometer\Dark_data\2017.07.10'
plot_dir = os.path.join(data_dir, 'Dark_Current_Image')
image_dir = os.path.join(data_dir, 'saved_quads')
image_save_dir = os.path.join(image_dir, 'saved_plots')
telemetry_dir = r'E:\Image Data\2018.07.10\processed\h5'
linearity = 0 # option to turn on/off linearity correction
smear = 1
cross_talk = 1
dark_current = 1
if not os.path.exists(plot_dir):
os.makedirs(plot_dir)
# data_path_all = sorted([each for each in os.listdir(image_dir)
# if each.startswith('2017_07_30_00_24_59_38064')
# and each.endswith('.sav')])
#
data_path_all = sorted([each for each in os.listdir(image_dir)
if each.endswith('.sav')])
print(len(data_path_all))
print('Total data = ', len(data_path_all))
count = 0
all_dark_current=[]
for data_path in data_path_all[1:]:
data_file = os.path.join(image_dir, data_path)
print(data_path)
# cc
telemetry_file_name = data_path.split('.')[0]
#################### Telemetry Statistics###############################
# Now lets parse the telemetry file to get the required information###
#Note; The function can return the CCD temp and FPE temp. But these are
#not needed on the image processing algorithm.. atleast for the time being
telemetry_file = os.path.join(telemetry_dir, telemetry_file_name+'.h5')
if os.path.exists(telemetry_file):
coadds, int_time, fpe_temp, fpa_temp = parse_telemetry_file(telemetry_dir,
telemetry_file_name)
print('FPE Temp. = ', round(fpe_temp, 2), 'FPA Temp = ',
round(fpa_temp, 2))
print(int_time)
print(coadds)
else:
print('Telemetry file missing')
coadds = 10
int_time = 160
fpe_temp = 35.8
fpa_temp = 11.9
##############Image Processing begins here##############################
#CC
full_frame = read_idl_file(data_file)
#check_image = create_image_active_region(full_frame)
#raw_image = create_final_image(np.array(full_frame))
#create_image(raw_image, 'a', 'b')
# print('Max. Val. Raw = ', np.max(raw_image))
quads = ['Quad A', 'Quad B', 'Quad C', 'Quad D']
##########################OFFSET REMOVAL###############################
# Input : Full_Frame TEMPO IMAGE
# Otput : Bias Subtracted Active Region. The active region dimensions are
#now 1028*1024. 2 pixel lines from SMEAR overclocks are now used for
#to store storage summation information
# For dark data, only offset removal is needed to compute the dark current
# For light data, additional processing such as linearity, smear and
#cross talk is needed
bias_removed_quads = perform_bias_subtraction(full_frame)
#raw_image = create_final_image(np.array(bias_removed_quads))
#create_image(raw_image/coadds, 'a', 'b')
#print('Max. Val. Offset Removed = ', np.max(bias_removed_quads))
text1 = telemetry_file_name + '.img' + ' (Raw Data)\n Int. time:' + str(round(int_time, 1)) + \
'ms, Co-adds:' + str(int(coadds)) + \
', FPE temp:' + str(round(fpe_temp, 1)) + 'C, ' + \
' FPA temp: ' + str(round(fpa_temp, 1)) + 'C'
# Now let us save the raw image
raw_image_save = os.path.join(image_save_dir, 'raw_image')
if not os.path.exists(raw_image_save):
os.makedirs(raw_image_save)
plot_save_dir = raw_image_save + '/'+ data_path+'.png'
#prnu_spectrometer[prnu_spectrometer < 0.9] = 0.9
#prnu_spectrometer[prnu_spectrometer > 1.2] = 1.2
#print(np.min(check_image))
#create_image(check_image, text1, plot_save_dir)
#---------------------------------------------------------------------
#########################NON LINEARITY CORRECTION#######################
# Input : Bias Subtracted Active regions
# Output : Linearized Active Region Quads
if linearity:
linearized_a = apply_linearity_correction(bias_removed_quads[0, :, :],
quads[0], coadds)
linearized_b = apply_linearity_correction(bias_removed_quads[1, :, :],
quads[1], coadds)
linearized_c = apply_linearity_correction(bias_removed_quads[2, :, :],
quads[2], coadds)
linearized_d = apply_linearity_correction(bias_removed_quads[3, :, :],
quads[3], coadds)
linearized_quads = np.array([linearized_a, linearized_b,
linearized_c, linearized_d])
#print(np.mean(linearized_quads))
else:
linearized_quads = bias_removed_quads
#----------------------------------------------------------------------
##########################SMEAR REMOVAL################################
# Input : linearized quads ( all quads together)
# Output : SMEAR offset corrected Quad
#### lets' create the masked array with outlier mask################
# The outlier mask is array of list of 4 quads.
#print('Max. Val. Linearized = ', np.max(linearized_quads))
outlier_mask = read_outlier_mask()
# Note : all the arrays after this step are masked arrays
if smear:
smear_removed_quads = perform_smear_removal(linearized_quads, int_time,
outlier_mask)
else:
smear_removed_quads = linearized_quads
#----------------------------------------------------------------------
##########################CROSS-TALK REMOVAL###########################
# Input : smear removed quads (all quads together)
# Output : cross talk removed quads
#print('Max. Val. Smeared = ', np.max(smear_removed_quads))
if cross_talk:
cross_talk_removed_quads = remove_cross_talk(np.array(smear_removed_quads))
else:
cross_talk_removed_quads = smear_removed_quads
#----------------------------------------------------------------------
#print('Max. Val. Cross Talked = ', np.max(cross_talk_removed_quads))
processed_image = create_final_image(np.array(cross_talk_removed_quads))
processed_image = processed_image
#prnu_map = parse_prnu_file()
prnu_map = read_prnu_files()
prnu_spectrometer = create_final_image(np.array(prnu_map))
prnu_spectrometer[prnu_spectrometer > 1.03] = 1.02
prnu_spectrometer[prnu_spectrometer < 0.97] = 0.98
outlier_spectrometer = create_final_image(np.array(outlier_mask))
nx_quad, ny_quad = processed_image.shape
outlier_spectrometer = np.reshape(outlier_spectrometer,
(nx_quad*ny_quad, 1))
#outlier_detectors = np.array(np.where([outlier_spectrometer == 1]))
#print('outliers =', outlier_detectors.shape[1])
outlier_spectrometer = np.reshape(outlier_spectrometer, (nx_quad, ny_quad))
#create_image(outlier_spectrometer,'outliers = '+str(outlier_detectors.shape[1]),'b')
processed_image = processed_image/(coadds*int_time)
processed_image[processed_image >= 0.8*16383] = 0
dark_current_each = np.median(processed_image, axis=1)
all_dark_current.append(dark_current_each)
#processed_image[processed_image==16383] = np.min(processed_image)
#processed_image[processed_image<0] = 0
#processed_image[processed_image==16383] = np.min(processed_image)
#print(np.min(processed_image))
text1 = telemetry_file_name+'.img' +', UV CCD \n' + \
'(Int. time:' + str(round(int_time,2))+ \
'ms, Co-adds:' +str(int(coadds))+\
', FPE temp:'+ str(round(fpe_temp, 2))+'C, ' + \
' FPA temp: ' + str(round(fpa_temp, 2))+'C)\n'+ \
'Mean DC = '+ str(round(np.mean(processed_image[0:1028:, :]),3)) +' DN/msec'
processed_image_save = os.path.join(image_save_dir, 'processed_image')
if not os.path.exists(processed_image_save):
os.makedirs(processed_image_save)
plot_save_dir = plot_dir + '/' + data_path + '.png'
#processed_image = uniform_filter(processed_image, size=(5, 3), mode='mirror')
dark_current = processed_image
med_val = np.median(dark_current)
dark_current[dark_current>0.15] = med_val
dark_current[dark_current<-0.1]= med_val
create_image(dark_current, text1, plot_save_dir)
Dark_Current_name = plot_dir +'/'+ 'dark_current.csv'
np.savetxt(Dark_Current_name, np.array(all_dark_current), delimiter=",")
print('DONE!!')
def main():
"""
This is the main function that does all the processing. It calls the required
functions to process the raw image. Raw images are saved in .sav (IDL)
format and read through this script.
"""
data_dir = r'F:\TEMPO\Data\GroundTest\FPS\Spectrometer\Spectral_Co_registration'
image_dir = os.path.join(data_dir, 'saved_quads')
save_dir = os.path.join(image_dir, 'processed_image')
image_save_dir = os.path.join(image_dir, 'saved_plots')
telemetry_dir = os.path.join(data_dir, 'processed/h5')
linearity = 0 # option to turn on/off linearity correction
smear = 1
cross_talk = 1
bandpass_val = []
bandpass_val_norm = []
normalization_factor_all = []
dark_current = 0
if not os.path.exists(save_dir):
os.makedirs(save_dir)
# data_path_all = sorted([each for each in os.listdir(image_dir)
# if each.startswith('2017_07_30_00_24_59_38064')
# and each.endswith('.sav')])
#
data_path_all = sorted(
[each for each in os.listdir(image_dir) if each.endswith('.sav')])
# #dark_data = data_path_all[1:len(data_path_all):2]
print('Total data = ', len(data_path_all))
# resultFile = open (os.path.join(save_dir,'file_name_all.csv'),'w')
#
# for results in data_path_all[0:15]:
# resultFile.write(results+"\n")
# resultFile.close()
if dark_current:
dark_current = calculate_dark_current(image_dir, coadds=1)
else:
dark_current = 0
count = 8
peak_loc_all = []
for data_path in data_path_all[8:]:
data_file = os.path.join(image_dir, data_path)
print(data_path)
# cc
telemetry_file_name = data_path.split('.')[0]
#################### Telemetry Statistics###############################
# Now lets parse the telemetry file to get the required information###
#Note; The function can return the CCD temp and FPE temp. But these are
#not needed on the image processing algorithm.. atleast for the time being
telemetry_file = os.path.join(telemetry_dir,
telemetry_file_name + '.h5')
if os.path.exists(telemetry_file):
coadds, int_time, fpe_temp, fpa_temp = parse_telemetry_file(
telemetry_dir, telemetry_file_name)
print('FPE Temp. = ', round(fpe_temp, 2), 'FPA Temp = ',
round(fpa_temp, 2))
else:
print('Telemetry file missing')
coadds = 10
int_time = 6000
fpe_temp = 43
fpa_temp = -21
##############Image Processing begins here##############################
full_frame = read_idl_file(data_file)
check_image = create_image_active_region(full_frame)
raw_image = create_final_image(np.array(full_frame))
# print('Max. Val. Raw = ', np.max(raw_image))
quads = ['Quad A', 'Quad B', 'Quad C', 'Quad D']
##########################OFFSET REMOVAL###############################
# Input : Full_Frame TEMPO IMAGE
# Otput : Bias Subtracted Active Region. The active region dimensions are
#now 1028*1024. 2 pixel lines from SMEAR overclocks are now used for
#to store storage summation information
# For dark data, only offset removal is needed to compute the dark current
# For light data, additional processing such as linearity, smear and
#cross talk is needed
bias_removed_quads = perform_bias_subtraction(full_frame)
#print('Max. Val. Offset Removed = ', np.max(bias_removed_quads))
text1 = telemetry_file_name+'.img' +' (Raw Data)\n Int. time:' + str(round(int_time, 1))+ \
'ms, Co-adds:' +str(int(coadds))+\
', FPE temp:'+ str(round(fpe_temp, 1))+'C, ' + \
' FPA temp: ' + str(round(fpa_temp, 1))+'C'
# Now let us save the raw image
raw_image_save = os.path.join(image_save_dir, 'raw_image')
if not os.path.exists(raw_image_save):
os.makedirs(raw_image_save)
plot_save_dir = raw_image_save + '/' + data_path + '.png'
#prnu_spectrometer[prnu_spectrometer < 0.9] = 0.9
#prnu_spectrometer[prnu_spectrometer > 1.2] = 1.2
#print(np.min(check_image))
#create_image(check_image, text1, plot_save_dir)
#---------------------------------------------------------------------
#########################NON LINEARITY CORRECTION#######################
# Input : Bias Subtracted Active regions
# Output : Linearized Active Region Quads
if linearity:
linearized_a = apply_linearity_correction(
bias_removed_quads[0, :, :], quads[0], coadds)
linearized_b = apply_linearity_correction(
bias_removed_quads[1, :, :], quads[1], coadds)
linearized_c = apply_linearity_correction(
bias_removed_quads[2, :, :], quads[2], coadds)
linearized_d = apply_linearity_correction(
bias_removed_quads[3, :, :], quads[3], coadds)
linearized_quads = np.array(
[linearized_a, linearized_b, linearized_c, linearized_d])
#print(np.mean(linearized_quads))
else:
linearized_quads = bias_removed_quads
#----------------------------------------------------------------------
##########################SMEAR REMOVAL################################
# Input : linearized quads ( all quads together)
# Output : SMEAR offset corrected Quad
#### lets' create the masked array with outlier mask################
# The outlier mask is array of list of 4 quads.
#print('Max. Val. Linearized = ', np.max(linearized_quads))
outlier_mask = read_outlier_mask()
# Note : all the arrays after this step are masked arrays
if smear:
smear_removed_quads = perform_smear_removal(
linearized_quads, int_time, outlier_mask)
else:
smear_removed_quads = linearized_quads
#----------------------------------------------------------------------
##########################CROSS-TALK REMOVAL###########################
# Input : smear removed quads (all quads together)
# Output : cross talk removed quads
#print('Max. Val. Smeared = ', np.max(smear_removed_quads))
if cross_talk:
cross_talk_removed_quads = remove_cross_talk(
np.array(smear_removed_quads))
else:
cross_talk_removed_quads = smear_removed_quads
#----------------------------------------------------------------------
#print('Max. Val. Cross Talked = ', np.max(cross_talk_removed_quads))
processed_image = create_final_image(
np.array(cross_talk_removed_quads))
processed_image = processed_image - dark_current
#prnu_map = parse_prnu_file()
prnu_map = read_prnu_files()
prnu_spectrometer = create_final_image(np.array(prnu_map))
prnu_spectrometer[prnu_spectrometer > 1.03] = 1.02
prnu_spectrometer[prnu_spectrometer < 0.97] = 0.98
outlier_spectrometer = create_final_image(np.array(outlier_mask))
nx_quad, ny_quad = processed_image.shape
outlier_spectrometer = np.reshape(outlier_spectrometer,
(nx_quad * ny_quad, 1))
outlier_detectors = np.array(np.where([outlier_spectrometer == 1]))
#print('outliers =', outlier_detectors.shape[1])
outlier_spectrometer = np.reshape(outlier_spectrometer,
(nx_quad, ny_quad))
#create_image(outlier_spectrometer,'outliers = '+str(outlier_detectors.shape[1]),'b')
processed_image = processed_image / (coadds * prnu_spectrometer)
#print(np.min(processed_image))
# Read the laser wavelength
wavelen = np.loadtxt(os.path.join(image_dir, 'Wavelength.csv'),
delimiter=',')
# Read the spectral index
spec_index = np.loadtxt(os.path.join(image_dir, 'Spectral_column.csv'),
delimiter=',')
wavelen = wavelen[count]
spec_pix = int(spec_index[count])
#print('Test Data= ', wavelen, spec_pix)
#processed_image[processed_image>=0.85*16383] = 16383
text1 = telemetry_file_name+'.img' +' (Laser WL:' +str(wavelen) +' nm)\n Int. time:' + str(coadds)+ \
'ms, Co-adds:' +str(int(coadds))+\
', FPE temp:'+ str(round(fpe_temp, 1))+'C, ' + \
' FPA temp: ' + str(round(fpa_temp, ))+'C'
processed_image_save = os.path.join(image_save_dir, 'processed_image')
if not os.path.exists(processed_image_save):
os.makedirs(processed_image_save)
plot_save_dir = processed_image_save + '/' + data_path + '.png'
#processed_image = uniform_filter(processed_image, size=(5, 3), mode='mirror')
create_image(processed_image, text1, plot_save_dir)
# processed_image = processed_image[max_row-12: max_row+12, :]
# processed_image = uniform_filter(processed_image, size=(10, 4), mode='mirror')
# plt.plot(processed_image[:, 7]/np.max(processed_image[:, 7]),'r', label = 'Pin Hole 1(Sp. Index : 7)')
# plt.plot(processed_image[:, 514]/np.max(processed_image[:, 514]),'b', label = 'Pin Hole 2(Sp. Index : 514)')
# plt.plot(processed_image[:, 1023]/np.max(processed_image[:, 1023]),'g', label = 'Pin Hole 3(Sp. Index : 1023)')
# plt.plot(processed_image[:, 1532]/np.max(processed_image[:, 1532]),'m', label = 'Pin Hole 4(Sp. Index : 1532)')
# plt.plot(processed_image[:, 2041]/np.max(processed_image[:, 2041]),'k', label = 'Pin Hole 5(Sp. Index : 2041)')
# plt.title('Profile of a pinhole illumiation along spectral direction')
# plt.xlabel('Spectral Pixel Index')
# plt.ylabel('Normalized Counts')
# plt.legend(loc='best')
# plt.grid(True,linestyle=':')
# plt.show()
# # print(wavelen)
peak_loc = fit_gaussian_func(processed_image, wavelen, spec_pix)
peak_loc_all.append(peak_loc)
count = count + 1
# plt.plot(processed_image[spec_pix, :], 'purple', label='2 lines up')
# plt.show()
# p1 = processed_image[spec_pix, 1:9 ]
# p2 = processed_image[spec_pix, 510:518]
# p3 = processed_image[spec_pix, 1019:1027]
# p4 = processed_image[spec_pix, 1528:1536]
# p5 = processed_image[spec_pix, 2037:2045]
# plt.plot(p1/np.max(p1), 'ro-', label='Pin Hole 1')
# plt.plot(p2/np.max(p2), 'bo-', label='Pin Hole 1')
# plt.plot(p3/np.max(p3), 'go-', label='Pin Hole 1')
# plt.plot(p4/np.max(p4), 'mo-', label='Pin Hole 1')
# plt.plot(p5/np.max(p5), 'ko-', label='Pin Hole 1')
# plt.grid(True, linestyle = ':')
# plt.legend(loc='best')
# plt.title('Spatial Profile of 5 pin hole')
# plt.xlabel('Spatial Pixel Index')
# plt.ylabel('Counts (DN)')
# plt.show()
#
final_data = os.path.join(image_dir, 'peak_loc_all.csv')
np.savetxt(final_data, np.array(peak_loc_all), delimiter=",")
def main():
"""
This is the main function that does all the processing. It calls the required
functions to process the raw image. Raw images are saved in .sav (IDL)
format and read through this script.
"""
data_dir = r'F:\TEMPO\Data\GroundTest\FPS\Spectrometer\Photon_Transfer_TVAC'
telemetry_dir = r'F:\TEMPO\Data\GroundTest\FPS\Spectrometer\2018.06.28'
image_dir = os.path.join(data_dir, 'saved_quads')
save_dir = os.path.join(image_dir, 'processed_image')
image_save_dir = os.path.join(image_dir, 'saved_plots')
telemetry_dir = os.path.join(telemetry_dir, 'processed/h5')
linearity = 0 # option to turn on/off linearity correction
smear = 1
cross_talk = 1
var_all = []
signal_all = []
all_image = []
dark_current = 0
if not os.path.exists(save_dir):
os.makedirs(save_dir)
# data_path_all = sorted([each for each in os.listdir(image_dir)
# if each.startswith('2017_07_30_00_24_59_38064')
# and each.endswith('.sav')])
#
data_path_all = sorted([
each for each in os.listdir(image_dir)
if each.endswith('2018_06_28_20_41_18_042102.img.sav')
])
# #dark_data = data_path_all[1:len(data_path_all):2]
print('Total data = ', len(data_path_all))
# resultFile = open (os.path.join(save_dir,'file_name_all.csv'),'w')
#
# for results in data_path_all[0:15]:
# resultFile.write(results+"\n")
# resultFile.close()
if dark_current:
dark_current = calculate_dark_current(image_dir, coadds=1)
else:
dark_current = 0
count = 0
#peak_loc_all = []
for data_path in data_path_all:
data_file = os.path.join(image_dir, data_path)
print(data_path)
# cc
telemetry_file_name = data_path.split('.')[0]
#################### Telemetry Statistics###############################
# Now lets parse the telemetry file to get the required information###
#Note; The function can return the CCD temp and FPE temp. But these are
#not needed on the image processing algorithm.. atleast for the time being
telemetry_file = os.path.join(telemetry_dir,
telemetry_file_name + '.h5')
if os.path.exists(telemetry_file):
coadds, int_time, fpe_temp, fpa_temp = parse_telemetry_file(
telemetry_dir, telemetry_file_name)
print('FPE Temp. = ', round(fpe_temp, 2), 'FPA Temp = ',
round(fpa_temp, 2))
print('Integ. Time =', int_time)
else:
print('Telemetry file missing')
coadds = 10
int_time = 6000
fpe_temp = 43
fpa_temp = -21.5
##############Image Processing begins here##############################
full_frame = read_idl_file(data_file)
#check_image = create_image_active_region(full_frame)
#raw_image = create_final_image(np.array(full_frame))
# print('Max. Val. Raw = ', np.max(raw_image))
quads = ['Quad A', 'Quad B', 'Quad C', 'Quad D']
##########################OFFSET REMOVAL###############################
# Input : Full_Frame TEMPO IMAGE
# Otput : Bias Subtracted Active Region. The active region dimensions are
#now 1028*1024. 2 pixel lines from SMEAR overclocks are now used for
#to store storage summation information
# For dark data, only offset removal is needed to compute the dark current
# For light data, additional processing such as linearity, smear and
#cross talk is needed
bias_removed_quads = perform_bias_subtraction(full_frame)
#print('Max. Val. Offset Removed = ', np.max(bias_removed_quads))
text1 = telemetry_file_name+'.img' +' (Raw Data)\n Int. time:' + str(round(int_time, 3))+ \
'ms, Co-adds:' +str(int(coadds))+\
', FPE temp:'+ str(round(fpe_temp, 1))+'C, ' + \
' FPA temp: ' + str(round(fpa_temp, 1))+'C'
# Now let us save the raw image
raw_image_save = os.path.join(image_save_dir, 'raw_image')
if not os.path.exists(raw_image_save):
os.makedirs(raw_image_save)
plot_save_dir = raw_image_save + '/' + data_path + '.png'
#prnu_spectrometer[prnu_spectrometer < 0.9] = 0.9
#prnu_spectrometer[prnu_spectrometer > 1.2] = 1.2
#print(np.min(check_image))
#create_image(check_image, text1, plot_save_dir)
#---------------------------------------------------------------------
#########################NON LINEARITY CORRECTION#######################
# Input : Bias Subtracted Active regions
# Output : Linearized Active Region Quads
if linearity:
linearized_a = apply_linearity_correction(
bias_removed_quads[0, :, :], quads[0], coadds)
linearized_b = apply_linearity_correction(
bias_removed_quads[1, :, :], quads[1], coadds)
linearized_c = apply_linearity_correction(
bias_removed_quads[2, :, :], quads[2], coadds)
linearized_d = apply_linearity_correction(
bias_removed_quads[3, :, :], quads[3], coadds)
linearized_quads = np.array(
[linearized_a, linearized_b, linearized_c, linearized_d])
#print(np.mean(linearized_quads))
else:
linearized_quads = bias_removed_quads
#----------------------------------------------------------------------
##########################SMEAR REMOVAL################################
# Input : linearized quads ( all quads together)
# Output : SMEAR offset corrected Quad
#### lets' create the masked array with outlier mask################
# The outlier mask is array of list of 4 quads.
#print('Max. Val. Linearized = ', np.max(linearized_quads))
outlier_mask = read_outlier_mask()
# Note : all the arrays after this step are masked arrays
if smear:
smear_removed_quads = perform_smear_removal(
linearized_quads, int_time, outlier_mask)
else:
smear_removed_quads = linearized_quads
#----------------------------------------------------------------------
##########################CROSS-TALK REMOVAL###########################
# Input : smear removed quads (all quads together)
# Output : cross talk removed quads
#print('Max. Val. Smeared = ', np.max(smear_removed_quads))
if cross_talk:
cross_talk_removed_quads = remove_cross_talk(
np.array(smear_removed_quads))
else:
cross_talk_removed_quads = smear_removed_quads
#----------------------------------------------------------------------
#print('Max. Val. Cross Talked = ', np.max(cross_talk_removed_quads))
processed_image = create_final_image(
np.array(cross_talk_removed_quads))
processed_image = processed_image - dark_current
#prnu_map = parse_prnu_file()
prnu_map = read_prnu_files()
prnu_spectrometer = create_final_image(np.array(prnu_map))
prnu_spectrometer[prnu_spectrometer > 1.03] = 1.02
prnu_spectrometer[prnu_spectrometer < 0.97] = 0.98
outlier_spectrometer = create_final_image(np.array(outlier_mask))
nx_quad, ny_quad = processed_image.shape
outlier_spectrometer = np.reshape(outlier_spectrometer,
(nx_quad * ny_quad, 1))
#outlier_detectors = np.array(np.where([outlier_spectrometer == 1]))
#print('outliers =', outlier_detectors.shape[1])
outlier_spectrometer = np.reshape(outlier_spectrometer,
(nx_quad, ny_quad))
#create_image(outlier_spectrometer,'outliers = '+str(outlier_detectors.shape[1]),'b')
processed_image = processed_image / (coadds * prnu_spectrometer)
processed_image[processed_image >= 0.85 * 16383] = 16383
#print(np.min(processed_image))
#processed_image[processed_image>=0.85*16383] = 16383
text1 = telemetry_file_name+'.img, ' + 'Int. time:' + str(round(int_time,2))+ \
'ms\n Co-adds:' +str(int(coadds))+\
', FPE temp:'+ str(round(fpe_temp, 1))+'C, ' + \
' FPA temp: ' + str(round(fpa_temp, ))+'C'
processed_image_save = os.path.join(image_save_dir, 'processed_image')
if not os.path.exists(processed_image_save):
os.makedirs(processed_image_save)
plot_save_dir = processed_image_save + '/' + data_path + '.png'
#processed_image = uniform_filter(processed_image, size=(5, 3), mode='mirror')
#create_image(processed_image, text1, plot_save_dir)
all_image.append(processed_image)
count = count + 1
var_all = np.var(all_image, axis=0)
signal_all = np.median(all_image, axis=0)
def main():
"""
This is the main function that does all the processing. It calls the required
functions to process the raw image. Raw images are saved in .sav (IDL)
format and read through this script.
"""
data_dir = r'F:\TEMPO\Data\GroundTest\FPS\Spectrometer\Read_Noise'
plot_dir = os.path.join(data_dir, 'Read_Noise_image')
image_dir = os.path.join(data_dir, 'saved_quads')
image_save_dir = os.path.join(image_dir, 'saved_plots')
telemetry_dir = r'F:\TEMPO\Data\GroundTest\FPS\Spectrometer\2018.06.28'
telemetry_dir = os.path.join(telemetry_dir, 'processed/h5')
linearity = 0 # option to turn on/off linearity correction
smear = 1
cross_talk = 1
if not os.path.exists(plot_dir):
os.makedirs(plot_dir)
# data_path_all = sorted([each for each in os.listdir(image_dir)
# if each.startswith('2017_07_30_00_24_59_38064')
# and each.endswith('.sav')])
#
data_path_all = sorted(
[each for each in os.listdir(image_dir) if each.endswith('.sav')])
print(len(data_path_all))
print('Total data = ', len(data_path_all))
count = 0
all_dark_current = []
for data_path in data_path_all:
data_file = os.path.join(image_dir, data_path)
print(data_path)
# cc
telemetry_file_name = data_path.split('.')[0]
#################### Telemetry Statistics###############################
# Now lets parse the telemetry file to get the required information###
#Note; The function can return the CCD temp and FPE temp. But these are
#not needed on the image processing algorithm.. atleast for the time being
telemetry_file = os.path.join(telemetry_dir,
telemetry_file_name + '.h5')
if os.path.exists(telemetry_file):
coadds, int_time, fpe_temp, fpa_temp = parse_telemetry_file(
telemetry_dir, telemetry_file_name)
print('FPE Temp. = ', round(fpe_temp, 2), 'FPA Temp = ',
round(fpa_temp, 2))
print(int_time)
print(coadds)
else:
print('Telemetry file missing')
coadds = 10
int_time = 160
fpe_temp = 35.8
fpa_temp = 11.9
##############Image Processing begins here##############################
#CC
full_frame = read_idl_file(data_file)
#check_image = create_image_active_region(full_frame)
#raw_image = create_final_image(np.array(full_frame))
#create_image(raw_image, 'a', 'b')
# print('Max. Val. Raw = ', np.max(raw_image))
quads = ['Quad A', 'Quad B', 'Quad C', 'Quad D']
##########################OFFSET REMOVAL###############################
# Input : Full_Frame TEMPO IMAGE
# Otput : Bias Subtracted Active Region. The active region dimensions are
#now 1028*1024. 2 pixel lines from SMEAR overclocks are now used for
#to store storage summation information
# For dark data, only offset removal is needed to compute the dark current
# For light data, additional processing such as linearity, smear and
#cross talk is needed
bias_removed_quads = perform_bias_subtraction(full_frame)
#raw_image = create_final_image(np.array(bias_removed_quads))
#create_image(raw_image/coadds, 'a', 'b')
#print('Max. Val. Offset Removed = ', np.max(bias_removed_quads))
text1 = telemetry_file_name + '.img' + ' (Raw Data)\n Int. time:' + str(round(int_time, 1)) + \
'ms, Co-adds:' + str(int(coadds)) + \
', FPE temp:' + str(round(fpe_temp, 1)) + 'C, ' + \
' FPA temp: ' + str(round(fpa_temp, 1)) + 'C'
# Now let us save the raw image
raw_image_save = os.path.join(image_save_dir, 'raw_image')
if not os.path.exists(raw_image_save):
os.makedirs(raw_image_save)
plot_save_dir = raw_image_save + '/' + data_path + '.png'
#prnu_spectrometer[prnu_spectrometer < 0.9] = 0.9
#prnu_spectrometer[prnu_spectrometer > 1.2] = 1.2
#print(np.min(check_image))
#create_image(check_image, text1, plot_save_dir)
#---------------------------------------------------------------------
#########################NON LINEARITY CORRECTION#######################
# Input : Bias Subtracted Active regions
# Output : Linearized Active Region Quads
if linearity:
linearized_a = apply_linearity_correction(
bias_removed_quads[0, :, :], quads[0], coadds)
linearized_b = apply_linearity_correction(
bias_removed_quads[1, :, :], quads[1], coadds)
linearized_c = apply_linearity_correction(
bias_removed_quads[2, :, :], quads[2], coadds)
linearized_d = apply_linearity_correction(
bias_removed_quads[3, :, :], quads[3], coadds)
linearized_quads = np.array(
[linearized_a, linearized_b, linearized_c, linearized_d])
#print(np.mean(linearized_quads))
else:
linearized_quads = bias_removed_quads
#----------------------------------------------------------------------
##########################SMEAR REMOVAL################################
# Input : linearized quads ( all quads together)
# Output : SMEAR offset corrected Quad
#### lets' create the masked array with outlier mask################
# The outlier mask is array of list of 4 quads.
#print('Max. Val. Linearized = ', np.max(linearized_quads))
outlier_mask = read_outlier_mask()
# Note : all the arrays after this step are masked arrays
if smear:
smear_removed_quads = perform_smear_removal(
linearized_quads, int_time, outlier_mask)
else:
smear_removed_quads = linearized_quads
#----------------------------------------------------------------------
##########################CROSS-TALK REMOVAL###########################
# Input : smear removed quads (all quads together)
# Output : cross talk removed quads
#print('Max. Val. Smeared = ', np.max(smear_removed_quads))
if cross_talk:
cross_talk_removed_quads = remove_cross_talk(
np.array(smear_removed_quads))
else:
cross_talk_removed_quads = smear_removed_quads
#----------------------------------------------------------------------
#print('Max. Val. Cross Talked = ', np.max(cross_talk_removed_quads))
processed_image = create_final_image(
np.array(cross_talk_removed_quads))
#processed_image = processed_image
#prnu_map = parse_prnu_file()
prnu_map = read_prnu_files()
prnu_spectrometer = create_final_image(np.array(prnu_map))
prnu_spectrometer[prnu_spectrometer > 1.03] = 1.02
prnu_spectrometer[prnu_spectrometer < 0.97] = 0.98
outlier_spectrometer = create_final_image(np.array(outlier_mask))
nx_quad, ny_quad = processed_image.shape
outlier_spectrometer = np.reshape(outlier_spectrometer,
(nx_quad * ny_quad, 1))
#outlier_detectors = np.array(np.where([outlier_spectrometer == 1]))
#print('outliers =', outlier_detectors.shape[1])
outlier_spectrometer = np.reshape(outlier_spectrometer,
(nx_quad, ny_quad))
#create_image(outlier_spectrometer,'outliers = '+str(outlier_detectors.shape[1]),'b')
processed_image = processed_image
processed_image[processed_image >= 0.8 * 16383] = 0
dark_current_each = processed_image
#print(dark_current_each)
all_dark_current.append(dark_current_each)
#processed_image[processed_image==16383] = np.min(processed_image)
#processed_image[processed_image<0] = 0
#processed_image[processed_image==16383] = np.min(processed_image)
#print(np.min(processed_image))
text1 = telemetry_file_name+'.img' +', UV CCD \n' + \
'(Int. time:' + str(round(int_time,2))+ \
'ms, Co-adds:' +str(int(coadds))+\
', FPE temp:'+ str(round(fpe_temp, 2))+'C, ' + \
' FPA temp: ' + str(round(fpa_temp, 2))+'C)'
processed_image_save = os.path.join(image_save_dir, 'processed_image')
if not os.path.exists(processed_image_save):
os.makedirs(processed_image_save)
plot_save_dir = plot_dir + '/' + data_path + '.png'
#create_image(processed_image, text1, plot_save_dir)
#processed_image = uniform_filter(processed_image, size=(5, 3), mode='mirror')
#dark_current = processed_image
#med_val = np.median(dark_current)
#dark_current[dark_current>0.15] = med_val
#dark_current[dark_current<-0.1]= med_val
all_dark_current = np.array(all_dark_current)
Dark_Current_name = plot_dir + '/' + 'read_noise.csv'
np.savetxt(Dark_Current_name, np.array(all_dark_current), delimiter=",")
#print(all_dark_current.shape)
read_Noise = np.var(all_dark_current, axis=0)
# print('read_noise_dimes = ', read_Noise.shape)
create_image(read_Noise, text1, plot_save_dir)
read_Noise = filter_outlier_median(read_Noise)
label2 = 'mean variance = ' + str(round(np.mean(read_Noise), 3))
#nx_quad, ny_quad = read_Noise.shape
plt.figure(figsize=(10, 8))
plt.hist(read_Noise, 60, normed=0, facecolor='red', alpha=1, label=label2)
#title = 'Histogram of Variance of photon transfer sequence at 0 ms'
plt.title(
'Histogram of variance of processed Laser Image\n Int., time = 0, Num. images= 100',
fontsize=14,
fontweight="bold")
plt.grid(True, linestyle=':')
plt.xlabel('Temporal Noise (Variance)', fontsize=14, fontweight="bold")
plt.ylabel('Frequency', fontsize=14, fontweight="bold")
plt.legend(loc='best')
ax = plt.gca()
#plt.xlim(0, 300)
ax.yaxis.set_major_formatter(FormatStrFormatter('%.0f'))
plt.show()
plt.savefig(plot_dir + '/' + '_Read_Noise.png', dpi=100)
plt.close('all')
print('DONE!!')