def main():
"""
Tme main function
"""
#nx_quad = 1056 # For Tempo
#ny_quad = 1046 # For Tempo
#nlat = nx_quad*2
#nspec = ny_quad*2
file_path = r'C:\Users\nmishra\Workspace\TEMPO\Data\GroundTest\FPS\Dark\FPS_Dark\saved_quads'
data_path_all = [
each for each in os.listdir(file_path) if each.endswith('.sav')
]
#names_to_check = ['SHORT', 'NOM', 'LONG']
names_to_check = ['SHORT', 'NOM', 'OP', 'LONG']
mean_mask_all_lower = []
median_mask_all_lower = []
mean_mask_all_upper = []
median_mask_all_upper = []
saturation_mask_upper = []
saturation_mask_lower = []
for data_path in data_path_all:
# Now let's find the integtration type
print(data_path)
full_frame, collection_type, frames, int_time = \
make_quads_from_IDL_sav_file(file_path, data_path, names_to_check)
#Let's fetch the quads now
#quad_A = full_frame[0:ny_quad, 0:nx_quad]
active_quad_A = np.array(full_frame[4:1028, 10:1034])
#quad_B = full_frame[ny_quad:, 0:nx_quad]
active_quad_B = np.array(full_frame[4:1028, 1078:2102])
#quad_C = full_frame[ny_quad:, nx_quad:]
active_quad_C = np.array(full_frame[1064:2088, 1078:2102])
#quad_D = full_frame[ny_quad:, 0:nx_quad]
active_quad_D = np.array(full_frame[1064:2088, 10:1034])
lower_quads = np.concatenate((active_quad_A, active_quad_B), axis=1)
upper_quads = np.concatenate((active_quad_D, active_quad_C), axis=1)
active_quads = [lower_quads, upper_quads]
#quads = ['Lower_quads','Upper_quads']
# Now let's save the full_frame images
plot_dir = file_path
image_dir = 'Quad_images'
save_quad_images = os.path.join(plot_dir, image_dir)
if not os.path.exists(save_quad_images):
os.makedirs(save_quad_images)
image_title = 'Full Frame Quad Image'
plot_full_frame_image(full_frame, image_title, collection_type, frames,
int_time, save_quad_images)
# Create list of directories To save histograms before and after
#outlier rejections
mean_plot = r'Outlier_mean' + '/' + 'Lower_quads'
median_plot = r'Outlier_median' + '/' + 'Lower_quads'
folder_name_hist = 'Hist_plot'
folder_name_mask = 'Mask_plot'
folder_name_sat = 'Saturation_plot'
save_mean_hist = os.path.join(plot_dir, mean_plot, folder_name_hist)
if not os.path.exists(save_mean_hist):
os.makedirs(save_mean_hist)
save_median_hist = os.path.join(plot_dir, median_plot,
folder_name_hist)
if not os.path.exists(save_median_hist):
os.makedirs(save_median_hist)
save_mean_mask = os.path.join(plot_dir, mean_plot, folder_name_mask)
if not os.path.exists(save_mean_mask):
os.makedirs(save_mean_mask)
save_median_mask = os.path.join(plot_dir, median_plot,
folder_name_mask)
if not os.path.exists(save_median_mask):
os.makedirs(save_median_mask)
save_sat_mask = os.path.join(plot_dir, median_plot, folder_name_sat)
if not os.path.exists(save_sat_mask):
os.makedirs(save_sat_mask)
# Now let's call the outlier functions. First identify saturated
#bits and then identify the 'hot and cold' outlier pixels
title = 'Histogram of Quads A & B'
#title = 'Histogram of Quads C & D'
sat_pixels = identify_saturation(lower_quads, save_sat_mask,
collection_type, frames, int_time)
#outlier_filt_mean, outlier_mean = reject_outlier_mean(active_quads[k])
outlier_filt_med, outlier_med = reject_outlier_median(lower_quads)
# Ok now lets' plot the histograms to understand what we see.
# First the mean approach
print('ok, let us plot the histograms now')
#plot_hist_image(active_quads[k], title, collection_type, frames,
#outlier_filt_mean, outlier_mean, int_time,
# save_mean_hist)
# Secondly the median absolute difference approach
plot_hist_image(lower_quads, title, collection_type, frames,
outlier_filt_med, outlier_med, int_time,
save_median_hist)
#outlier_med= None
# Ok, lets now create a binary mask of outlier pixels
# First with the mean based approach
# mean_mask = create_outlier_mask(active_quads[k], outlier_mean[0],
# collection_type,
# frames, save_mean_mask)
# if k == 0:
# mean_mask_all_lower.append(mean_mask)
#
# else :
# mean_mask_all_upper.append(mean_mask)
#
# # Secondly, the same for median ased approach
#if len(outlier_med) == 1:
# outlier_med = 0
#else:
#outlier_med = outlier_med[1]
median_mask = create_outlier_mask(lower_quads, outlier_med[1],
collection_type, frames,
save_median_mask)
lower_quads = None
outlier_med = None
median_mask_all_lower.append(median_mask[:])
median_mask = None
saturation_mask_lower.append(sat_pixels[:])
# Ok now, lets' or all the median masks
print(np.array(median_mask_all_lower).shape)
final_mask_median_lower = np.bitwise_or.reduce(median_mask_all_lower[:])
plt.figure()
plt.imshow(np.invert(final_mask_median_lower),
cmap='bwr',
interpolation='none',
origin='lower')
#cbar = plt.colorbar(image)
plt.title('Binary outlier mask', fontsize=14)
plt.xlabel('# of spatial pixels', fontsize=12)
plt.ylabel('# of spectral pixels', fontsize=12)
plt.grid(False)
plt.savefig(r'C:\Users\nmishra\Desktop' + '/1.png',
dpi=100,
bbox_inches="tight")
cc
plt.figure()
plt.imshow(np.invert(median_mask_all_lower[1]),
cmap='bwr',
interpolation='none',
origin='lower')
#cbar = plt.colorbar(image)
plt.title('Binary outlier mask', fontsize=14)
plt.xlabel('# of spatial pixels', fontsize=12)
plt.ylabel('# of spectral pixels', fontsize=12)
plt.grid(False)
plt.savefig(r'C:\Users\nmishra\Desktop' + '/2.png',
dpi=100,
bbox_inches="tight")
def main():
"""
Tme main function
"""
#nx_quad = 1056 # For Tempo
#ny_quad = 1046 # For Tempo
#nlat = nx_quad*2
#nspec = ny_quad*2
file_path = r'C:\Users\nmishra\Workspace\TEMPO\Data\GroundTest\FPS\Integration_Sweep\Dark\saved_quads'
save_dir = r'C:\Users\nmishra\Workspace\TEMPO\outlier_mask'
data_path_all = [each for each in os.listdir(file_path) if each.endswith('.sav')]
#names_to_check = ['SHORT', 'NOM', 'LONG']
names_to_check = ['SHORT', 'NOM','OP', 'LONG']
mean_mask_all_lower = []
median_mask_all_lower = []
mean_mask_all_upper = []
median_mask_all_upper = []
saturation_mask_upper = []
saturation_mask_lower = []
for data_path in data_path_all:
print(data_path)
full_frame, collection_type, frames,int_time = \
make_quads_from_IDL_sav_file(file_path, data_path, names_to_check)
# all_full_frame contains all 100 frames
#Let's fetch the quads now
#quad_A = full_frame[0:ny_quad, 0:nx_quad]
quad_A = full_frame[:, 0, :, :]
active_quad_A = np.mean(quad_A[:, 4:1028, 10:1034], axis=0)
quad_B = full_frame[:, 1, :, :]
active_quad_B = np.mean(quad_B[:, 4:1028, 10:1034], axis=0)
quad_C = full_frame[:, 2, :, :]
active_quad_C = np.mean(quad_C[:, 4:1028, 10:1034], axis=0)
quad_D = full_frame[:, 3, :, :]
active_quad_D = np.mean(quad_D[:, 4:1028, 10:1034], axis=0)
lower_quads = np.concatenate((active_quad_A, active_quad_B), axis=1)
upper_quads = np.concatenate((active_quad_D, active_quad_C), axis=1)
active_quads =[lower_quads, upper_quads]
# for overclocks
#lower_quads = np.concatenate((bias_A, bias_B), axis=1)
#upper_quads = np.concatenate((bias_D, bias_C), axis=1)
active_quads =[lower_quads, upper_quads]
quads = ['Lower_quads','Upper_quads']
# Now let's save the full_frame images
plot_dir = save_dir
image_dir = 'Quad_images_'
save_quad_images = os.path.join(plot_dir, image_dir)
if not os.path.exists(save_quad_images):
os.makedirs(save_quad_images)
image_title = 'Full Frame Quad Image'
# plot_full_frame_image(full_frame, image_title, collection_type, frames,
# int_time, save_quad_images)
#
# Create list of directories To save histograms before and after
#outlier rejections
for k in np.arange(0,2):
#mean_plot = r'Outlier_mean_tsoc'+'/'+ quads[k]
median_plot = r'Outlier_median'+'/'+ quads[k]
folder_name_hist = 'Hist_plot'
folder_name_mask = 'Mask_plot'
folder_name_sat = 'Saturation_plot'
save_median_hist = os.path.join(plot_dir, median_plot, folder_name_hist)
if not os.path.exists(save_median_hist):
os.makedirs(save_median_hist)
save_median_mask = os.path.join(plot_dir, median_plot, folder_name_mask)
if not os.path.exists(save_median_mask):
os.makedirs(save_median_mask)
save_sat_mask = os.path.join(plot_dir, median_plot, folder_name_sat)
if not os.path.exists(save_sat_mask):
os.makedirs(save_sat_mask)
# Now let's call the outlier functions. First identify saturated
#bits and then identify the 'hot and cold' outlier pixels
if k == 0:
title = 'Histogram of Quads A & B'
else:
title = 'Histogram of Quads C & D'
sat_pixels = identify_saturation(active_quads[k],
save_sat_mask,
collection_type,
frames, int_time)
#outlier_filt_mean, outlier_mean = reject_outlier_mean(active_quads[k])
outlier_filt_med, outlier_med = reject_outlier_median(active_quads[k])
# Ok now lets' plot the histograms to understand what we see.
# First the mean approach
print('ok, let us plot the histograms now')
#plot_hist_image(active_quads[k], title, collection_type, frames,
#outlier_filt_mean, outlier_mean, int_time,
# save_mean_hist)
# Secondly the median absolute difference approach
plot_hist_image(active_quads[k], title, collection_type, frames,
outlier_filt_med, outlier_med, int_time,
save_median_hist)
#outlier_med= None
# Ok, lets now create a binary mask of outlier pixels
# First with the mean based approach
# mean_mask = create_outlier_mask(active_quads[k], outlier_mean[0],
# collection_type,
# frames, save_mean_mask)
# if k == 0:
# mean_mask_all_lower.append(mean_mask)
#
# else :
# mean_mask_all_upper.append(mean_mask)
#
# # Secondly, the same for median ased approach
if len(outlier_med) == 1:
outlier_med = 0
else:
outlier_med = outlier_med[1]
if k == 0:
title = 'Binary Outlier Mask of Quads A & B'
else:
title = 'Binary Outlier Mask of Quads C & D'
median_mask = create_outlier_mask(active_quads[k], outlier_med,
title, collection_type, frames,
save_median_mask)
active_quads[k] = None
outlier_med = None
if k == 0:
median_mask_all_lower.append(median_mask[:])
median_mask=None
saturation_mask_lower.append(sat_pixels[:])
sat_pixels = None
elif k==1 :
median_mask_all_upper.append(median_mask[:])
median_mask = None
saturation_mask_upper.append(sat_pixels[:])
sat_pixels = None
#***************************************************************************
# The following function creates final mask based on 80% occurence criteria
final_path = r'C:\Users\nmishra\Workspace\TEMPO\outlier_mask\Outlier_median\save_masks'
if not os.path.exists(final_path):
os.makedirs(final_path)
quad_name = 'lower_quads'
title = 'Quads A & B outlier mask (80% criteria)'
final_mask_lower = create_final_mask(median_mask_all_lower, quad_name, title, final_path)
quad_name = 'upper_quads'
title = 'Quads C & D outlier mask (80% criteria)'
final_mask_upper = create_final_mask(median_mask_all_upper, quad_name, title, final_path)
final_outlier_mask = np.concatenate((final_mask_lower, final_mask_upper),
axis=0)
plt.figure()
plt.imshow(final_outlier_mask, cmap='bwr',
interpolation='none', origin='lower')
#cbar = plt.colorbar(image)
nx_quad, ny_quad = np.array(final_outlier_mask).shape
final_outliers = np.array(np.where(np.reshape(final_outlier_mask,
(nx_quad*ny_quad, 1))==0)).shape[1]
plt.title('TEMPO outlier Mask (outliers = '+ str(final_outliers)+')',
fontsize=14)
plt.xlabel('# of spatial pixels', fontsize=12)
plt.ylabel('# of spectral pixels', fontsize=12)
plt.grid(False)
#final_path = r'C:\Users\nmishra\Workspace\TEMPO\Data\GroundTest\FPS\Integration_Sweep\Dark\saved_quads\Outlier_median\save_masks'
if not os.path.exists(final_path):
os.makedirs(final_path)
plt.savefig(final_path+'/'+'final_mask.png', dpi=100, bbox_inches="tight")
#***************************************************************************
#***************************************************************************
# if we want to 'OR" all the mask, use the following function
quad_name = 'lower_quads_or'
title = 'Quads A & B outlier mask (Logical OR)'
final_mask_lower_ored= create_ORed_mask(median_mask_all_lower, quad_name, title, final_path)
quad_name = 'upper_quads_ored'
title = 'Quads C & D outlier mask (Logical OR)'
final_mask_upper_ored = create_ORed_mask(median_mask_all_upper, quad_name, title, final_path)
final_outlier_mask_ored = np.concatenate((final_mask_lower_ored,
final_mask_upper_ored), axis=0)
mask_directory = r'C:\Users\nmishra\Workspace\TEMPO\outlier_mask\Outlier_median\save_masks'
mask_name = mask_directory+'/'+'final_outlier_mask_2_sigma.csv'
np.savetxt(mask_name, np.array(final_outlier_mask_ored), delimiter =",")
plt.figure()
plt.imshow(np.invert(final_outlier_mask_ored), cmap='bwr',
interpolation='none', origin='lower')
#cbar = plt.colorbar(image)
nx_quad, ny_quad = np.array(final_outlier_mask_ored).shape
final_outliers_ored = np.array(np.where(np.reshape(final_outlier_mask_ored,
(nx_quad*ny_quad, 1))==1)).shape[1]
plt.title('TEMPO Outlier Mask (outliers = '+ str(final_outliers_ored)+')',
fontsize=14)
plt.xlabel('# of spatial pixels', fontsize=12)
plt.ylabel('# of spectral pixels', fontsize=12)
plt.grid(False)
plt.savefig(final_path+'/'+'final_mask_ored.png', dpi=100, bbox_inches="tight")
def main():
"""
Tme main function
"""
#nx_quad = 1056 # For Tempo
#ny_quad = 1046 # For Tempo
#nlat = nx_quad*2
#nspec = ny_quad*2
file_path1 = r'F:\TEMPO\Data\GroundTest\FPS\Integration_Sweep\Light\Saved_quads'
save_dir = r'C:\Users\nmishra\Workspace\TEMPO\Data\GroundTest\FPS\Signal_dependent_offset\Light_Data'
all_int_files = [each for each in os.listdir(file_path1) \
if each.endswith('.dat.sav')]
if 'Integration_Sweep' in file_path1:
saturated_collects = [
'FT6_LONG_INT_130018.dat.sav', #'FT6_SHORT_INT_0.dat.sav',
'FT6_LONG_INT_134990.dat.sav',
'FT6_LONG_INT_139961.dat.sav',
'FT6_LONG_INT_145028.dat.sav',
'FT6_LONG_INT_149999.dat.sav',
'FT6_LONG_INT_154970.dat.sav',
'FT6_LONG_INT_160037.dat.sav',
'FT6_LONG_INT_165008.dat.sav',
'FT6_LONG_INT_169980.dat.sav',
'FT6_LONG_INT_175047.dat.sav',
'FT6_LONG_INT_180018.dat.sav',
'FT6_LONG_INT_184989.dat.sav',
'FT6_LONG_INT_189960.dat.sav',
'FT6_LONG_INT_195027.dat.sav',
'FT6_LONG_INT_199999.dat.sav'
]
elif 'Intensity_Sweep' in file_path1:
saturated_collects = [
'162_OP_INT_118000.dat.sav', '164_OP_INT_118000.dat.sav',
'166_OP_INT_118000.dat.sav', '168_OP_INT_118000.dat.sav',
'170_OP_INT_118000.dat.sav', '172_OP_INT_118000.dat.sav',
'174_OP_INT_118000.dat.sav', '176_OP_INT_118000.dat.sav',
'178_OP_INT_118000.dat.sav', '180_OP_INT_118000.dat.sav',
'182_OP_INT_118000.dat.sav', '184_OP_INT_118000.dat.sav',
'186_OP_INT_118000.dat.sav', '188_OP_INT_118000.dat.sav',
'190_OP_INT_118000.dat.sav', '192_OP_INT_118000.dat.sav',
'194_OP_INT_118000.dat.sav', '196_OP_INT_118000.dat.sav',
'198_OP_INT_118000.dat.sav', '200_OP_INT_118000.dat.sav',
'202_OP_INT_118000.dat.sav'
]
nominal_int_files = [
items for items in all_int_files
if not items.endswith(tuple(saturated_collects))
if items in all_int_files
]
for i in range(0, 4):
dframe1 = pd.DataFrame()
dframe2 = pd.DataFrame()
dframe3 = pd.DataFrame()
all_int_time = []
active_quad_even_all = []
active_quad_odd_all = []
active_quad_even_all_outlier_filt = []
active_quad_odd_all_outlier_filt = []
tsoc_even_all = []
tsoc_odd_all = []
tsoc_even_all_outlier_filt = []
tsoc_odd_all_outlier_filt = []
unct_spectral_even = []
unct_spectral_odd = []
#nominal_int_files = [nominal_int_files[0], nominal_int_files[1]]
for data_files in nominal_int_files:
data_path_name_split = data_files.split('_')
data_file = os.path.join(file_path1, data_files)
print(data_file)
IDL_variable = readsav(data_file)
if 'Intensity_Sweep' in file_path1:
int_time = data_path_name_split[0]
string1 = 'VA_'
string2 = 'VA Setting = '
else:
int_time = round(int(data_path_name_split[-1].split('.')[0]))
string1 = 'Integ_time_'
string2 = 'Int.time = '
#print(int_time)
all_int_time.append(int_time)
quads = ['Quad A', 'Quad B', 'Quad C', 'Quad D']
all_full_frame = IDL_variable.q
quads = ['Quad A', 'Quad B', 'Quad C', 'Quad D']
all_full_frame = IDL_variable.q
quad = all_full_frame[:, i, :, :]
tsoc_all = quad[:, 4:1028, 1034:1056]
active_quad = np.mean(quad[:, 4:1028, 10:1034], axis=0)
active_quad, smear = perform_smear_subtraction(
active_quad, int_time)
active_quad[active_quad == 16383] = 'nan'
active_quad_even = active_quad[:, ::2]
active_quad_even = np.nanmean(active_quad_even, axis=1)
active_quad_odd = np.nanmean((active_quad[:, 1::2]), axis=1)
#active_quad_even_outlier_filt = np.mean((active_quad[:, ::2]), axis=1)
#active_quad_odd_outlier_filt = np.mean((active_quad[:, 1::2]), axis=1)
active_quad_even_all.append(active_quad_even)
active_quad_odd_all.append(active_quad_odd)
# active_quad_even_all_outlier_filt.append(active_quad_even_outlier_filt)
#active_quad_odd_all_outlier_filt.append(active_quad_odd_outlier_filt)
quad_dir = quads[i]
#----------------------------------------------------------------#
# Ok, let's plot the histogram of saved quads
all_frames_hist = 'all_frames_hist'
save_dir_image = os.path.join(save_dir, quad_dir, all_frames_hist)
if not os.path.exists(save_dir_image):
os.makedirs(save_dir_image)
# separate out even and odd lines
even_samples_all = tsoc_all[:, :, ::2]
odd_samples_all = tsoc_all[:, :, 1::2]
even_samples_avg = np.mean(even_samples_all, axis=0)
odd_samples_avg = np.mean(odd_samples_all, axis=0)
# tsoc_even_all.append(np.mean((even_samples_avg)))
# tsoc_even_all_outlier_filt.append(np.mean(filter_outlier_median(even_samples_avg)))
# tsoc_odd_all.append(np.mean((odd_samples_avg)))
# tsoc_odd_all_outlier_filt.append(np.mean(filter_outlier_median(odd_samples_avg)))
title = 'Histogram of Serial Overclocks (All 100 Frames)\n ' + quads[
i] + ', ' + string2 + str(int_time) #+ r" $\mu$" +'secs'
figure_name = save_dir_image + '/' + string1 + str(
int_time) + '_image.png'
plot_hist(even_samples_all, odd_samples_all, title, figure_name)
avg_frames_hist = 'avg_frames_hist'
save_dir_image = os.path.join(save_dir, quad_dir, avg_frames_hist)
if not os.path.exists(save_dir_image):
os.makedirs(save_dir_image)
title = 'Histogram of Serial Overclocks (Avg. of 100 Frames)\n ' + quads[
i] + ', ' + string2 + str(int_time) #+ r" $\mu$" +'secs'
figure_name = save_dir_image + '/' + string1 + str(
int_time) + '_image.png'
plot_hist(even_samples_avg, odd_samples_avg, title, figure_name)
final_two_lines = 'final_two_lines'
save_dir_image = os.path.join(save_dir, quad_dir, final_two_lines)
if not os.path.exists(save_dir_image):
os.makedirs(save_dir_image)
even_samples_used = np.mean(even_samples_avg, axis=1)
unct_spectral_even.append(
np.std(even_samples_used) / np.mean(even_samples_used))
odd_samples_used = np.mean(odd_samples_avg, axis=1)
unct_spectral_odd.append(
np.std(odd_samples_used) / np.mean(odd_samples_used))
title = 'Histogram of Serial Overclocks (Avg for even and odd lines)\n ' + quads[
i] + ', ' + string2 + str(int_time) #+ r" $\mu$" +'secs'
figure_name = save_dir_image + '/' + string1 + str(
int_time) + '_image.png'
#plot_hist(even_samples_used, odd_samples_used, title, figure_name)
tsoc_profile = 'tsoc_plot'
save_tsoc_profile = os.path.join(save_dir, quad_dir, tsoc_profile)
if not os.path.exists(save_tsoc_profile):
os.makedirs(save_tsoc_profile)
figure_name = save_tsoc_profile + '/' + string1 + str(
int_time) + '_image.png'
title = 'Profile of Serial Overclocks ' + quads[
i] + ', ' + string2 + str(int_time) #+ r" $\mu$" +'secs'
even_samples_mean, odd_samples_mean = plot_few_tsocs(
even_samples_avg, odd_samples_avg, figure_name, title)
# save average in spatial direction
tsoc_even_all.append(even_samples_mean)
tsoc_odd_all.append(odd_samples_mean)
odd_even_lines = [even_samples_avg, odd_samples_avg]
odd_even_lines_name = ['Even Lines', 'Odd Lines']
for k in np.arange(0, 2):
# Lets make directory of
#k=1
median_plot = r'Outlier_median_tsoc' + '/' + odd_even_lines_name[
k]
folder_name_hist = 'Hist_plot'
folder_name_mask = 'Mask_plot'
folder_name_sat = 'Saturation_plot'
save_median_hist = os.path.join(save_dir, quad_dir,
median_plot, folder_name_hist)
if not os.path.exists(save_median_hist):
os.makedirs(save_median_hist)
save_median_mask = os.path.join(save_dir, quad_dir,
median_plot, folder_name_mask)
if not os.path.exists(save_median_mask):
os.makedirs(save_median_mask)
save_sat_mask = os.path.join(save_dir, quad_dir, median_plot,
folder_name_sat)
if not os.path.exists(save_sat_mask):
os.makedirs(save_sat_mask)
figure_name = save_sat_mask + '/' + string1 + str(
int_time) + '_image.png'
sat_pixels = identify_saturation(odd_even_lines[k], int_time,
figure_name)
outlier_filt_med, outlier_med = reject_outlier_median(
odd_even_lines[k])
if len(outlier_filt_med) == 1:
outlier_med = 0
else:
outlier_med = outlier_med[1]
title = 'Binary Outlier Mask of Trailing Overclocks (' + quads[
i] + ', ' + odd_even_lines_name[k] + ', ' + string2 + str(
int_time) #+' micro secs)'
figure_name = save_median_mask + '/' + string1 + str(
int_time) + '_image.png'
median_mask = create_outlier_mask(odd_even_lines[k],
outlier_med, title,
figure_name)
title = 'Histogram of Trailing Overclocks (' + quads[
i] + ', ' + odd_even_lines_name[k] + ', ' + string2 + str(
int_time) + ')'
title1 = 'outliers = ' + str(outlier_med.shape[0])
figure_name = save_median_hist + '/' + string1 + str(
int_time) + '_image.png'
xlim_low = [800, 825]
xlim_high = [815, 840]
#plot_hist_image(odd_even_lines[k], title, title1, outlier_filt_med, outlier_med, figure_name, xlim_low[k], xlim_high[k])
# print(all_int_time)
# print(active_quad_even_all_outlier_filt)
# print(active_quad_odd_all_outlier_filt)
# print(tsoc_even_all_outlier_filt)
# print(tsoc_odd_all_outlier_filt)
#
#dframe1 = pd.DataFrame(
# {'Avg_Active_Quad_even' : active_quad_even_all,
# 'Avg_Active_Quad_odd' : active_quad_odd_all,
# 'Avg_tsoc_Quad_even' : tsoc_even_all,
# 'Avg_tsoc_Quad_odd': tsoc_odd_all
# })
# dframe2 = pd.DataFrame(
# {'Int_time.' : all_int_time,
# 'Avg_Active_Quad_even' : active_quad_even_all_outlier_filt,
# 'Avg_Active_Quad_odd' : active_quad_odd_all_outlier_filt,
# 'Avg_tsoc_Quad_even' : tsoc_even_all_outlier_filt,
# 'Avg_tsoc_Quad_odd': tsoc_odd_all_outlier_filt
# })
# dframe3 = pd.DataFrame(
# {'Int_time.' : all_int_time,
# 'Unct_tsoc_Quad_even' : unct_spectral_even,
# 'Unct_tsoc_Quad_odd': unct_spectral_odd
# })
data_to_be_saved = np.concatenate(
(active_quad_even_all, active_quad_odd_all, tsoc_even_all,
tsoc_odd_all),
axis=0)
csv_save_dir = os.path.join(save_dir, quad_dir)
if not os.path.exists(csv_save_dir):
os.makedirs(csv_save_dir)
csv_file_name1 = csv_save_dir + '/' + quads[
i] + '_Signal_dependent_offset_more_outliers.csv'
np.savetxt(csv_file_name1,
np.array(data_to_be_saved).T,
delimiter=',',
fmt='%1.2f')
def main():
"""
Tme main function
"""
#nx_quad = 1056 # For Tempo
#ny_quad = 1046 # For Tempo
#nlat = nx_quad*2
#nspec = ny_quad*2
file_path = r'F:\TEMPO\Data\GroundTest\FPS\Spectrometer\2017.06.28\saved_quads'
save_dir = r'C:\Users\nmishra\Workspace\TEMPO\outlier_mask_spectrometer'
all_int_files = [
each for each in os.listdir(file_path) if each.endswith('.sav')
]
#data_path_all = [items for items in all_int_files if not items.endswith('_INT_0.dat.sav')]
#names_to_check = ['SHORT', 'NOM', 'LONG']
#names_to_check = ['SHORT', 'NOM','OP', 'LONG']
median_mask_A = []
median_mask_B = []
median_mask_C = []
median_mask_D = []
saturation_mask_A = []
saturation_mask_B = []
saturation_mask_C = []
saturation_mask_D = []
for data_path in all_int_files:
print(data_path)
data_file = os.path.join(file_path, data_path)
IDL_variable = readsav(data_file)
full_frame = IDL_variable.quads
active_quad_A = full_frame[0, 4:1028, 10:1034]
active_quad_B = full_frame[1, 4:1028, 10:1034]
active_quad_C = full_frame[2, 4:1028, 10:1034]
active_quad_D = full_frame[3, 4:1028, 10:1034]
active_quads = [
active_quad_A, active_quad_B, active_quad_C, active_quad_D
]
quads = ['Quad A', 'Quad B', 'Quad C', 'Quad D']
# Now let's save the full_frame images
int_time = 139.97
collection_type = 'Nom Int.'
frames = data_path.split('.')[0]
for k in np.arange(0, 4):
#mean_plot = r'Outlier_mean_tsoc'+'/'+ quads[k]
median_plot = r'Outlier_median' + '/' + quads[k]
image_dir = 'Saved_Images'
folder_name_hist = 'Hist_plot'
folder_name_mask = 'Mask_plot'
folder_name_sat = 'Saturation_plot'
save_median_hist = os.path.join(save_dir, median_plot,
folder_name_hist)
if not os.path.exists(save_median_hist):
os.makedirs(save_median_hist)
saved_images = os.path.join(save_dir, median_plot, image_dir)
if not os.path.exists(saved_images):
os.makedirs(saved_images)
title = quads[0] + 'image (Spectrometer)'
figure_name = saved_images + '/' + frames + '.png'
create_image(active_quads[k], title, figure_name)
save_median_mask = os.path.join(save_dir, median_plot,
folder_name_mask)
if not os.path.exists(save_median_mask):
os.makedirs(save_median_mask)
save_sat_mask = os.path.join(save_dir, median_plot,
folder_name_sat)
if not os.path.exists(save_sat_mask):
os.makedirs(save_sat_mask)
# Now let's call the outlier functions. First identify saturated
#bits and then identify the 'hot and cold' outlier pixels
title = 'Histogram of ' + quads[k]
sat_pixels = identify_saturation(active_quads[k], save_sat_mask,
collection_type, frames, int_time)
#outlier_filt_mean, outlier_mean = reject_outlier_mean(active_quads[k])
outlier_filt_med, outlier_med = reject_outlier_median(
active_quads[k])
# Ok now lets' plot the histograms to understand what we see.
# First the mean approach
print('ok, let us plot the histograms now')
plot_hist_image(active_quads[k], title, collection_type, frames,
outlier_filt_med, outlier_med, int_time,
save_median_hist)
if len(outlier_med) == 1:
outlier_med = 0
else:
outlier_med = outlier_med[1]
title = 'Binary Outlier Mask of ' + quads[k]
median_mask = create_outlier_mask(active_quads[k], outlier_med,
title, collection_type, frames,
save_median_mask)
active_quads[k] = None
outlier_med = None
if k == 0:
median_mask_A.append(median_mask[:])
median_mask = None
saturation_mask_A.append(sat_pixels[:])
sat_pixels = None
elif k == 1:
median_mask_B.append(median_mask[:])
median_mask = None
saturation_mask_B.append(sat_pixels[:])
sat_pixels = None
elif k == 2:
median_mask_C.append(median_mask[:])
median_mask = None
saturation_mask_C.append(sat_pixels[:])
sat_pixels = None
elif k == 3:
median_mask_D.append(median_mask[:])
median_mask = None
saturation_mask_D.append(sat_pixels[:])
sat_pixels = None
#***************************************************************************
# The following function creates final mask based on 80% occurence criteria
final_path = r'C:\Users\nmishra\Workspace\TEMPO\outlier_mask_spectrometer\Outlier_median\save_masks_80%'
if not os.path.exists(final_path):
os.makedirs(final_path)
quad_name = 'Quad A'
title = 'Quad A outlier mask (50% criteria)'
final_mask_A = create_final_mask(median_mask_A, quad_name, title,
final_path)
mask_name_A = final_path + '/' + 'quad_A_outlier_mask.csv'
np.savetxt(mask_name_A, np.array(final_mask_A), delimiter=",")
quad_name = 'Quad B'
title = 'Quad B outlier mask (50% criteria)'
final_mask_B = create_final_mask(median_mask_B, quad_name, title,
final_path)
mask_name_B = final_path + '/' + 'quad_B_outlier_mask.csv'
np.savetxt(mask_name_B, np.array(final_mask_B), delimiter=",")
quad_name = 'Quad C'
title = 'Quad C outlier mask (50% criteria)'
final_mask_C = create_final_mask(median_mask_C, quad_name, title,
final_path)
mask_name_C = final_path + '/' + 'quad_C_outlier_mask.csv'
np.savetxt(mask_name_C, np.array(final_mask_C), delimiter=",")
quad_name = 'Quad D'
title = 'Quad D outlier mask (50% criteria)'
final_mask_D = create_final_mask(median_mask_D, quad_name, title,
final_path)
mask_name_D = final_path + '/' + 'quad_D_outlier_mask.csv'
np.savetxt(mask_name_D, np.array(final_mask_D), delimiter=",")
lower_quads = np.concatenate((final_mask_A, final_mask_B), axis=1)
upper_quads = np.concatenate((final_mask_D, final_mask_C), axis=1)
final_outlier_mask = np.concatenate((lower_quads, upper_quads), axis=0)
mask_name = final_path + '/' + 'final_outlier_mask_80%.csv'
np.savetxt(mask_name, np.array(final_outlier_mask), delimiter=",")
plt.figure()
final_outlier_mask = np.invert(final_outlier_mask)
plt.imshow(final_outlier_mask,
cmap='bwr',
interpolation='none',
origin='lower')
#cbar = plt.colorbar(image)
nx_quad, ny_quad = np.array(final_outlier_mask).shape
final_outliers = np.array(
np.where(np.reshape(final_outlier_mask, (nx_quad * ny_quad,
1)) == 0)).shape[1]
plt.title(
'TEMPO outlier Mask (50% Occurence Criteria)\n (Num. outliers = ' +
str(final_outliers) + ')',
fontsize=14)
plt.xlabel('# of spatial pixels', fontsize=12)
plt.ylabel('# of spectral pixels', fontsize=12)
plt.grid(False)
plt.savefig(final_path + '/' + 'final_mask_50%.png',
dpi=200,
bbox_inches="tight")
#***************************************************************************
#***************************************************************************
# if we want to 'OR" all the mask, use the following function
final_path = r'C:\Users\nmishra\Workspace\TEMPO\outlier_mask_spectrometer\Outlier_median\save_masks_logical_OR'
if not os.path.exists(final_path):
os.makedirs(final_path)
quad_name = 'Quad A'
title = 'Quads A outlier mask (Logical OR)'
final_mask_A = create_ORed_mask(median_mask_A, quad_name, title,
final_path)
mask_name_A = final_path + '/' + 'quad_A_outlier_mask_ored.csv'
np.savetxt(mask_name_A, np.array(final_mask_A), delimiter=",")
quad_name = 'Quad B'
title = 'Quads B outlier mask (Logical OR)'
final_mask_B = create_ORed_mask(median_mask_B, quad_name, title,
final_path)
mask_name_B = final_path + '/' + 'quad_B_outlier_mask_ored.csv'
np.savetxt(mask_name_B, np.array(final_mask_B), delimiter=",")
quad_name = 'Quad C'
title = 'Quads C outlier mask (Logical OR)'
final_mask_C = create_ORed_mask(median_mask_C, quad_name, title,
final_path)
mask_name_C = final_path + '/' + 'quad_C_outlier_mask_ored.csv'
np.savetxt(mask_name_C, np.array(final_mask_C), delimiter=",")
quad_name = 'Quad D'
title = 'Quads D outlier mask (Logical OR)'
final_mask_D = create_ORed_mask(median_mask_D, quad_name, title,
final_path)
mask_name_D = final_path + '/' + 'quad_D_outlier_mask_ored.csv'
np.savetxt(mask_name_D, np.array(final_mask_D), delimiter=",")
quad_name = 'upper_quads_ored'
title = 'Quads C & D outlier mask (Logical OR)'
lower_quads = np.concatenate((final_mask_A, final_mask_B), axis=1)
upper_quads = np.concatenate((final_mask_D, final_mask_C), axis=1)
final_outlier_mask_ored = np.concatenate((lower_quads, upper_quads),
axis=0)
mask_name = final_path + '/' + 'final_outlier_mask_ored.csv'
np.savetxt(mask_name, np.array(final_outlier_mask_ored), delimiter=",")
plt.figure()
plt.imshow(np.invert(final_outlier_mask_ored),
cmap='bwr',
interpolation='none',
origin='lower')
nx_quad, ny_quad = np.array(final_outlier_mask_ored).shape
final_outliers_ored = np.array(
np.where(
np.reshape(final_outlier_mask_ored, (nx_quad * ny_quad,
1)) == 1)).shape[1]
plt.title('TEMPO Outlier Mask (Logical OR)\n (Num. outliers = ' +
str(final_outliers_ored) + ')',
fontsize=14)
plt.xlabel('# of spatial pixels', fontsize=12)
plt.ylabel('# of spectral pixels', fontsize=12)
plt.grid(False)
plt.savefig(final_path + '/' + 'final_mask_ored.png',
dpi=200,
bbox_inches="tight")
def main():
"""
Tme main function
"""
#nx_quad = 1056 # For Tempo
#ny_quad = 1046 # For Tempo
#nlat = nx_quad*2
#nspec = ny_quad*2
file_path = r'C:\Users\nmishra\Workspace\TEMPO\Data\GroundTest\FPS\Integration_Sweep\Dark'
save_dir = r'C:\Users\nmishra\Workspace\TEMPO\outlier_mask_3_sigma'
data_path_all = [
each for each in os.listdir(file_path) if each.endswith('.sav')
]
#data_path_all = [items for items in all_int_files if not items.endswith('.dat.sav')]
#names_to_check = ['SHORT', 'NOM', 'LONG']
names_to_check = ['SHORT', 'NOM', 'OP', 'LONG']
median_mask_A = []
median_mask_B = []
median_mask_C = []
median_mask_D = []
saturation_mask_A = []
saturation_mask_B = []
saturation_mask_C = []
saturation_mask_D = []
for data_path in data_path_all:
print(data_path)
full_frame, collection_type, frames,int_time = \
make_quads_from_IDL_sav_file(file_path, data_path, names_to_check)
# all_full_frame contains all 100 frames
#Let's fetch the quads now
#quad_A = full_frame[0:ny_quad, 0:nx_quad]
quad_A = full_frame[:, 0, :, :]
active_quad_A = np.mean(quad_A[:, 2:1030, 10:1034], axis=0)
quad_B = full_frame[:, 1, :, :]
active_quad_B = np.mean(quad_B[:, 2:1030, 10:1034], axis=0)
quad_C = full_frame[:, 2, :, :]
active_quad_C = np.mean(quad_C[:, 2:1030, 10:1034], axis=0)
quad_D = full_frame[:, 3, :, :]
active_quad_D = np.mean(quad_D[:, 2:1030, 10:1034], axis=0)
#lower_quads = np.concatenate((active_quad_A, active_quad_B), axis=1)
#upper_quads = np.concatenate((active_quad_D, active_quad_C), axis=1)
#active_quads =[lower_quads, upper_quads]
# for overclocks
#lower_quads = np.concatenate((bias_A, bias_B), axis=1)
#upper_quads = np.concatenate((bias_D, bias_C), axis=1)
active_quads = [
active_quad_A, active_quad_B, active_quad_C, active_quad_D
]
quads = ['Quad A', 'Quad B', 'Quad C', 'Quad D']
# Now let's save the full_frame images
plot_dir = save_dir
image_dir = 'Quad_images_'
save_quad_images = os.path.join(plot_dir, image_dir)
# if not os.path.exists(save_quad_images):
# os.makedirs(save_quad_images)
# image_title = 'Full Frame Quad Image'
# plot_full_frame_image(full_frame, image_title, collection_type, frames,
# int_time, save_quad_images)
#
# Create list of directories To save histograms before and after
#outlier rejections
for k in np.arange(0, 4):
#mean_plot = r'Outlier_mean_tsoc'+'/'+ quads[k]
median_plot = r'Outlier_median' + '/' + quads[k]
folder_name_hist = 'Hist_plot'
folder_name_mask = 'Mask_plot'
folder_name_sat = 'Saturation_plot'
save_median_hist = os.path.join(plot_dir, median_plot,
folder_name_hist)
if not os.path.exists(save_median_hist):
os.makedirs(save_median_hist)
save_median_mask = os.path.join(plot_dir, median_plot,
folder_name_mask)
if not os.path.exists(save_median_mask):
os.makedirs(save_median_mask)
save_sat_mask = os.path.join(plot_dir, median_plot,
folder_name_sat)
if not os.path.exists(save_sat_mask):
os.makedirs(save_sat_mask)
# Now let's call the outlier functions. First identify saturated
#bits and then identify the 'hot and cold' outlier pixels
title = 'Histogram of ' + quads[k]
sat_pixels = identify_saturation(active_quads[k], save_sat_mask,
collection_type, frames, int_time)
#outlier_filt_mean, outlier_mean = reject_outlier_mean(active_quads[k])
outlier_filt_med, outlier_med = reject_outlier_median(
active_quads[k])
# Ok now lets' plot the histograms to understand what we see.
# First the mean approach
#print('ok, let us plot the histograms now')
plot_hist_image(active_quads[k], title, collection_type, frames,
outlier_filt_med, outlier_med, int_time,
save_median_hist)
if len(outlier_med) == 1:
outlier_med = 0
else:
outlier_med = outlier_med[1]
title = 'Binary Outlier Mask of ' + quads[k]
median_mask = create_outlier_mask(active_quads[k], outlier_med,
title, collection_type, frames,
save_median_mask)
#print(median_mask.max(), median_mask.min())
#cc
active_quads[k] = None
outlier_med = None
if k == 0:
median_mask_A.append(median_mask[:])
median_mask = None
saturation_mask_A.append(sat_pixels[:])
sat_pixels = None
elif k == 1:
median_mask_B.append(median_mask[:])
median_mask = None
saturation_mask_B.append(sat_pixels[:])
sat_pixels = None
elif k == 2:
median_mask_C.append(median_mask[:])
median_mask = None
saturation_mask_C.append(sat_pixels[:])
sat_pixels = None
elif k == 3:
median_mask_D.append(median_mask[:])
median_mask = None
saturation_mask_D.append(sat_pixels[:])
sat_pixels = None
#***************************************************************************
# The following function creates final mask based on 80% occurence criteria
final_path = r'C:\Users\nmishra\Workspace\TEMPO\outlier_mask\Outlier_median\save_masks_80\to_SAO%'
if not os.path.exists(final_path):
os.makedirs(final_path)
quad_name = 'Quad A'
title = 'Quad A outlier mask (50% criteria)'
final_mask_A = create_final_mask(median_mask_A, quad_name, title,
final_path)
#print(np.array(final_mask_A).max(), np.array( final_mask_A).min())
mask_name_A = final_path + '/' + 'quad_A_outlier_mask.csv'
np.savetxt(mask_name_A, np.array(final_mask_A), delimiter=",")
#cc
quad_name = 'Quad B'
title = 'Quad B outlier mask (50% criteria)'
final_mask_B = create_final_mask(median_mask_B, quad_name, title,
final_path)
mask_name_B = final_path + '/' + 'quad_B_outlier_mask.csv'
np.savetxt(mask_name_B, np.array(final_mask_B), delimiter=",")
quad_name = 'Quad C'
title = 'Quad C outlier mask (50% criteria)'
final_mask_C = create_final_mask(median_mask_C, quad_name, title,
final_path)
mask_name_C = final_path + '/' + 'quad_C_outlier_mask.csv'
np.savetxt(mask_name_C, np.array(final_mask_C), delimiter=",")
quad_name = 'Quad D'
title = 'Quad D outlier mask (50% criteria)'
final_mask_D = create_final_mask(median_mask_D, quad_name, title,
final_path)
mask_name_D = final_path + '/' + 'quad_D_outlier_mask.csv'
np.savetxt(mask_name_D, np.array(final_mask_D), delimiter=",")
lower_quads = np.concatenate((final_mask_A, final_mask_B), axis=1)
upper_quads = np.concatenate((final_mask_D, final_mask_C), axis=1)
final_outlier_mask = np.concatenate((lower_quads, upper_quads), axis=0)
mask_name = final_path + '/' + 'final_outlier_mask_80%.csv'
np.savetxt(mask_name, np.array(final_outlier_mask), delimiter=",")
plt.figure()
final_outlier_mask = np.invert(final_outlier_mask)
plt.imshow(final_outlier_mask,
cmap='bwr',
interpolation='none',
origin='lower')
#cbar = plt.colorbar(image)
nx_quad, ny_quad = np.array(final_outlier_mask).shape
final_outliers = np.array(
np.where(np.reshape(final_outlier_mask, (nx_quad * ny_quad,
1)) == 0)).shape[1]
plt.title(
'TEMPO outlier Mask (50% Occurence Criteria)\n (Num. outliers = ' +
str(final_outliers) + ')',
fontsize=14)
plt.xlabel('# of spatial pixels', fontsize=12)
plt.ylabel('# of spectral pixels', fontsize=12)
plt.grid(False)
plt.savefig(final_path + '/' + 'final_mask_50%.png',
dpi=200,
bbox_inches="tight")
#***************************************************************************
#***************************************************************************
# if we want to 'OR" all the mask, use the following function
final_path = r'C:\Users\nmishra\Workspace\TEMPO\outlier_mask\Outlier_median\save_masks_logical_OR\to_SAO'
if not os.path.exists(final_path):
os.makedirs(final_path)
quad_name = 'Quad A'
title = 'Quads A outlier mask (Logical OR)'
final_mask_A = create_ORed_mask(median_mask_A, quad_name, title,
final_path)
mask_name_A = final_path + '/' + 'quad_A_outlier_mask_ored.csv'
np.savetxt(mask_name_A, np.array(final_mask_A), delimiter=",")
quad_name = 'Quad B'
title = 'Quads B outlier mask (Logical OR)'
final_mask_B = create_ORed_mask(median_mask_B, quad_name, title,
final_path)
mask_name_B = final_path + '/' + 'quad_B_outlier_mask_ored.csv'
np.savetxt(mask_name_B, np.array(final_mask_B), delimiter=",")
quad_name = 'Quad C'
title = 'Quads C outlier mask (Logical OR)'
final_mask_C = create_ORed_mask(median_mask_C, quad_name, title,
final_path)
mask_name_C = final_path + '/' + 'quad_C_outlier_mask_ored.csv'
np.savetxt(mask_name_C, np.array(final_mask_C), delimiter=",")
quad_name = 'Quad D'
title = 'Quads D outlier mask (Logical OR)'
final_mask_D = create_ORed_mask(median_mask_D, quad_name, title,
final_path)
mask_name_D = final_path + '/' + 'quad_D_outlier_mask_ored.csv'
np.savetxt(mask_name_D, np.array(final_mask_D), delimiter=",")
quad_name = 'upper_quads_ored'
title = 'Quads C & D outlier mask (Logical OR)'
lower_quads = np.concatenate((final_mask_A, final_mask_B), axis=1)
upper_quads = np.concatenate((final_mask_D, final_mask_C), axis=1)
final_outlier_mask_ored = np.concatenate((lower_quads, upper_quads),
axis=0)
mask_name = final_path + '/' + 'final_outlier_mask_ored.csv'
np.savetxt(mask_name, np.array(final_outlier_mask_ored), delimiter=",")
plt.figure()
plt.imshow(np.invert(final_outlier_mask_ored),
cmap='bwr',
interpolation='none',
origin='lower')
nx_quad, ny_quad = np.array(final_outlier_mask_ored).shape
final_outliers_ored = np.array(
np.where(
np.reshape(final_outlier_mask_ored, (nx_quad * ny_quad,
1)) == 1)).shape[1]
plt.title('TEMPO Outlier Mask (Logical OR)\n (Num. outliers = ' +
str(final_outliers_ored) + ')',
fontsize=14)
plt.xlabel('# of spatial pixels', fontsize=12)
plt.ylabel('# of spectral pixels', fontsize=12)
plt.grid(False)
plt.savefig(final_path + '/' + 'final_mask_ored.png',
dpi=200,
bbox_inches="tight")