def main():
"""
Tme main function
"""
#nx_quad = 1056 # For Tempo
#ny_quad = 1046 # For Tempo
#nlat = nx_quad*2
#nspec = ny_quad*2
file_path = r'C:\Users\nmishra\Workspace\TEMPO\Data\GroundTest\FPS\Dark\FPS_Dark\saved_quads'
data_path_all = [
each for each in os.listdir(file_path) if each.endswith('.sav')
]
#names_to_check = ['SHORT', 'NOM', 'LONG']
names_to_check = ['SHORT', 'NOM', 'OP', 'LONG']
mean_mask_all_lower = []
median_mask_all_lower = []
mean_mask_all_upper = []
median_mask_all_upper = []
saturation_mask_upper = []
saturation_mask_lower = []
for data_path in data_path_all:
# Now let's find the integtration type
print(data_path)
full_frame, collection_type, frames, int_time = \
make_quads_from_IDL_sav_file(file_path, data_path, names_to_check)
#Let's fetch the quads now
#quad_A = full_frame[0:ny_quad, 0:nx_quad]
active_quad_A = np.array(full_frame[4:1028, 10:1034])
#quad_B = full_frame[ny_quad:, 0:nx_quad]
active_quad_B = np.array(full_frame[4:1028, 1078:2102])
#quad_C = full_frame[ny_quad:, nx_quad:]
active_quad_C = np.array(full_frame[1064:2088, 1078:2102])
#quad_D = full_frame[ny_quad:, 0:nx_quad]
active_quad_D = np.array(full_frame[1064:2088, 10:1034])
lower_quads = np.concatenate((active_quad_A, active_quad_B), axis=1)
upper_quads = np.concatenate((active_quad_D, active_quad_C), axis=1)
active_quads = [lower_quads, upper_quads]
#quads = ['Lower_quads','Upper_quads']
# Now let's save the full_frame images
plot_dir = file_path
image_dir = 'Quad_images'
save_quad_images = os.path.join(plot_dir, image_dir)
if not os.path.exists(save_quad_images):
os.makedirs(save_quad_images)
image_title = 'Full Frame Quad Image'
plot_full_frame_image(full_frame, image_title, collection_type, frames,
int_time, save_quad_images)
# Create list of directories To save histograms before and after
#outlier rejections
mean_plot = r'Outlier_mean' + '/' + 'Lower_quads'
median_plot = r'Outlier_median' + '/' + 'Lower_quads'
folder_name_hist = 'Hist_plot'
folder_name_mask = 'Mask_plot'
folder_name_sat = 'Saturation_plot'
save_mean_hist = os.path.join(plot_dir, mean_plot, folder_name_hist)
if not os.path.exists(save_mean_hist):
os.makedirs(save_mean_hist)
save_median_hist = os.path.join(plot_dir, median_plot,
folder_name_hist)
if not os.path.exists(save_median_hist):
os.makedirs(save_median_hist)
save_mean_mask = os.path.join(plot_dir, mean_plot, folder_name_mask)
if not os.path.exists(save_mean_mask):
os.makedirs(save_mean_mask)
save_median_mask = os.path.join(plot_dir, median_plot,
folder_name_mask)
if not os.path.exists(save_median_mask):
os.makedirs(save_median_mask)
save_sat_mask = os.path.join(plot_dir, median_plot, folder_name_sat)
if not os.path.exists(save_sat_mask):
os.makedirs(save_sat_mask)
# Now let's call the outlier functions. First identify saturated
#bits and then identify the 'hot and cold' outlier pixels
title = 'Histogram of Quads A & B'
#title = 'Histogram of Quads C & D'
sat_pixels = identify_saturation(lower_quads, save_sat_mask,
collection_type, frames, int_time)
#outlier_filt_mean, outlier_mean = reject_outlier_mean(active_quads[k])
outlier_filt_med, outlier_med = reject_outlier_median(lower_quads)
# Ok now lets' plot the histograms to understand what we see.
# First the mean approach
print('ok, let us plot the histograms now')
#plot_hist_image(active_quads[k], title, collection_type, frames,
#outlier_filt_mean, outlier_mean, int_time,
# save_mean_hist)
# Secondly the median absolute difference approach
plot_hist_image(lower_quads, title, collection_type, frames,
outlier_filt_med, outlier_med, int_time,
save_median_hist)
#outlier_med= None
# Ok, lets now create a binary mask of outlier pixels
# First with the mean based approach
# mean_mask = create_outlier_mask(active_quads[k], outlier_mean[0],
# collection_type,
# frames, save_mean_mask)
# if k == 0:
# mean_mask_all_lower.append(mean_mask)
#
# else :
# mean_mask_all_upper.append(mean_mask)
#
# # Secondly, the same for median ased approach
#if len(outlier_med) == 1:
# outlier_med = 0
#else:
#outlier_med = outlier_med[1]
median_mask = create_outlier_mask(lower_quads, outlier_med[1],
collection_type, frames,
save_median_mask)
lower_quads = None
outlier_med = None
median_mask_all_lower.append(median_mask[:])
median_mask = None
saturation_mask_lower.append(sat_pixels[:])
# Ok now, lets' or all the median masks
print(np.array(median_mask_all_lower).shape)
final_mask_median_lower = np.bitwise_or.reduce(median_mask_all_lower[:])
plt.figure()
plt.imshow(np.invert(final_mask_median_lower),
cmap='bwr',
interpolation='none',
origin='lower')
#cbar = plt.colorbar(image)
plt.title('Binary outlier mask', fontsize=14)
plt.xlabel('# of spatial pixels', fontsize=12)
plt.ylabel('# of spectral pixels', fontsize=12)
plt.grid(False)
plt.savefig(r'C:\Users\nmishra\Desktop' + '/1.png',
dpi=100,
bbox_inches="tight")
cc
plt.figure()
plt.imshow(np.invert(median_mask_all_lower[1]),
cmap='bwr',
interpolation='none',
origin='lower')
#cbar = plt.colorbar(image)
plt.title('Binary outlier mask', fontsize=14)
plt.xlabel('# of spatial pixels', fontsize=12)
plt.ylabel('# of spectral pixels', fontsize=12)
plt.grid(False)
plt.savefig(r'C:\Users\nmishra\Desktop' + '/2.png',
dpi=100,
bbox_inches="tight")
def main():
"""
Tme main function
"""
#nx_quad = 1056 # For Tempo
#ny_quad = 1046 # For Tempo
#nlat = nx_quad*2
#nspec = ny_quad*2
file_path = r'C:\Users\nmishra\Workspace\TEMPO\Data\GroundTest\FPS\Integration_Sweep\Dark\saved_quads'
save_dir = r'C:\Users\nmishra\Workspace\TEMPO\outlier_mask'
data_path_all = [each for each in os.listdir(file_path) if each.endswith('.sav')]
#names_to_check = ['SHORT', 'NOM', 'LONG']
names_to_check = ['SHORT', 'NOM','OP', 'LONG']
mean_mask_all_lower = []
median_mask_all_lower = []
mean_mask_all_upper = []
median_mask_all_upper = []
saturation_mask_upper = []
saturation_mask_lower = []
for data_path in data_path_all:
print(data_path)
full_frame, collection_type, frames,int_time = \
make_quads_from_IDL_sav_file(file_path, data_path, names_to_check)
# all_full_frame contains all 100 frames
#Let's fetch the quads now
#quad_A = full_frame[0:ny_quad, 0:nx_quad]
quad_A = full_frame[:, 0, :, :]
active_quad_A = np.mean(quad_A[:, 4:1028, 10:1034], axis=0)
quad_B = full_frame[:, 1, :, :]
active_quad_B = np.mean(quad_B[:, 4:1028, 10:1034], axis=0)
quad_C = full_frame[:, 2, :, :]
active_quad_C = np.mean(quad_C[:, 4:1028, 10:1034], axis=0)
quad_D = full_frame[:, 3, :, :]
active_quad_D = np.mean(quad_D[:, 4:1028, 10:1034], axis=0)
lower_quads = np.concatenate((active_quad_A, active_quad_B), axis=1)
upper_quads = np.concatenate((active_quad_D, active_quad_C), axis=1)
active_quads =[lower_quads, upper_quads]
# for overclocks
#lower_quads = np.concatenate((bias_A, bias_B), axis=1)
#upper_quads = np.concatenate((bias_D, bias_C), axis=1)
active_quads =[lower_quads, upper_quads]
quads = ['Lower_quads','Upper_quads']
# Now let's save the full_frame images
plot_dir = save_dir
image_dir = 'Quad_images_'
save_quad_images = os.path.join(plot_dir, image_dir)
if not os.path.exists(save_quad_images):
os.makedirs(save_quad_images)
image_title = 'Full Frame Quad Image'
# plot_full_frame_image(full_frame, image_title, collection_type, frames,
# int_time, save_quad_images)
#
# Create list of directories To save histograms before and after
#outlier rejections
for k in np.arange(0,2):
#mean_plot = r'Outlier_mean_tsoc'+'/'+ quads[k]
median_plot = r'Outlier_median'+'/'+ quads[k]
folder_name_hist = 'Hist_plot'
folder_name_mask = 'Mask_plot'
folder_name_sat = 'Saturation_plot'
save_median_hist = os.path.join(plot_dir, median_plot, folder_name_hist)
if not os.path.exists(save_median_hist):
os.makedirs(save_median_hist)
save_median_mask = os.path.join(plot_dir, median_plot, folder_name_mask)
if not os.path.exists(save_median_mask):
os.makedirs(save_median_mask)
save_sat_mask = os.path.join(plot_dir, median_plot, folder_name_sat)
if not os.path.exists(save_sat_mask):
os.makedirs(save_sat_mask)
# Now let's call the outlier functions. First identify saturated
#bits and then identify the 'hot and cold' outlier pixels
if k == 0:
title = 'Histogram of Quads A & B'
else:
title = 'Histogram of Quads C & D'
sat_pixels = identify_saturation(active_quads[k],
save_sat_mask,
collection_type,
frames, int_time)
#outlier_filt_mean, outlier_mean = reject_outlier_mean(active_quads[k])
outlier_filt_med, outlier_med = reject_outlier_median(active_quads[k])
# Ok now lets' plot the histograms to understand what we see.
# First the mean approach
print('ok, let us plot the histograms now')
#plot_hist_image(active_quads[k], title, collection_type, frames,
#outlier_filt_mean, outlier_mean, int_time,
# save_mean_hist)
# Secondly the median absolute difference approach
plot_hist_image(active_quads[k], title, collection_type, frames,
outlier_filt_med, outlier_med, int_time,
save_median_hist)
#outlier_med= None
# Ok, lets now create a binary mask of outlier pixels
# First with the mean based approach
# mean_mask = create_outlier_mask(active_quads[k], outlier_mean[0],
# collection_type,
# frames, save_mean_mask)
# if k == 0:
# mean_mask_all_lower.append(mean_mask)
#
# else :
# mean_mask_all_upper.append(mean_mask)
#
# # Secondly, the same for median ased approach
if len(outlier_med) == 1:
outlier_med = 0
else:
outlier_med = outlier_med[1]
if k == 0:
title = 'Binary Outlier Mask of Quads A & B'
else:
title = 'Binary Outlier Mask of Quads C & D'
median_mask = create_outlier_mask(active_quads[k], outlier_med,
title, collection_type, frames,
save_median_mask)
active_quads[k] = None
outlier_med = None
if k == 0:
median_mask_all_lower.append(median_mask[:])
median_mask=None
saturation_mask_lower.append(sat_pixels[:])
sat_pixels = None
elif k==1 :
median_mask_all_upper.append(median_mask[:])
median_mask = None
saturation_mask_upper.append(sat_pixels[:])
sat_pixels = None
#***************************************************************************
# The following function creates final mask based on 80% occurence criteria
final_path = r'C:\Users\nmishra\Workspace\TEMPO\outlier_mask\Outlier_median\save_masks'
if not os.path.exists(final_path):
os.makedirs(final_path)
quad_name = 'lower_quads'
title = 'Quads A & B outlier mask (80% criteria)'
final_mask_lower = create_final_mask(median_mask_all_lower, quad_name, title, final_path)
quad_name = 'upper_quads'
title = 'Quads C & D outlier mask (80% criteria)'
final_mask_upper = create_final_mask(median_mask_all_upper, quad_name, title, final_path)
final_outlier_mask = np.concatenate((final_mask_lower, final_mask_upper),
axis=0)
plt.figure()
plt.imshow(final_outlier_mask, cmap='bwr',
interpolation='none', origin='lower')
#cbar = plt.colorbar(image)
nx_quad, ny_quad = np.array(final_outlier_mask).shape
final_outliers = np.array(np.where(np.reshape(final_outlier_mask,
(nx_quad*ny_quad, 1))==0)).shape[1]
plt.title('TEMPO outlier Mask (outliers = '+ str(final_outliers)+')',
fontsize=14)
plt.xlabel('# of spatial pixels', fontsize=12)
plt.ylabel('# of spectral pixels', fontsize=12)
plt.grid(False)
#final_path = r'C:\Users\nmishra\Workspace\TEMPO\Data\GroundTest\FPS\Integration_Sweep\Dark\saved_quads\Outlier_median\save_masks'
if not os.path.exists(final_path):
os.makedirs(final_path)
plt.savefig(final_path+'/'+'final_mask.png', dpi=100, bbox_inches="tight")
#***************************************************************************
#***************************************************************************
# if we want to 'OR" all the mask, use the following function
quad_name = 'lower_quads_or'
title = 'Quads A & B outlier mask (Logical OR)'
final_mask_lower_ored= create_ORed_mask(median_mask_all_lower, quad_name, title, final_path)
quad_name = 'upper_quads_ored'
title = 'Quads C & D outlier mask (Logical OR)'
final_mask_upper_ored = create_ORed_mask(median_mask_all_upper, quad_name, title, final_path)
final_outlier_mask_ored = np.concatenate((final_mask_lower_ored,
final_mask_upper_ored), axis=0)
mask_directory = r'C:\Users\nmishra\Workspace\TEMPO\outlier_mask\Outlier_median\save_masks'
mask_name = mask_directory+'/'+'final_outlier_mask_2_sigma.csv'
np.savetxt(mask_name, np.array(final_outlier_mask_ored), delimiter =",")
plt.figure()
plt.imshow(np.invert(final_outlier_mask_ored), cmap='bwr',
interpolation='none', origin='lower')
#cbar = plt.colorbar(image)
nx_quad, ny_quad = np.array(final_outlier_mask_ored).shape
final_outliers_ored = np.array(np.where(np.reshape(final_outlier_mask_ored,
(nx_quad*ny_quad, 1))==1)).shape[1]
plt.title('TEMPO Outlier Mask (outliers = '+ str(final_outliers_ored)+')',
fontsize=14)
plt.xlabel('# of spatial pixels', fontsize=12)
plt.ylabel('# of spectral pixels', fontsize=12)
plt.grid(False)
plt.savefig(final_path+'/'+'final_mask_ored.png', dpi=100, bbox_inches="tight")
def main():
"""
Tme main function
"""
#nx_quad = 1056 # For Tempo
#ny_quad = 1046 # For Tempo
#nlat = nx_quad*2
#nspec = ny_quad*2
file_path = r'F:\TEMPO\Data\GroundTest\FPS\Spectrometer\2017.06.28\saved_quads'
save_dir = r'C:\Users\nmishra\Workspace\TEMPO\outlier_mask_spectrometer'
all_int_files = [
each for each in os.listdir(file_path) if each.endswith('.sav')
]
#data_path_all = [items for items in all_int_files if not items.endswith('_INT_0.dat.sav')]
#names_to_check = ['SHORT', 'NOM', 'LONG']
#names_to_check = ['SHORT', 'NOM','OP', 'LONG']
median_mask_A = []
median_mask_B = []
median_mask_C = []
median_mask_D = []
saturation_mask_A = []
saturation_mask_B = []
saturation_mask_C = []
saturation_mask_D = []
for data_path in all_int_files:
print(data_path)
data_file = os.path.join(file_path, data_path)
IDL_variable = readsav(data_file)
full_frame = IDL_variable.quads
active_quad_A = full_frame[0, 4:1028, 10:1034]
active_quad_B = full_frame[1, 4:1028, 10:1034]
active_quad_C = full_frame[2, 4:1028, 10:1034]
active_quad_D = full_frame[3, 4:1028, 10:1034]
active_quads = [
active_quad_A, active_quad_B, active_quad_C, active_quad_D
]
quads = ['Quad A', 'Quad B', 'Quad C', 'Quad D']
# Now let's save the full_frame images
int_time = 139.97
collection_type = 'Nom Int.'
frames = data_path.split('.')[0]
for k in np.arange(0, 4):
#mean_plot = r'Outlier_mean_tsoc'+'/'+ quads[k]
median_plot = r'Outlier_median' + '/' + quads[k]
image_dir = 'Saved_Images'
folder_name_hist = 'Hist_plot'
folder_name_mask = 'Mask_plot'
folder_name_sat = 'Saturation_plot'
save_median_hist = os.path.join(save_dir, median_plot,
folder_name_hist)
if not os.path.exists(save_median_hist):
os.makedirs(save_median_hist)
saved_images = os.path.join(save_dir, median_plot, image_dir)
if not os.path.exists(saved_images):
os.makedirs(saved_images)
title = quads[0] + 'image (Spectrometer)'
figure_name = saved_images + '/' + frames + '.png'
create_image(active_quads[k], title, figure_name)
save_median_mask = os.path.join(save_dir, median_plot,
folder_name_mask)
if not os.path.exists(save_median_mask):
os.makedirs(save_median_mask)
save_sat_mask = os.path.join(save_dir, median_plot,
folder_name_sat)
if not os.path.exists(save_sat_mask):
os.makedirs(save_sat_mask)
# Now let's call the outlier functions. First identify saturated
#bits and then identify the 'hot and cold' outlier pixels
title = 'Histogram of ' + quads[k]
sat_pixels = identify_saturation(active_quads[k], save_sat_mask,
collection_type, frames, int_time)
#outlier_filt_mean, outlier_mean = reject_outlier_mean(active_quads[k])
outlier_filt_med, outlier_med = reject_outlier_median(
active_quads[k])
# Ok now lets' plot the histograms to understand what we see.
# First the mean approach
print('ok, let us plot the histograms now')
plot_hist_image(active_quads[k], title, collection_type, frames,
outlier_filt_med, outlier_med, int_time,
save_median_hist)
if len(outlier_med) == 1:
outlier_med = 0
else:
outlier_med = outlier_med[1]
title = 'Binary Outlier Mask of ' + quads[k]
median_mask = create_outlier_mask(active_quads[k], outlier_med,
title, collection_type, frames,
save_median_mask)
active_quads[k] = None
outlier_med = None
if k == 0:
median_mask_A.append(median_mask[:])
median_mask = None
saturation_mask_A.append(sat_pixels[:])
sat_pixels = None
elif k == 1:
median_mask_B.append(median_mask[:])
median_mask = None
saturation_mask_B.append(sat_pixels[:])
sat_pixels = None
elif k == 2:
median_mask_C.append(median_mask[:])
median_mask = None
saturation_mask_C.append(sat_pixels[:])
sat_pixels = None
elif k == 3:
median_mask_D.append(median_mask[:])
median_mask = None
saturation_mask_D.append(sat_pixels[:])
sat_pixels = None
#***************************************************************************
# The following function creates final mask based on 80% occurence criteria
final_path = r'C:\Users\nmishra\Workspace\TEMPO\outlier_mask_spectrometer\Outlier_median\save_masks_80%'
if not os.path.exists(final_path):
os.makedirs(final_path)
quad_name = 'Quad A'
title = 'Quad A outlier mask (50% criteria)'
final_mask_A = create_final_mask(median_mask_A, quad_name, title,
final_path)
mask_name_A = final_path + '/' + 'quad_A_outlier_mask.csv'
np.savetxt(mask_name_A, np.array(final_mask_A), delimiter=",")
quad_name = 'Quad B'
title = 'Quad B outlier mask (50% criteria)'
final_mask_B = create_final_mask(median_mask_B, quad_name, title,
final_path)
mask_name_B = final_path + '/' + 'quad_B_outlier_mask.csv'
np.savetxt(mask_name_B, np.array(final_mask_B), delimiter=",")
quad_name = 'Quad C'
title = 'Quad C outlier mask (50% criteria)'
final_mask_C = create_final_mask(median_mask_C, quad_name, title,
final_path)
mask_name_C = final_path + '/' + 'quad_C_outlier_mask.csv'
np.savetxt(mask_name_C, np.array(final_mask_C), delimiter=",")
quad_name = 'Quad D'
title = 'Quad D outlier mask (50% criteria)'
final_mask_D = create_final_mask(median_mask_D, quad_name, title,
final_path)
mask_name_D = final_path + '/' + 'quad_D_outlier_mask.csv'
np.savetxt(mask_name_D, np.array(final_mask_D), delimiter=",")
lower_quads = np.concatenate((final_mask_A, final_mask_B), axis=1)
upper_quads = np.concatenate((final_mask_D, final_mask_C), axis=1)
final_outlier_mask = np.concatenate((lower_quads, upper_quads), axis=0)
mask_name = final_path + '/' + 'final_outlier_mask_80%.csv'
np.savetxt(mask_name, np.array(final_outlier_mask), delimiter=",")
plt.figure()
final_outlier_mask = np.invert(final_outlier_mask)
plt.imshow(final_outlier_mask,
cmap='bwr',
interpolation='none',
origin='lower')
#cbar = plt.colorbar(image)
nx_quad, ny_quad = np.array(final_outlier_mask).shape
final_outliers = np.array(
np.where(np.reshape(final_outlier_mask, (nx_quad * ny_quad,
1)) == 0)).shape[1]
plt.title(
'TEMPO outlier Mask (50% Occurence Criteria)\n (Num. outliers = ' +
str(final_outliers) + ')',
fontsize=14)
plt.xlabel('# of spatial pixels', fontsize=12)
plt.ylabel('# of spectral pixels', fontsize=12)
plt.grid(False)
plt.savefig(final_path + '/' + 'final_mask_50%.png',
dpi=200,
bbox_inches="tight")
#***************************************************************************
#***************************************************************************
# if we want to 'OR" all the mask, use the following function
final_path = r'C:\Users\nmishra\Workspace\TEMPO\outlier_mask_spectrometer\Outlier_median\save_masks_logical_OR'
if not os.path.exists(final_path):
os.makedirs(final_path)
quad_name = 'Quad A'
title = 'Quads A outlier mask (Logical OR)'
final_mask_A = create_ORed_mask(median_mask_A, quad_name, title,
final_path)
mask_name_A = final_path + '/' + 'quad_A_outlier_mask_ored.csv'
np.savetxt(mask_name_A, np.array(final_mask_A), delimiter=",")
quad_name = 'Quad B'
title = 'Quads B outlier mask (Logical OR)'
final_mask_B = create_ORed_mask(median_mask_B, quad_name, title,
final_path)
mask_name_B = final_path + '/' + 'quad_B_outlier_mask_ored.csv'
np.savetxt(mask_name_B, np.array(final_mask_B), delimiter=",")
quad_name = 'Quad C'
title = 'Quads C outlier mask (Logical OR)'
final_mask_C = create_ORed_mask(median_mask_C, quad_name, title,
final_path)
mask_name_C = final_path + '/' + 'quad_C_outlier_mask_ored.csv'
np.savetxt(mask_name_C, np.array(final_mask_C), delimiter=",")
quad_name = 'Quad D'
title = 'Quads D outlier mask (Logical OR)'
final_mask_D = create_ORed_mask(median_mask_D, quad_name, title,
final_path)
mask_name_D = final_path + '/' + 'quad_D_outlier_mask_ored.csv'
np.savetxt(mask_name_D, np.array(final_mask_D), delimiter=",")
quad_name = 'upper_quads_ored'
title = 'Quads C & D outlier mask (Logical OR)'
lower_quads = np.concatenate((final_mask_A, final_mask_B), axis=1)
upper_quads = np.concatenate((final_mask_D, final_mask_C), axis=1)
final_outlier_mask_ored = np.concatenate((lower_quads, upper_quads),
axis=0)
mask_name = final_path + '/' + 'final_outlier_mask_ored.csv'
np.savetxt(mask_name, np.array(final_outlier_mask_ored), delimiter=",")
plt.figure()
plt.imshow(np.invert(final_outlier_mask_ored),
cmap='bwr',
interpolation='none',
origin='lower')
nx_quad, ny_quad = np.array(final_outlier_mask_ored).shape
final_outliers_ored = np.array(
np.where(
np.reshape(final_outlier_mask_ored, (nx_quad * ny_quad,
1)) == 1)).shape[1]
plt.title('TEMPO Outlier Mask (Logical OR)\n (Num. outliers = ' +
str(final_outliers_ored) + ')',
fontsize=14)
plt.xlabel('# of spatial pixels', fontsize=12)
plt.ylabel('# of spectral pixels', fontsize=12)
plt.grid(False)
plt.savefig(final_path + '/' + 'final_mask_ored.png',
dpi=200,
bbox_inches="tight")
def main():
"""
Tme main function
"""
#nx_quad = 1056 # For Tempo
#ny_quad = 1046 # For Tempo
#nlat = nx_quad*2
#nspec = ny_quad*2
file_path = r'C:\Users\nmishra\Workspace\TEMPO\Data\GroundTest\FPS\Integration_Sweep\Dark'
save_dir = r'C:\Users\nmishra\Workspace\TEMPO\outlier_mask_3_sigma'
data_path_all = [
each for each in os.listdir(file_path) if each.endswith('.sav')
]
#data_path_all = [items for items in all_int_files if not items.endswith('.dat.sav')]
#names_to_check = ['SHORT', 'NOM', 'LONG']
names_to_check = ['SHORT', 'NOM', 'OP', 'LONG']
median_mask_A = []
median_mask_B = []
median_mask_C = []
median_mask_D = []
saturation_mask_A = []
saturation_mask_B = []
saturation_mask_C = []
saturation_mask_D = []
for data_path in data_path_all:
print(data_path)
full_frame, collection_type, frames,int_time = \
make_quads_from_IDL_sav_file(file_path, data_path, names_to_check)
# all_full_frame contains all 100 frames
#Let's fetch the quads now
#quad_A = full_frame[0:ny_quad, 0:nx_quad]
quad_A = full_frame[:, 0, :, :]
active_quad_A = np.mean(quad_A[:, 2:1030, 10:1034], axis=0)
quad_B = full_frame[:, 1, :, :]
active_quad_B = np.mean(quad_B[:, 2:1030, 10:1034], axis=0)
quad_C = full_frame[:, 2, :, :]
active_quad_C = np.mean(quad_C[:, 2:1030, 10:1034], axis=0)
quad_D = full_frame[:, 3, :, :]
active_quad_D = np.mean(quad_D[:, 2:1030, 10:1034], axis=0)
#lower_quads = np.concatenate((active_quad_A, active_quad_B), axis=1)
#upper_quads = np.concatenate((active_quad_D, active_quad_C), axis=1)
#active_quads =[lower_quads, upper_quads]
# for overclocks
#lower_quads = np.concatenate((bias_A, bias_B), axis=1)
#upper_quads = np.concatenate((bias_D, bias_C), axis=1)
active_quads = [
active_quad_A, active_quad_B, active_quad_C, active_quad_D
]
quads = ['Quad A', 'Quad B', 'Quad C', 'Quad D']
# Now let's save the full_frame images
plot_dir = save_dir
image_dir = 'Quad_images_'
save_quad_images = os.path.join(plot_dir, image_dir)
# if not os.path.exists(save_quad_images):
# os.makedirs(save_quad_images)
# image_title = 'Full Frame Quad Image'
# plot_full_frame_image(full_frame, image_title, collection_type, frames,
# int_time, save_quad_images)
#
# Create list of directories To save histograms before and after
#outlier rejections
for k in np.arange(0, 4):
#mean_plot = r'Outlier_mean_tsoc'+'/'+ quads[k]
median_plot = r'Outlier_median' + '/' + quads[k]
folder_name_hist = 'Hist_plot'
folder_name_mask = 'Mask_plot'
folder_name_sat = 'Saturation_plot'
save_median_hist = os.path.join(plot_dir, median_plot,
folder_name_hist)
if not os.path.exists(save_median_hist):
os.makedirs(save_median_hist)
save_median_mask = os.path.join(plot_dir, median_plot,
folder_name_mask)
if not os.path.exists(save_median_mask):
os.makedirs(save_median_mask)
save_sat_mask = os.path.join(plot_dir, median_plot,
folder_name_sat)
if not os.path.exists(save_sat_mask):
os.makedirs(save_sat_mask)
# Now let's call the outlier functions. First identify saturated
#bits and then identify the 'hot and cold' outlier pixels
title = 'Histogram of ' + quads[k]
sat_pixels = identify_saturation(active_quads[k], save_sat_mask,
collection_type, frames, int_time)
#outlier_filt_mean, outlier_mean = reject_outlier_mean(active_quads[k])
outlier_filt_med, outlier_med = reject_outlier_median(
active_quads[k])
# Ok now lets' plot the histograms to understand what we see.
# First the mean approach
#print('ok, let us plot the histograms now')
plot_hist_image(active_quads[k], title, collection_type, frames,
outlier_filt_med, outlier_med, int_time,
save_median_hist)
if len(outlier_med) == 1:
outlier_med = 0
else:
outlier_med = outlier_med[1]
title = 'Binary Outlier Mask of ' + quads[k]
median_mask = create_outlier_mask(active_quads[k], outlier_med,
title, collection_type, frames,
save_median_mask)
#print(median_mask.max(), median_mask.min())
#cc
active_quads[k] = None
outlier_med = None
if k == 0:
median_mask_A.append(median_mask[:])
median_mask = None
saturation_mask_A.append(sat_pixels[:])
sat_pixels = None
elif k == 1:
median_mask_B.append(median_mask[:])
median_mask = None
saturation_mask_B.append(sat_pixels[:])
sat_pixels = None
elif k == 2:
median_mask_C.append(median_mask[:])
median_mask = None
saturation_mask_C.append(sat_pixels[:])
sat_pixels = None
elif k == 3:
median_mask_D.append(median_mask[:])
median_mask = None
saturation_mask_D.append(sat_pixels[:])
sat_pixels = None
#***************************************************************************
# The following function creates final mask based on 80% occurence criteria
final_path = r'C:\Users\nmishra\Workspace\TEMPO\outlier_mask\Outlier_median\save_masks_80\to_SAO%'
if not os.path.exists(final_path):
os.makedirs(final_path)
quad_name = 'Quad A'
title = 'Quad A outlier mask (50% criteria)'
final_mask_A = create_final_mask(median_mask_A, quad_name, title,
final_path)
#print(np.array(final_mask_A).max(), np.array( final_mask_A).min())
mask_name_A = final_path + '/' + 'quad_A_outlier_mask.csv'
np.savetxt(mask_name_A, np.array(final_mask_A), delimiter=",")
#cc
quad_name = 'Quad B'
title = 'Quad B outlier mask (50% criteria)'
final_mask_B = create_final_mask(median_mask_B, quad_name, title,
final_path)
mask_name_B = final_path + '/' + 'quad_B_outlier_mask.csv'
np.savetxt(mask_name_B, np.array(final_mask_B), delimiter=",")
quad_name = 'Quad C'
title = 'Quad C outlier mask (50% criteria)'
final_mask_C = create_final_mask(median_mask_C, quad_name, title,
final_path)
mask_name_C = final_path + '/' + 'quad_C_outlier_mask.csv'
np.savetxt(mask_name_C, np.array(final_mask_C), delimiter=",")
quad_name = 'Quad D'
title = 'Quad D outlier mask (50% criteria)'
final_mask_D = create_final_mask(median_mask_D, quad_name, title,
final_path)
mask_name_D = final_path + '/' + 'quad_D_outlier_mask.csv'
np.savetxt(mask_name_D, np.array(final_mask_D), delimiter=",")
lower_quads = np.concatenate((final_mask_A, final_mask_B), axis=1)
upper_quads = np.concatenate((final_mask_D, final_mask_C), axis=1)
final_outlier_mask = np.concatenate((lower_quads, upper_quads), axis=0)
mask_name = final_path + '/' + 'final_outlier_mask_80%.csv'
np.savetxt(mask_name, np.array(final_outlier_mask), delimiter=",")
plt.figure()
final_outlier_mask = np.invert(final_outlier_mask)
plt.imshow(final_outlier_mask,
cmap='bwr',
interpolation='none',
origin='lower')
#cbar = plt.colorbar(image)
nx_quad, ny_quad = np.array(final_outlier_mask).shape
final_outliers = np.array(
np.where(np.reshape(final_outlier_mask, (nx_quad * ny_quad,
1)) == 0)).shape[1]
plt.title(
'TEMPO outlier Mask (50% Occurence Criteria)\n (Num. outliers = ' +
str(final_outliers) + ')',
fontsize=14)
plt.xlabel('# of spatial pixels', fontsize=12)
plt.ylabel('# of spectral pixels', fontsize=12)
plt.grid(False)
plt.savefig(final_path + '/' + 'final_mask_50%.png',
dpi=200,
bbox_inches="tight")
#***************************************************************************
#***************************************************************************
# if we want to 'OR" all the mask, use the following function
final_path = r'C:\Users\nmishra\Workspace\TEMPO\outlier_mask\Outlier_median\save_masks_logical_OR\to_SAO'
if not os.path.exists(final_path):
os.makedirs(final_path)
quad_name = 'Quad A'
title = 'Quads A outlier mask (Logical OR)'
final_mask_A = create_ORed_mask(median_mask_A, quad_name, title,
final_path)
mask_name_A = final_path + '/' + 'quad_A_outlier_mask_ored.csv'
np.savetxt(mask_name_A, np.array(final_mask_A), delimiter=",")
quad_name = 'Quad B'
title = 'Quads B outlier mask (Logical OR)'
final_mask_B = create_ORed_mask(median_mask_B, quad_name, title,
final_path)
mask_name_B = final_path + '/' + 'quad_B_outlier_mask_ored.csv'
np.savetxt(mask_name_B, np.array(final_mask_B), delimiter=",")
quad_name = 'Quad C'
title = 'Quads C outlier mask (Logical OR)'
final_mask_C = create_ORed_mask(median_mask_C, quad_name, title,
final_path)
mask_name_C = final_path + '/' + 'quad_C_outlier_mask_ored.csv'
np.savetxt(mask_name_C, np.array(final_mask_C), delimiter=",")
quad_name = 'Quad D'
title = 'Quads D outlier mask (Logical OR)'
final_mask_D = create_ORed_mask(median_mask_D, quad_name, title,
final_path)
mask_name_D = final_path + '/' + 'quad_D_outlier_mask_ored.csv'
np.savetxt(mask_name_D, np.array(final_mask_D), delimiter=",")
quad_name = 'upper_quads_ored'
title = 'Quads C & D outlier mask (Logical OR)'
lower_quads = np.concatenate((final_mask_A, final_mask_B), axis=1)
upper_quads = np.concatenate((final_mask_D, final_mask_C), axis=1)
final_outlier_mask_ored = np.concatenate((lower_quads, upper_quads),
axis=0)
mask_name = final_path + '/' + 'final_outlier_mask_ored.csv'
np.savetxt(mask_name, np.array(final_outlier_mask_ored), delimiter=",")
plt.figure()
plt.imshow(np.invert(final_outlier_mask_ored),
cmap='bwr',
interpolation='none',
origin='lower')
nx_quad, ny_quad = np.array(final_outlier_mask_ored).shape
final_outliers_ored = np.array(
np.where(
np.reshape(final_outlier_mask_ored, (nx_quad * ny_quad,
1)) == 1)).shape[1]
plt.title('TEMPO Outlier Mask (Logical OR)\n (Num. outliers = ' +
str(final_outliers_ored) + ')',
fontsize=14)
plt.xlabel('# of spatial pixels', fontsize=12)
plt.ylabel('# of spectral pixels', fontsize=12)
plt.grid(False)
plt.savefig(final_path + '/' + 'final_mask_ored.png',
dpi=200,
bbox_inches="tight")
def main():
"""
Tme main function """
file_path = r'C:\Users\nmishra\Workspace\TEMPO\Data\GroundTest\FPS\Dark_Current_Radiation_testing\Pre_rad'
data_path_all = [
each for each in os.listdir(file_path) if each.endswith('.fits')
]
collection_type = 'Pred rad Test'
median_mask_all_quad_A = []
median_mask_all_quad_B = []
for data_path in data_path_all:
full_frame, int_time, temp, frames = make_quads_from_fits_file(
file_path, data_path)
# Radiation testing was done only to 1 CCD ( quads A and B)
#There are 16 frames each for Quads A and B. CLoser inspection suggested
#that you can average these 16 frames and work on average
# Let's fetch the quads now
ndimes, nx_quad, ny_quad = full_frame.shape
quad_A = np.median(full_frame[0:ndimes:2, :, :], axis=0)
active_quad_A = np.array(quad_A[4:1028, 10:1034])
quad_B = np.median(full_frame[1:ndimes:2, :, :], axis=0)
active_quad_B = np.array(quad_B[4:1028, 10:1034])
quads_name = ['Quad A', 'Quad B']
lower_quads = [active_quad_A, active_quad_B]
lower_quads_all = [quad_A, quad_B]
plot_dir = file_path
# Create list of directories To save histograms before and after
#outlier rejections
for k in np.arange(0, 2):
image_dir = r'Quad_images' + '/' + quads_name[k]
median_plot = r'Outlier_median' + '/' + quads_name[k]
folder_name_hist = 'Hist_plot'
folder_name_mask = 'Mask_plot'
folder_name_sat = 'Saturation_plot'
save_quad_images = os.path.join(plot_dir, image_dir)
if not os.path.exists(save_quad_images):
os.makedirs(save_quad_images)
save_median_hist = os.path.join(plot_dir, median_plot,
folder_name_hist)
if not os.path.exists(save_median_hist):
os.makedirs(save_median_hist)
save_median_mask = os.path.join(plot_dir, median_plot,
folder_name_mask)
if not os.path.exists(save_median_mask):
os.makedirs(save_median_mask)
save_sat_mask = os.path.join(plot_dir, median_plot,
folder_name_sat)
if not os.path.exists(save_sat_mask):
os.makedirs(save_sat_mask)
# Lets plt the frame first
if k == 0:
image_title = 'Full Frame Quad A Image'
else:
image_title = 'Full Frame Quad B Image'
int_time = str(int_time)
title = image_title + ' ('+ collection_type+')' + '\n Int. time = ' + int_time + \
', Temp = ' +temp
plt.figure()
image = plt.imshow(lower_quads_all[k],
cmap='nipy_spectral',
interpolation='none',
origin='lower')
cbar = plt.colorbar(image)
plt.title(title, fontsize=14)
plt.xlabel('# of spatial pixels', fontsize=12)
plt.ylabel('# of spectral pixels', fontsize=12)
plt.grid(False)
plt.savefig(save_quad_images+'/'+ quads_name[k]+'_'+\
frames+'.png',dpi=100,bbox_inches="tight")
plt.close('all')
# Now let's call the outlier functions. First identify saturated
#bits and then identify the 'hot and cold' outlier pixels
if k == 0:
title = 'Histogram of Quad A'
else:
title = 'Histogram of Quad B'
# sat_pixels = identify_saturation(lower_quads[k],
# save_sat_mask,
# collection_type,
# frames, int_time)
outlier_filt_med, outlier_med = reject_outlier_median(
lower_quads[k])
# Ok now lets' plot the histograms to understand what we see.
# First the mean approach
print('ok, let us plot the histograms now')
# Secondly the median absolute difference approach
plot_hist_image(lower_quads[k], title, collection_type, frames,
outlier_filt_med, outlier_med, int_time,
save_median_hist)
if len(outlier_med) == 1:
outlier_med = 0
else:
outlier_med = outlier_med[1]
if k == 0:
title = 'Binary Outlier Mask of Quad A'
else:
title = 'Binary Outlier Mask of Quad B'
median_mask = create_outlier_mask(lower_quads[k], outlier_med,
title, collection_type, frames,
save_median_mask)
lower_quads[k] = None
outlier_med = None
if k == 0:
median_mask_all_quad_A.append(median_mask[:])
median_mask = None
#saturation_mask_lower.append(sat_pixels[:])
#sat_pixels = None
elif k == 1:
median_mask_all_quad_B.append(median_mask[:])
median_mask = None
#saturation_mask_upper.append(sat_pixels[:])
#sat_pixels = None
#***************************************************************************
# The following function creates final mask based on 80% occurence criteria
final_path = file_path + '\Outlier_median\save_masks'
if not os.path.exists(final_path):
os.makedirs(final_path)
quad_name = 'QuadA_80%'
title = 'Quad A outlier mask (80% occurence criteria)'
final_mask_quad_A = create_final_mask(median_mask_all_quad_A, quad_name,
title, final_path)
quad_name = 'QuadB_80%'
title = 'Quad B outlier mask (80% ocurence criteria)'
final_mask_quad_B = create_final_mask(median_mask_all_quad_B, quad_name,
title, final_path)
final_outlier_mask = np.concatenate((final_mask_quad_A, final_mask_quad_B),
axis=1)
plt.figure()
plt.imshow(final_outlier_mask,
cmap='bwr',
interpolation='none',
origin='lower')
nx_quad, ny_quad = np.array(final_outlier_mask).shape
final_outliers = np.array(
np.where(np.reshape(final_outlier_mask, (nx_quad * ny_quad,
1)) == 0)).shape[1]
plt.title('Quad A and B Outlier Mask' + ' (outliers = ' +
str(final_outliers) + ')',
fontsize=14)
plt.xlabel('# of spatial pixels', fontsize=12)
plt.ylabel('# of spectral pixels', fontsize=12)
plt.grid(False)
plt.savefig(final_path + '/' + 'final_mask.png',
dpi=100,
bbox_inches="tight")
#***************************************************************************
#***************************************************************************
# if we want to 'OR" all the mask, use the following function
quad_name = 'QuadA_ored'
title = 'Quad A outlier mask (Logical OR)'
final_mask_lower_ored = create_ORed_mask(median_mask_all_quad_A, quad_name,
title, final_path)
quad_name = 'QuadB_ored'
title = 'Quad B outlier mask (Logical OR)'
final_mask_upper_ored = create_ORed_mask(median_mask_all_quad_B, quad_name,
title, final_path)
final_outlier_mask_ored = np.concatenate(
(final_mask_lower_ored, final_mask_upper_ored), axis=1)
plt.figure()
plt.imshow(np.invert(final_outlier_mask_ored),
cmap='bwr',
interpolation='none',
origin='lower')
nx_quad, ny_quad = np.array(final_outlier_mask_ored).shape
final_outliers_ored = np.array(
np.where(
np.reshape(final_outlier_mask_ored, (nx_quad * ny_quad,
1)) == 1)).shape[1]
plt.title('Quad A & B Outlier Mask' + ' (outliers = ' +
str(final_outliers_ored) + ')',
fontsize=14)
plt.xlabel('# of spatial pixels', fontsize=12)
plt.ylabel('# of spectral pixels', fontsize=12)
plt.grid(False)
plt.savefig(final_path + '/' + 'final_mask_ored.png',
dpi=100,
bbox_inches="tight")
def main():
"""
Tme main function
"""
nx_quad = 1056 # For Tempo
ny_quad = 1046 # For Tempo
nlat = nx_quad * 2
nspec = ny_quad * 2
file_path = r'C:\Users\nmishra\Workspace\TEMPO\Data\GroundTest\FPS\Light'
data_path = [
each for each in os.listdir(file_path) if each.endswith('.dat')
]
names_to_check = ['SHORT', 'NOM', 'LONG']
for data_path in data_path:
#data_path = 'SPI_20160912142518420_LIGHT_INTEGRATION_SWEEP_207_FT6_NOM_INT_99999.dat'
# Now let's find the integtration type
data_path_name_split = data_path.split('_')
#print(data_path_name_split)
integration_type = [
x for x in names_to_check if x in data_path_name_split
]
frames = data_path_name_split[-1]
data_file = os.path.join(file_path, data_path)
fileinfo = get_size(data_file)
size_file = fileinfo.st_size
#-------------Some printing to do Sanity checks----------------------------
#print ("size of the file is ", size_file, "bytes")
nframes = size_file / (nlat * nspec * 2)
#print("num of frames : ", nframes)
#dt = np.dtype(')
result = np.fromfile(data_file, dtype='>u2', count=-1) & int(
'3fff', 16)
# Refer to 2450865 Ball document
#print(result.size)
#*Ok, let's do few steps before we move into data analysis
# 1. Create full frame from raw FPE.
# 2. Arrange full frame into various quads.
full_frame = make_full_frame_from_raw_fpe(result, nx_quad, ny_quad)
quads = full_frame_to_quads(full_frame, nx_quad, ny_quad)
# Now let's save the full_frame images
plot_dir = r'C:\Users\nmishra\Desktop\test'
image_dir = 'Quad_images'
save_quad_images = os.path.join(plot_dir, image_dir)
if not os.path.exists(save_quad_images): os.makedirs(save_quad_images)
aligned_quads = 'aligned_quads'
save_aligned_quads = os.path.join(plot_dir, aligned_quads)
if not os.path.exists(save_aligned_quads):
os.makedirs(save_aligned_quads)
collection_type = integration_type[0]
image_title = 'Full Frame Quad Image'
plot_full_frame_image(full_frame, image_title, collection_type, frames,
save_quad_images)
plot_each_quad(quads, image_title, collection_type, frames,
save_aligned_quads)
quads = full_frame_to_quads(full_frame, nx_quad, ny_quad)
active_pixel_quads = find_active_region_each_quads(quads)
title = 'Histogram of active pixels'
# plot_hist_each_quad(active_pixel_quads,title, collection_type,frames)
# Creste list of directories To save histograms before and after
#outlier rejections
mean_plot = 'Outlier_mean'
median_plot = 'Outlier_median'
folder_name_hist = 'Hist_plot'
folder_name_mask = 'Mask_plot'
save_mean_hist = os.path.join(plot_dir, mean_plot, folder_name_hist)
if not os.path.exists(save_mean_hist): os.makedirs(save_mean_hist)
save_median_hist = os.path.join(plot_dir, median_plot,
folder_name_hist)
if not os.path.exists(save_median_hist): os.makedirs(save_median_hist)
save_mean_mask = os.path.join(plot_dir, mean_plot, folder_name_mask)
if not os.path.exists(save_mean_mask): os.makedirs(save_mean_mask)
save_median_mask = os.path.join(plot_dir, median_plot,
folder_name_mask)
if not os.path.exists(save_median_mask): os.makedirs(save_median_mask)
# Now let's call the outlier functions
outlier_filt_mean, outlier_mean = reject_outlier_mean(
active_pixel_quads)
outlier_filt_med, outlier_med = reject_outlier_median(
active_pixel_quads)
# Ok now lets' plot the histograms to understand what we see.
# First the mean approach
plot_hist_image(active_pixel_quads, title, collection_type, frames,
outlier_filt_mean, outlier_mean, save_mean_hist)
# Secondly the median absolute difference approach
plot_hist_image(active_pixel_quads, title, collection_type, frames,
outlier_filt_med, outlier_med, save_median_hist)
# Ok, lets now create a binary mask of outlier pixels
# First with the mean based approach
create_outlier_mask(active_pixel_quads, outlier_med, collection_type,
frames, save_mean_mask)
# Secondly, the same for median based approach
create_outlier_mask(active_pixel_quads, outlier_med, collection_type,
frames, save_median_mask)