def load_flat_img(image_filenames, closest_index, half_images_to_average,
channel, bias_img):
# global bias_img
if 0:
# print(all_flat_channel_means[closest_index], input_channel_mean)
flat_image = histogram_gap.load_raw_image(
image_filenames[closest_index], bias_img)
flat_image_channel = flat_image[:, :, channel]
else:
if 0:
flat_image_channel_stack = None
for i, image_index in enumerate(
range(
max(0, closest_index - half_images_to_average),
min(
len(image_filenames) - 1,
closest_index + half_images_to_average + 1))):
flat_image = histogram_gap.load_raw_image(
image_filenames[image_index], bias_img)
flat_image_channel = flat_image[:, :, channel]
if flat_image_channel_stack is None:
flat_image_channel_stack = np.zeros(
(2 * half_images_to_average + 1, ) +
flat_image_channel.shape,
dtype=flat_image_channel.dtype)
flat_image_channel_stack[i] = flat_image_channel
else:
flat_image_channel_stack = []
for i, image_index in enumerate(
range(
max(0, closest_index - half_images_to_average),
min(
len(image_filenames) - 1,
closest_index + half_images_to_average + 1))):
flat_image = histogram_gap.load_raw_image(
image_filenames[image_index], bias_img)
flat_image_channel = flat_image[:, :, channel]
flat_image_channel_stack.append(flat_image_channel)
flat_image_channel_stack = np.array(flat_image_channel_stack)
#todo: normalize image brightnesses before combining?
image_means = np.mean(flat_image_channel_stack, axis=(1, 2))
# print(image_means)
for i in range(flat_image_channel_stack.shape[0]):
# normalization_factor = image_means[half_images_to_average] / image_means[i]
normalization_factor = image_means[image_means.shape[0] //
2] / image_means[i]
flat_image_channel_stack[i] *= normalization_factor
flat_image_channel = np.mean(flat_image_channel_stack, axis=0)
# print(flat_image.shape)
return flat_image_channel
def test_get_flat():
# folder = 'F:/2020/2020-04-09/shirt_dusk_flats_progression'
folder = 'F:/2020/2020-04-11/135mm_sky_flats_progression'
bias_folder = 'K:/orion_135mm_bothnights/bias'
bias_img = load_dark(bias_folder)
downsize_factor = 4
test_img_path = 'F:/Pictures/Lightroom/2020/2020-02-18/orion_135mm/DSC03637.ARW'
test_img = histogram_gap.load_raw_image(test_img_path, bias_img)
test_img = np.transpose(test_img, (2, 0, 1))
flat = get_subimg_matched_flat(None, test_img, folder, bias_img)
for channel_index in range(4):
print('testing channel ', channel_index)
channel_flat = flat[channel_index]
test_img_channel = test_img[channel_index]
plt.imshow(channel_flat)
plt.title('channel flat')
plt.show()
calibrated_img = test_img_channel / channel_flat
plt.imshow(test_img_channel)
plt.title('test img')
plt.show()
plt.imshow(calibrated_img)
plt.title('calibrated test img')
plt.show()
# plt.hist(calibrated_img.flatten(), bins = 1000)
# plt.show()
benchmark_flat = full_flat_sequence.get_flat_matching_brightness_histogram_match(
folder,
test_img_channel,
channel_index,
bias_img=bias_img,
half_images_to_average=5)
plt.imshow(benchmark_flat)
plt.title('benchmark')
plt.show()
benchmark_ratio = channel_flat / benchmark_flat
plt.title('difference v benchmark')
plt.imshow(benchmark_ratio)
plt.show()
def get_progression_flat_mean(folder, bias_img=None):
cache_name = os.path.join(folder, 'mean_frame.tiff')
means_cache_name = os.path.join(folder, 'mean_brightnesses.npy')
image_names = list(
map(lambda s2: os.path.join(folder, s2),
filter(lambda s: s.endswith('.ARW'), os.listdir(folder))))
#is this ok in general? probably?
# def sortKey(s):
# result = int(s.split(os.path.sep)[-1].strip('DSC').split('.')[-2])
# if result > 9000: result -= 10000
# return result
# image_names.sort(key = sortKey)
# print(image_names)
if os.path.exists(cache_name): # and False:
sum_image = tiff.imread(cache_name)
all_image_means = np.load(means_cache_name)
else:
sum_image = None
all_image_means = []
for img_fn in tqdm.tqdm(image_names):
img = histogram_gap.load_raw_image(img_fn, bias_img)
channel_means = np.mean(img, axis=(0, 1))
img = img / channel_means[np.newaxis, np.newaxis, :]
all_image_means.append(channel_means)
# print(channel_means)
if sum_image is None:
sum_image = img
else:
sum_image += img
sum_image /= len(image_names)
all_image_means = np.array(all_image_means)
# plt.plot(all_image_means)
# plt.show()
tiff.imwrite(cache_name, sum_image)
np.save(means_cache_name, all_image_means)
return sum_image, all_image_means, image_names
def test_get_flat_faster():
folder = 'F:/2020/2020-04-11/135mm_sky_flats_progression'
bias_folder = 'K:/orion_135mm_bothnights/bias'
bias_img = load_dark(bias_folder)
downsize_factor = 4
test_img_path = 'F:/Pictures/Lightroom/2020/2020-02-18/orion_135mm/DSC03637.ARW'
test_img = histogram_gap.load_raw_image(test_img_path, bias_img)
test_img = np.transpose(test_img, (2, 0, 1))
# resize = 4
# test_img = test_img[0:1, :test_img.shape[1]//(resize*downsize_factor) * downsize_factor, :test_img.shape[2]//(resize*downsize_factor) * downsize_factor]
# test_img = test_img[0:1]
print('test_img: ', test_img.shape)
# for channel_index in range(test_img.shape[0]):
# channel_progression, channel_means = load_channel_progression_brightness_filtered(folder, bias_img, downsize_factor, channel_index, num_images_to_avg=5)
load_all_channels(folder, bias_img, downsize_factor)
start = datetime.datetime.now()
flat_new = get_subimg_matched_flat2(None, test_img, folder, bias_img)
print('new time: ', (datetime.datetime.now() - start))
if 1:
start = datetime.datetime.now()
flat_reference = get_subimg_matched_flat(None, test_img, folder,
bias_img)
print('reference time: ', (datetime.datetime.now() - start))
if np.all(flat_new == flat_reference):
print('ALL GOOD!')
else:
print('***not equal***')
avg_diff = np.mean(
np.abs(flat_new - flat_reference) / flat_reference)
print('average relative difference: ', avg_diff)
ratio = (flat_new / flat_reference)[0]
plt.imshow(ratio)
plt.show()
def test():
flats_progression_folder = 'F:/2020/2020-04-11/135mm_sky_flats_progression'
# test_folder = 'F:/Pictures/Lightroom/2020/2020-02-18/blue_sky_flats/4'
test_folder = 'F:/2020/2020-04-11/flats_135mm_tshirtwall'
bias_folder = 'K:/orion_135mm_bothnights/bias'
bias_img = load_dark(bias_folder)
for img_fn in os.listdir(test_folder):
img = histogram_gap.load_raw_image(os.path.join(test_folder, img_fn),
master_dark=bias_img)
img = np.transpose(img, (2, 0, 1))
flat_img = subimg_full_flat_sequence.get_subimg_matched_flat2(
None, img, flats_progression_folder, bias_img, downsize_factor=4)
calibrated_test_img = img / flat_img
# calibrated_test_img = img
for channel in range(4):
calibrated_channel = calibrated_test_img[channel]
calibrated_channel = remove_gradient(calibrated_channel,
quadratic=True)
calibrated_channel /= np.mean(calibrated_channel)
overall_shape = gaussian_filter(calibrated_channel, sigma=50)
calibrated_channel -= overall_shape
std_dev = np.std(calibrated_channel)
print(std_dev)
plt.subplot(2, 1, 1)
plt.imshow(np.clip(calibrated_channel, -std_dev, std_dev))
plt.title(str(std_dev))
plt.subplot(2, 1, 2)
plt.imshow(overall_shape)
plt.show()
def test():
bias_folder = 'K:/orion_135mm_bothnights/bias'
master_bias = load_dark(bias_folder)
flats_progression_folder = 'F:/2020/2020-04-09/shirt_dusk_flats_progression'
test_img_path = 'F:/Pictures/Lightroom/2020/2020-02-29/orion_600mm/DSC05669.ARW'
show_relative_flats(test_img_path, master_bias)
test_img = histogram_gap.load_raw_image(test_img_path, None)
for channel in range(test_img.shape[-1]):
matching_flat = get_flat_matching_brightness(flats_progression_folder,
test_img[:, :, channel],
channel,
half_images_to_average=3)
plt.subplot(2, 2, 1)
display_image(test_img[:, :, channel])
plt.title('test img')
plt.subplot(2, 2, 2)
display_image(matching_flat)
plt.title('matching flat')
ratios = (test_img[:, :, channel] / matching_flat).flatten()
plt.subplot(2, 2, 3)
bin_range = 0.1
bins = np.linspace(1 - bin_range, 1 + bin_range, 1000)
plt.hist(ratios, bins=bins)
plt.yscale('log', nonposy='clip')
plt.grid(True)
plt.show()
print(np.mean(test_img, axis=(0, 1)))
def get_mean_flat(folder, bias_or_dark_frame, force_reload=False):
cache_filename = os.path.join(folder, 'master_flat.tif')
if os.path.exists(cache_filename) and not force_reload:
print('loading flat from cache...', cache_filename)
result = tiff.imread(cache_filename)
return result
else:
filenames = list(
map(lambda s2: os.path.join(folder, s2),
filter(lambda s: s.endswith('.ARW'), os.listdir(folder))))
print('filenames in folder: ', folder)
print(filenames)
image_stack = None
for i, filename in enumerate(filenames):
raw_img = histogram_gap.load_raw_image(filename,
bias_or_dark_frame)
if image_stack is None:
image_stack = np.zeros((len(filenames), ) + raw_img.shape,
dtype=raw_img.dtype)
print(image_stack.shape)
image_stack[i] = raw_img
#TODO: better than straight mean
result = np.mean(image_stack, axis=0)
# result = np.mean(astropy.stats.sigma_clip(image_stack, sigma=2, axis=0), axis=0)
tiff.imwrite(cache_filename, result)
return result
def show_relative_flats(test_img_path, bias_img=None):
test_img = histogram_gap.load_raw_image(test_img_path, bias_img)
print(test_img.shape)
for channel in range(4):
test_channel = test_img[:, :, channel]
flat_channel = 1.1 * test_img[:, :, channel]
relative_flat = get_relative_flat(flats_progression_folder,
test_channel, flat_channel, channel,
bias_img)
# for channel in [3]:
# matching_flat = get_flat_matching_brightness(flats_progression_folder, test_img[:, :, channel], channel, bias_img = bias_img)
# test_img_offset = test_img * 1.1
# matching_flat_offset = get_flat_matching_brightness(flats_progression_folder, test_img_offset[:, :, channel], channel, bias_img = bias_img)
# relative_flat = matching_flat / matching_flat_offset
# plt.imshow(relative_flat)
display_image(relative_flat)
plt.show()
def test2():
bias_folder = 'K:/orion_135mm_bothnights/bias'
master_bias = load_dark(bias_folder)
# flats_progression_folder = 'F:/2020/2020-04-09/shirt_dusk_flats_progression'
flats_progression_folder = 'F:/2020/2020-04-11/135mm_sky_flats_progression'
test_folder = 'F:/2020/2020-04-10/casseiopeia_pano_135mm/3'
n_test_images = 3
test_filenames = list(
map(lambda s2: os.path.join(test_folder, s2),
filter(lambda s: s.endswith('.ARW'), os.listdir(test_folder))))
for i in range(0, len(test_filenames), n_test_images):
fns = test_filenames[i:i + n_test_images]
test_imgs = [
histogram_gap.load_raw_image(fn, master_bias) for fn in fns
]
mean_test_img = np.mean(test_imgs, axis=0)
for channel in range(mean_test_img.shape[-1]):
test_img_channel = mean_test_img[:, :, channel]
# matching_flat = get_flat_matching_brightness(flats_progression_folder, test_img_channel, channel, half_images_to_average=7)
matching_flat = get_flat_matching_brightness_histogram_match(
flats_progression_folder,
test_img_channel,
channel,
half_images_to_average=7)
corrected_test_img = test_img_channel / matching_flat
ratio_range = 0.05
ratios = corrected_test_img[np.where(
np.abs(corrected_test_img - 1) < 0.05)].flatten()
def display_image(img, z=1):
disp_image = img.copy()
disp_image = remove_gradient(disp_image)
disp_image = gaussian_filter(disp_image,
mode='nearest',
sigma=5)
low = np.percentile(disp_image, z)
high = np.percentile(disp_image, 100 - z)
disp_image = np.clip(disp_image, low, high)
plt.imshow(disp_image)
plt.subplot(2, 2, 1)
display_image(test_img_channel)
plt.title('raw test img')
plt.subplot(2, 2, 2)
display_image(matching_flat)
plt.title('matching flat')
plt.subplot(2, 2, 3)
display_image(corrected_test_img)
plt.title('corrected test img')
plt.subplot(2, 2, 4)
plt.hist(ratios, bins=1001)
plt.grid(True)
plt.title('flat : bright ratio img')
plt.show()
def make_animation():
folder = 'F:/2020/2020-04-09/shirt_dusk_flats_progression'
bias_img = None
overall_mean, all_image_means, image_filenames = get_progression_flat_mean(
folder, bias_img=bias_img)
if 0:
for channel in range(4):
plt.plot(all_image_means[:, channel])
plt.grid(True)
plt.show()
for channel in range(4):
plt.imshow(overall_mean[:, :, channel])
plt.title(str(channel))
plt.show()
if 1:
size = (3012, 2012)
channel_index = 0
out = cv2.VideoWriter('channel_%d.avi' % channel_index,
cv2.VideoWriter_fourcc(*'mp4v'),
15,
size,
isColor=False)
image_names = list(
map(lambda s2: os.path.join(folder, s2),
filter(lambda s: s.endswith('.ARW'),
os.listdir(folder)))) #[::10]
image_names = image_names[::-1]
# print(image_names)
for i, img_fn in enumerate(tqdm.tqdm(image_names)):
if 0:
img = histogram_gap.load_raw_image(os.path.join(
folder, img_fn),
master_dark=bias_img)
channel = img[:, :, channel_index]
else:
channel = load_flat_img(image_names,
i,
half_images_to_average=7,
channel=channel_index,
bias_img=bias_img)
channel /= overall_mean[:, :, channel_index]
channel_mean = np.mean(channel)
channel /= channel_mean
channel -= 1
scale = 10
channel *= scale
channel = np.clip(255 * (channel + 0.5), 0, 255).astype(np.uint8)
out.write(channel)
out.release()
def main():
"""
workflow:
calc bias frame - sigma mean
remove histogram gaps
folder input
cache
calc dark frame - sigma mean
remove histogram gaps
folder input
cache
calc series of flat frames - sigma mean
remove histogram gaps
subtract bias
folder input
cache
for each light frame:
remove histogram gaps
subtract dark frame
for each channel:
pick optimal weighting of flat frames
divide by flat
save image as tif
-> pixinsight
debayer
register
integrate
"""
if 1:
lights_in_folder = 'K:/orion_135mm_bothnights/lights_in'
darks_folder = 'K:/orion_135mm_bothnights/darks'
bias_folder = 'K:/orion_135mm_bothnights/bias'
output_folder = 'K:/orion_135mm_bothnights2'
# flats_folder = 'F:/2020/2020-04-06/blue_sky_flats'
flats_folder = 'F:/Pictures/Lightroom/2020/2020-02-18/blue_sky_flats'
flats_progression_folder = 'F:/2020/2020-04-09/shirt_dusk_flats_progression'
elif 0:
lights_in_folder = 'F:/Pictures/Lightroom/2020/2020-02-19/flame_horsehead_600mm'
darks_folder = 'F:/Pictures/Lightroom/2020/2020-02-19/darks'
bias_folder = 'F:/Pictures/Lightroom/2020/2020-02-19/bias'
flats_folder = 'F:/Pictures/Lightroom/2020/2020-02-20/flats'
# flats_folder = 'F:/2020/2020-04-07/flats_600mm'
flats_progression_folder = 'F:/2020/2020-04-09/shirt_dusk_flats_progression'
calibrated_lights_out_folder = 'K:/flame_horsehead_600mm/lights_out_flat_sequence'
elif 0:
lights_in_folder = 'F:/Pictures/Lightroom/2020/2020-02-29/orion_600mm'
darks_folder = 'F:/Pictures/Lightroom/2020/2020-02-19/darks'
bias_folder = 'F:/2020/2020-04-17/bias_iso800'
flats_progression_folder = 'F:/2020/2020-04-09/shirt_dusk_flats_progression'
flats_folder = 'F:/Pictures/Lightroom/2020/2020-02-29/flats'
# flats_folder = 'F:/2020/2020-04-07/flats_600mm'
calibrated_lights_out_folder = 'K:/orion_600mm/lights_out'
elif 0:
bias_folder = 'F:/Pictures/Lightroom/2020/2020-02-19/bias'
darks_folder = 'F:/Pictures/Lightroom/2020/2020-02-19/darks'
# flats_progression_folder = 'F:/2020/2020-04-09/shirt_dusk_flats_progression'
flats_progression_folder = 'F:/2020/2020-04-11/135mm_sky_flats_progression'
# flats_folder = 'F:/2020/2020-04-06/blue_sky_flats'
flats_folder = 'F:/2020/2020-04-11/flats_135mm_tshirtwall'
# lights_in_folder = 'F:/2020/2020-04-10/casseiopeia_pano_135mm/1'
# calibrated_lights_out_folder = 'K:/casseiopeia_pano/lights_1'
# lights_in_folder = 'F:/2020/2020-04-10/casseiopeia_pano_135mm/2'
# calibrated_lights_out_folder = 'K:/casseiopeia_pano/lights_2'
lights_in_folder = 'F:/2020/2020-04-10/casseiopeia_pano_135mm/1'
# calibrated_lights_out_folder = 'K:/casseiopeia_pano/lights_3'
calibrated_lights_out_folder = 'K:/casseiopeia_pano/lights_1_blurredflats'
elif 0:
bias_folder = 'F:/Pictures/Lightroom/2020/2020-02-19/bias'
darks_folder = 'F:/Pictures/Lightroom/2020/2020-02-19/darks'
flats_folder = 'F:/2020/2020-04-16/rho_ophiuchi/flats'
lights_in_folder = 'F:/2020/2020-04-16/rho_ophiuchi/lights'
flats_progression_folder = 'F:/2020/2020-04-11/135mm_sky_flats_progression'
calibrated_lights_out_folder = 'K:/rho_oph/lights'
elif 0:
bias_folder = 'F:/2020/2020-04-17/bias_iso800'
darks_folder = 'F:/2020/2020-04-17/darks_iso800_2mins'
flats_folder = 'F:/2020/2020-04-17/flats_135mmf4_v2'
lights_in_folder = 'F:/2020/2020-04-17/coma_cluster_135mmf4'
flats_progression_folder = 'F:/2020/2020-04-11/135mm_sky_flats_progression'
calibrated_lights_out_folder = 'K:/coma_135mm_f4/lights'
elif 0:
lights_in_folder = 'F:/2020/2020-04-15/leo_triplet_600mm'
darks_folder = 'F:/Pictures/Lightroom/2020/2020-02-19/darks'
bias_folder = 'F:/2020/2020-04-17/bias_iso800'
flats_progression_folder = 'F:/2020/2020-04-09/shirt_dusk_flats_progression'
flats_folder = 'F:/2020/2020-04-15/flats'
calibrated_lights_out_folder = 'K:/leo_triplet_600mm/lights_out'
elif 0:
lights_in_folder = 'F:/Pictures/Lightroom/2020_2018/2018-02-01/andromeda_600mm'
darks_folder = 'F:/Pictures/Lightroom/2020/2020-02-19/darks'
bias_folder = 'F:/2020/2020-04-17/bias_iso800'
flats_progression_folder = 'F:/2020/2020-04-09/shirt_dusk_flats_progression'
flats_folder = 'F:/Pictures/Lightroom/2020_2018/2018-02-01/flats_600mm'
calibrated_lights_out_folder = 'K:/andromeda_600mm/lights_out'
elif 0:
darks_folder = 'F:/Pictures/Lightroom/2020/2020-03-07/darks_135mm_30s_iso100'
bias_folder = 'F:/Pictures/Lightroom/2020/2020-03-07/bias_135mm_iso100'
flats_progression_folder = 'F:/2020/2020-04-11/135mm_sky_flats_progression'
flats_folder = 'F:/2020/2020-04-12/flats_135mm_clothsky'
lights_in_folder = 'F:/2020/2020-04-12/coma_cluster_135mm'
calibrated_lights_out_folder = 'K:/coma_cluster_135mm/lights_out'
elif 0:
lights_in_folder = 'F:/Pictures/Lightroom/2020/2020-02-20/pleiades'
darks_folder = 'F:/Pictures/Lightroom/2020/2020-02-19/darks'
bias_folder = 'F:/2020/2020-04-17/bias_iso800'
flats_progression_folder = 'F:/2020/2020-04-09/shirt_dusk_flats_progression'
flats_folder = 'F:/Pictures/Lightroom/2020/2020-02-20/pleiades_flats'
calibrated_lights_out_folder = 'Z:/astro_processing/pleiades/lights_pleiades_flats'
elif 1:
darks_folder = 'K:/orion_135mm_bothnights/darks'
bias_folder = 'K:/orion_135mm_bothnights/bias'
flats_progression_folder = 'F:/2020/2020-04-11/135mm_sky_flats_progression'
flats_folder = 'F:/2020/2020-04-22/24mm_flats'
# lights_in_folder = 'F:/2020/2020-04-11/rice_lake_nightscape/tracked'
# calibrated_lights_out_folder = 'K:/rice_lake/tracked_lights'
lights_in_folder = 'F:/2020/2020-04-11/rice_lake_nightscape/untracked'
calibrated_lights_out_folder = 'K:/rice_lake/untracked_lights'
if not os.path.exists(output_folder): os.mkdir(output_folder)
calibrated_lights_out_folder = os.path.join(output_folder, 'calibrated')
if not os.path.exists(calibrated_lights_out_folder):
os.mkdir(calibrated_lights_out_folder)
#Todo: make smarter mean. also cache
master_dark = load_dark(darks_folder)
master_bias = load_dark(bias_folder)
flat_images, flat_subfolders = load_flats_from_subfolders(
flats_folder, master_bias)
flat_images_rgb = np.transpose(flat_images, (0, 3, 1, 2))
print(flat_images.shape, flat_images_rgb.shape)
filenames = list(
filter(lambda s: s.endswith('.ARW'), os.listdir(lights_in_folder)))[:7]
print('number of lights: ', len(filenames))
for filename in tqdm.tqdm(filenames):
full_path = os.path.join(lights_in_folder, filename)
output_path = os.path.join(calibrated_lights_out_folder,
filename.rstrip('.ARW') + '.tif')
# if os.path.exists(output_path): continue
img_minus_dark = histogram_gap.load_raw_image(full_path, master_dark)
img_minus_dark_rgb = np.transpose(img_minus_dark, (2, 0, 1))
if 0:
calibrated_flat_img_rgb, exposure_index = get_exposure_matched_flat(
flat_images_rgb, img_minus_dark_rgb)
elif 0:
calibrated_flat_img_rgb = full_flat_sequence.get_flat_and_bandingfix_flat(
flat_images_rgb, img_minus_dark_rgb, flats_progression_folder,
master_bias)
elif 0:
calibrated_flat_img_rgb = subimg_full_flat_sequence.get_subimg_matched_flat2(
flat_images_rgb, img_minus_dark_rgb, flats_progression_folder,
master_bias)
elif 1:
calibrated_flat_img_rgb = subimg_full_flat_sequence.get_flat_and_subimg_matched_flat(
flat_images_rgb, img_minus_dark_rgb, flats_progression_folder,
master_bias)
else:
calibrated_flat_img_rgb = 1
if calibrated_flat_img_rgb is None:
print("***Failed to find matching flat")
continue
calibrated_img_rgb = img_minus_dark_rgb / calibrated_flat_img_rgb
calibrated_img_rgb = np.clip(calibrated_img_rgb, 0, 1)
# calibrated_img_rgb = img_minus_dark_rgb
# print('calibrated rgb: ', calibrated_img_rgb.shape)
calibrated_img = flatten_channel_image(calibrated_img_rgb)
tiff.imwrite(output_path, calibrated_img.astype('float32'))
# exit(0)
pi_script_path = pixinsight_preprocess.create_pixinsight_preprocess_script(
output_folder, calibrated_lights_out_folder, filenames)
cmd = '"C:/Program Files/PixInsight/bin/PixInsight.exe" --run=%s' % pi_script_path
os.system(cmd)
def load_channel_progression(folder, bias_img, downsize_factor, channel_index):
image_names = list(
map(lambda s2: os.path.join(folder, s2),
filter(lambda s: s.endswith('.ARW'), os.listdir(folder))))
def sortKey(s):
result = int(s.split(os.path.sep)[-1].strip('DSC').split('.')[-2])
if result > 9000: result -= 10000
return result
image_names.sort(key=sortKey)
print('# flat images: ', len(image_names))
# print(image_names)
cache_filename = cache_base + '/flat_channel_cache%d_%d.npy' % (
channel_index, downsize_factor)
if not os.path.exists(cache_filename):
print('no cache file...')
channel_cache = None
for i, img_fn in enumerate(tqdm.tqdm(image_names)):
if 1:
img = histogram_gap.load_raw_image(os.path.join(
folder, img_fn),
master_dark=bias_img)
channel = img[:, :, channel_index]
else:
channel = full_flat_sequence.load_flat_img(
image_names,
i,
half_images_to_average=3,
channel=channel_index,
bias_img=bias_img)
channel_downsized = channel.reshape(
(channel.shape[0] // downsize_factor, downsize_factor,
channel.shape[1] // downsize_factor,
downsize_factor)).mean(3).mean(1)
if channel_cache is None:
channel_cache = np.zeros(
(len(image_names), channel_downsized.shape[0],
channel_downsized.shape[1]),
dtype=np.float32)
channel_cache[i] = channel_downsized
if 0:
plt.imshow(channel)
plt.show()
plt.imshow(channel_downsized)
plt.show()
np.save(cache_filename, channel_cache)
print('saved flat channel cache')
else:
print('loading from cache file...')
start = datetime.datetime.now()
channel_cache = np.load(cache_filename)
print('done', (datetime.datetime.now() - start))
print(channel_cache.shape)
return channel_cache
def test_flatten_channel():
# folder = 'F:/2020/2020-04-09/shirt_dusk_flats_progression'
folder = 'F:/2020/2020-04-11/135mm_sky_flats_progression'
bias_folder = 'K:/orion_135mm_bothnights/bias'
bias_img = load_dark(bias_folder)
downsize_factor = 4
channel_index = 0
test_img_path = 'F:/Pictures/Lightroom/2020/2020-02-18/orion_135mm/DSC03637.ARW'
test_img = histogram_gap.load_raw_image(test_img_path, bias_img)
test_img_channel = test_img[:, :, channel_index]
#todo: spatial and temporal filtering
# channel_progression = load_channel_progression(folder, bias_img, downsize_factor, channel_index)
# channel_progression, image_means = load_channel_progression_brightness_filtered(folder, bias_img, downsize_factor, channel_index, num_images_to_avg=5)
channel_progression, image_means = load_channel_progression_brightness_filtered_spatialfiltered(
folder, bias_img, downsize_factor, channel_index, num_images_to_avg=5)
channel_flat = np.zeros_like(test_img_channel)
print(test_img_channel.shape, channel_progression.shape,
channel_flat.shape)
for yi in tqdm.tqdm(range(test_img_channel.shape[0])):
if 0:
for xi in range(test_img_channel.shape[1]):
input_pixel = test_img_channel[yi, xi]
progression = channel_progression[:, yi // downsize_factor,
xi // downsize_factor]
index = np.argmin(np.abs(progression - input_pixel))
if 0:
print(progression.shape, index)
plt.plot(np.arange(0, len(progression)), progression)
plt.scatter([index], [input_pixel], 'r')
plt.grid(True)
plt.show()
normalized_progression = progression / image_means
output = normalized_progression[index] #* image_means[index]
# output = image_means[index] /
# output = progression[index] / (np.mean(progression) * image_means[index])
# print(progression[index], np.mean(progression), image_means[index])
# print(output)
channel_flat[yi, xi] = output
else:
# input_pixels = test_img_channel[yi]
input_pixels = test_img_channel[yi]
input_pixels = input_pixels.reshape(
(input_pixels.shape[0] // downsize_factor,
downsize_factor)).mean(1)
progressions = channel_progression[:, yi // downsize_factor, :]
indices = np.argmin(np.abs(progressions - input_pixels), axis=0)
# print('indices: ', indices.shape)
# print('progressions: ', progressions.shape)
# normalized_progressions = progressions / image_means
# normalized_progressions = progressions.copy()
# for i in range(normalized_progressions.shape[0]):
# normalized_progressions[i] /= image_means[i]
# outputs = normalized_progressions[:, indices]
# print(outputs.shape, normalized_progressions.shape)
outputs = np.zeros((len(indices)), dtype=channel_flat.dtype)
for i in range(indices.shape[0]):
# normalized_progression = progressions[:, i] / image_means
# output = normalized_progression[indices[i]]
output = progressions[indices[i], i] / image_means[indices[i]]
outputs[i] = output
channel_flat[yi] = np.repeat(outputs, downsize_factor)
# print(channel_flat[yi, :10], outputs[:10])
# exit(0)
plt.imshow(channel_flat)
plt.title('channel flat')
plt.show()
calibrated_img = test_img_channel / channel_flat
plt.imshow(test_img_channel)
plt.title('test img')
plt.show()
plt.imshow(calibrated_img)
plt.title('calibrated test img')
plt.show()
# plt.hist(calibrated_img.flatten(), bins = 1000)
# plt.show()
benchmark_flat = full_flat_sequence.get_flat_matching_brightness_histogram_match(
folder,
test_img_channel,
channel_index,
bias_img=bias_img,
half_images_to_average=5)
plt.imshow(benchmark_flat)
plt.title('benchmark')
plt.show()
benchmark_ratio = channel_flat / benchmark_flat
plt.title('difference v benchmark')
plt.imshow(benchmark_ratio)
plt.show()