def test_add_modules(self):
pipeline = Pypeline(self.test_dir, self.test_dir, self.test_dir)
# --- Reading Modules ---
# default location
reading = FitsReadingModule(name_in="reading",
input_dir=None,
image_tag="im_arr")
pipeline.add_module(reading)
# no default location
reading2 = FitsReadingModule(name_in="reading2",
input_dir=self.test_dir,
image_tag="im_arr2")
pipeline.add_module(reading2)
# --- Processing Module ---
process = BadPixelSigmaFilterModule(image_in_tag="im_arr")
pipeline.add_module(process)
# --- Writing Module ---
# default location
write = FitsWritingModule(name_in="writing",
file_name="result.fits",
data_tag="im_arr")
pipeline.add_module(write)
# no default location
write2 = FitsWritingModule(name_in="writing2",
file_name="result.fits",
data_tag="im_arr",
output_dir=self.test_dir)
pipeline.add_module(write2)
with pytest.warns(UserWarning):
pipeline.add_module(write2)
pipeline.run()
os.remove(self.test_dir+"result.fits")
os.remove(self.test_data)
# --- Reading Module ---
# run_module
pipeline = Pypeline(self.test_dir, self.test_dir, self.test_dir)
reading = FitsReadingModule(name_in="reading",
input_dir=None,
image_tag="im_arr")
pipeline.add_module(reading)
pipeline.run_module("reading")
os.remove(self.test_data)
def test_run_non_valid_module_list(self):
pipeline = Pypeline(self.test_dir, self.test_dir, self.test_dir)
# --- Reading Modules ---
reading = FitsReadingModule(name_in="reading")
pipeline.add_module(reading)
# --- Processing Module ---
process = BadPixelSigmaFilterModule(name_in="filter", image_in_tag="im_list")
pipeline.add_module(process)
# --- Writing Module ---
write = FitsWritingModule(name_in="writing",
file_name="result.fits",
data_tag="im_list")
pipeline.add_module(write)
with pytest.raises(AttributeError):
pipeline.run()
with pytest.raises(AttributeError):
pipeline.run_module("filter")
with pytest.raises(AttributeError):
pipeline.run_module("writing")
assert pipeline.validate_pipeline_module("test") is None
os.remove(self.test_data)
def test_remove_module(self):
pipeline = Pypeline(self.test_dir, self.test_dir, self.test_dir)
# --- Reading Modules ---
reading = FitsReadingModule(name_in="reading")
pipeline.add_module(reading)
# --- Processing Module ---
process = BadPixelSigmaFilterModule(name_in="filter", image_in_tag="im_list")
pipeline.add_module(process)
assert pipeline.get_module_names() == ["reading", "filter"]
pipeline.remove_module("reading")
assert pipeline.get_module_names() == ["filter"]
assert pipeline.remove_module("filter") is True
os.remove(self.test_data)
def test_bad_pixel(self):
bad_pixel = BadPixelSigmaFilterModule(name_in="bad_pixel",
image_in_tag="im_bg",
image_out_tag="im_bp",
box=9,
sigma=8,
iterate=3)
self.pipeline.add_module(bad_pixel)
self.pipeline.run_module("bad_pixel")
data = self.pipeline.get_data("im_bp")
assert np.allclose(data[0, 0, 0],
0.00037132392435389595,
rtol=limit,
atol=0.)
assert np.allclose(np.mean(data),
2.3675404363850964e-07,
rtol=limit,
atol=0.)
assert data.shape == (78, 100, 100)
def test_bad_pixel_sigma_filter(self):
sigma = BadPixelSigmaFilterModule(name_in="sigma",
image_in_tag="images",
image_out_tag="sigma",
box=9,
sigma=5,
iterate=1)
self.pipeline.add_module(sigma)
self.pipeline.run()
storage = DataStorage(self.test_dir + "/PynPoint_database.hdf5")
storage.open_connection()
data = storage.m_data_bank["sigma"]
assert np.allclose(data[0, 0, 0],
0.00032486907273264834,
rtol=limit,
atol=0.)
assert np.allclose(data[0, 10, 10],
0.025022559679385093,
rtol=limit,
atol=0.)
assert np.allclose(data[0, 20, 20],
0.024962143884217046,
rtol=limit,
atol=0.)
assert np.allclose(np.mean(data),
6.721637736047109e-07,
rtol=limit,
atol=0.)
storage.close_connection()
def run(self):
"""
Run method of the module. Cuts out the detector sections at the different dither positions,
prepares the PCA background subtraction, applies a bad pixel correction, locates the star
in each image, runs the PCA background subtraction, combines the output from the different
dither positions is written to a single database tag.
:return: None
"""
def _admin_start(count, n_dither, position, star_pos):
if self.m_crop or self.m_prepare or self.m_pca_background:
print "Processing dither position "+str(count+1)+" out of "+str(n_dither)+"..."
print "Center position =", position
if self.m_cubes is None and self.m_center is not None:
print "DITHER_X, DITHER_Y =", tuple(star_pos)
def _admin_end(count, n_dither):
if self.m_combine == "mean":
tags.append("dither_mean"+str(count+1))
elif self.m_combine == "pca":
tags.append("dither_pca_res"+str(count+1))
if self.m_crop or self.m_prepare or self.m_pca_background:
print "Processing dither position "+str(count+1)+ \
" out of "+str(n_dither)+"... [DONE]"
n_dither, star_pos = self._initialize()
tags = []
for i, position in enumerate(self.m_center):
_admin_start(i, n_dither, position, star_pos[i])
if self.m_crop:
crop = CropImagesModule(size=self.m_size,
center=position,
name_in="crop"+str(i),
image_in_tag=self.m_image_in_tag,
image_out_tag="dither_crop"+str(i+1))
crop.connect_database(self._m_data_base)
crop.run()
if self.m_prepare:
prepare = PCABackgroundPreparationModule(dither=(n_dither,
self.m_cubes,
star_pos[i]),
mean=False,
name_in="prepare"+str(i),
image_in_tag="dither_crop"+str(i+1),
star_out_tag="dither_star"+str(i+1),
mean_out_tag="dither_mean"+str(i+1),
background_out_tag="dither_background" \
+str(i+1))
prepare.connect_database(self._m_data_base)
prepare.run()
if self.m_pca_background:
if self.m_bad_pixel is None:
tag_extract = "dither_mean"+str(i+1)
else:
tag_extract = "dither_bad"+str(i+1)
bad = BadPixelSigmaFilterModule(name_in="bad"+str(i),
image_in_tag="dither_mean"+str(i+1),
image_out_tag="dither_bad"+str(i+1),
map_out_tag="dither_bpmap"+str(i+1),
box=self.m_bad_pixel[0],
sigma=self.m_bad_pixel[1],
iterate=self.m_bad_pixel[2])
bad.connect_database(self._m_data_base)
bad.run()
star = StarExtractionModule(name_in="star"+str(i),
image_in_tag=tag_extract,
image_out_tag=None,
position_out_tag="dither_star"+str(i+1),
image_size=None,
fwhm_star=self.m_gaussian,
position=self.m_subframe)
star.connect_database(self._m_data_base)
star.run()
pca = PCABackgroundSubtractionModule(pca_number=self.m_pca_number,
mask_star=self.m_mask_star,
mask_planet=self.m_mask_planet,
name_in="pca_background"+str(i),
star_in_tag="dither_star"+str(i+1),
background_in_tag="dither_background"+str(i+1),
residuals_out_tag="dither_pca_res"+str(i+1),
fit_out_tag="dither_pca_fit"+str(i+1),
mask_out_tag="dither_pca_mask"+str(i+1))
pca.connect_database(self._m_data_base)
pca.run()
_admin_end(i, n_dither)
if self.m_combine is not None:
combine = CombineTagsModule(name_in="combine",
check_attr=True,
image_in_tags=tags,
image_out_tag="dither_combine")
combine.connect_database(self._m_data_base)
combine.run()
sort = SortParangModule(name_in="sort",
image_in_tag="dither_combine",
image_out_tag=self.m_image_out_tag)
sort.connect_database(self._m_data_base)
sort.run()
def test_docs(self):
read_science = FitsReadingModule(name_in="read_science",
input_dir=self.test_dir + "adi/",
image_tag="im_arr")
self.pipeline.add_module(read_science)
read_dark = FitsReadingModule(name_in="read_dark",
input_dir=self.test_dir + "dark/",
image_tag="dark_arr")
self.pipeline.add_module(read_dark)
read_flat = FitsReadingModule(name_in="read_flat",
input_dir=self.test_dir + "flat/",
image_tag="flat_arr")
self.pipeline.add_module(read_flat)
remove_last = RemoveLastFrameModule(name_in="last_frame",
image_in_tag="im_arr",
image_out_tag="im_arr_last")
self.pipeline.add_module(remove_last)
cutting = RemoveLinesModule(lines=(0, 0, 0, 2),
name_in="cut_lines",
image_in_tag="im_arr_last",
image_out_tag="im_arr_cut")
self.pipeline.add_module(cutting)
dark_sub = DarkCalibrationModule(name_in="dark_subtraction",
image_in_tag="im_arr_cut",
dark_in_tag="dark_arr",
image_out_tag="dark_sub_arr")
flat_sub = FlatCalibrationModule(name_in="flat_subtraction",
image_in_tag="dark_sub_arr",
flat_in_tag="flat_arr",
image_out_tag="flat_sub_arr")
self.pipeline.add_module(dark_sub)
self.pipeline.add_module(flat_sub)
bg_subtraction = MeanBackgroundSubtractionModule(
shift=None,
cubes=1,
name_in="background_subtraction",
image_in_tag="flat_sub_arr",
image_out_tag="bg_cleaned_arr")
self.pipeline.add_module(bg_subtraction)
bp_cleaning = BadPixelSigmaFilterModule(name_in="sigma_filtering",
image_in_tag="bg_cleaned_arr",
image_out_tag="bp_cleaned_arr")
self.pipeline.add_module(bp_cleaning)
extraction = StarExtractionModule(name_in="star_cutting",
image_in_tag="bp_cleaned_arr",
image_out_tag="im_arr_extract",
image_size=0.6,
fwhm_star=0.1,
position=None)
# Required for ref_image_in_tag in StarAlignmentModule, otherwise a random frame is used
ref_extract = StarExtractionModule(name_in="star_cut_ref",
image_in_tag="bp_cleaned_arr",
image_out_tag="im_arr_ref",
image_size=0.6,
fwhm_star=0.1,
position=None)
alignment = StarAlignmentModule(name_in="star_align",
image_in_tag="im_arr_extract",
ref_image_in_tag="im_arr_ref",
image_out_tag="im_arr_aligned",
accuracy=10,
resize=2)
self.pipeline.add_module(extraction)
self.pipeline.add_module(ref_extract)
self.pipeline.add_module(alignment)
angle_calc = AngleInterpolationModule(name_in="angle_calculation",
data_tag="im_arr_aligned")
self.pipeline.add_module(angle_calc)
subset = StackAndSubsetModule(name_in="stacking_subset",
image_in_tag="im_arr_aligned",
image_out_tag="im_arr_stacked",
random=None,
stacking=4)
self.pipeline.add_module(subset)
psf_sub = PSFSubtractionModule(pca_number=5,
name_in="PSF_subtraction",
images_in_tag="im_arr_stacked",
reference_in_tag="im_arr_stacked",
res_mean_tag="res_mean")
self.pipeline.add_module(psf_sub)
writing = FitsWritingModule(name_in="Fits_writing",
file_name="test.fits",
data_tag="res_mean")
self.pipeline.add_module(writing)
self.pipeline.run()
data = self.pipeline.get_data("im_arr")
assert np.allclose(data[0, 61, 39],
-0.00022889163546536875,
rtol=limit,
atol=0.)
data = self.pipeline.get_data("dark_arr")
assert np.allclose(data[0, 61, 39],
2.368170995592123e-05,
rtol=limit,
atol=0.)
data = self.pipeline.get_data("flat_arr")
assert np.allclose(data[0, 61, 39],
0.98703416941301647,
rtol=limit,
atol=0.)
data = self.pipeline.get_data("im_arr_last")
assert np.allclose(data[0, 61, 39],
-0.00022889163546536875,
rtol=limit,
atol=0.)
data = self.pipeline.get_data("im_arr_cut")
assert np.allclose(data[0, 61, 39],
-0.00022889163546536875,
rtol=limit,
atol=0.)
data = self.pipeline.get_data("dark_sub_arr")
assert np.allclose(data[0, 61, 39],
-0.00021601281733413911,
rtol=limit,
atol=0.)
data = self.pipeline.get_data("flat_sub_arr")
assert np.allclose(data[0, 61, 39],
-0.00021647987125847178,
rtol=limit,
atol=0.)
data = self.pipeline.get_data("bg_cleaned_arr")
assert np.allclose(data[0, 61, 39],
-0.00013095662386792948,
rtol=limit,
atol=0.)
data = self.pipeline.get_data("bp_cleaned_arr")
assert np.allclose(data[0, 61, 39],
-0.00013095662386792948,
rtol=limit,
atol=0.)
data = self.pipeline.get_data("im_arr_extract")
assert np.allclose(data[0, 10, 10],
0.052958146579313935,
rtol=limit,
atol=0.)
data = self.pipeline.get_data("im_arr_aligned")
assert np.allclose(data[0, 10, 10],
1.1307471842831197e-05,
rtol=limit,
atol=0.)
data = self.pipeline.get_data("im_arr_stacked")
assert np.allclose(data[0, 10, 10],
2.5572805947810986e-05,
rtol=limit,
atol=0.)
data = self.pipeline.get_data("res_mean")
assert np.allclose(data[38, 22],
0.00018312083384477404,
rtol=limit,
atol=0.)
assert np.allclose(np.mean(data),
-1.598348168584834e-07,
rtol=limit,
atol=0.)
assert data.shape == (44, 44)