def test_run_non_existing_module(self):
pipeline = Pypeline(self.test_dir, self.test_dir, self.test_dir)
with pytest.warns(UserWarning):
pipeline.run_module("test")
os.remove(self.test_data)
def test_add_non_module_as_module(self):
pipeline = Pypeline(self.test_dir, self.test_dir, self.test_dir)
with pytest.raises(AssertionError):
pipeline.add_module(None)
os.remove(self.test_data)
def test_create_instance_missing_directory(self):
dir_non_exists = self.test_dir + "none/"
dir_exists = self.test_dir
with pytest.raises(AssertionError):
Pypeline(dir_non_exists, dir_exists, dir_exists)
with pytest.raises(AssertionError):
Pypeline(dir_exists, dir_non_exists, dir_non_exists)
with pytest.raises(AssertionError):
Pypeline()
def test_create_instance_using_existing_database(self):
np.random.seed(1)
images = np.random.normal(loc=0, scale=2e-4, size=(10, 100, 100))
h5f = h5py.File(self.test_data, "w")
dset = h5f.create_dataset("images", data=images)
dset.attrs['PIXSCALE'] = 0.01
h5f.close()
pipeline = Pypeline(self.test_dir, self.test_dir, self.test_dir)
data = pipeline.get_data("images")
assert np.allclose(data[0, 0, 0], 0.00032486907273264834, rtol=limit, atol=0.)
assert np.allclose(np.mean(data), 1.0506056979365338e-06, rtol=limit, atol=0.)
assert pipeline.get_attribute("images", "PIXSCALE") == 0.01
os.remove(self.test_data)
def test_create_instance_new_data_base(self):
pipeline = Pypeline(self.test_dir, self.test_dir, self.test_dir)
pipeline.m_data_storage.open_connection()
pipeline.m_data_storage.close_connection()
del pipeline
os.remove(self.test_data)
def create_wdir(cls, dir_in, **kwargs):
tmp_pypeline = Pypeline(dir_in, dir_in, dir_in)
obj = cls(tmp_pypeline)
obj._save_kwargs(**kwargs)
reading = FitsReadingModule(name_in="reading_mod",
input_dir=dir_in,
image_tag=obj._m_image_data_tag)
obj._pypeline.add_module(reading)
obj._pypeline.run_module("reading_mod")
obj._prepare_data()
return obj
class TestDarkAndFlatCalibration(object):
def setup(self):
self.test_dir = os.path.dirname(__file__) + "/"
self.pipeline = Pypeline(self.test_dir, self.test_dir, self.test_dir)
def test_dark_and_flat_calibration(self):
dark = DarkCalibrationModule(name_in="dark",
image_in_tag="images",
dark_in_tag="dark",
image_out_tag="dark_cal")
self.pipeline.add_module(dark)
flat = FlatCalibrationModule(name_in="flat",
image_in_tag="dark_cal",
flat_in_tag="flat",
image_out_tag="flat_cal")
self.pipeline.add_module(flat)
self.pipeline.run()
storage = DataStorage(self.test_dir + "/PynPoint_database.hdf5")
storage.open_connection()
data = storage.m_data_bank["dark"]
assert np.allclose(data[0, 10, 10],
3.528694163309295e-05,
rtol=limit,
atol=0.)
assert np.allclose(np.mean(data),
7.368663496379876e-07,
rtol=limit,
atol=0.)
data = storage.m_data_bank["flat"]
assert np.allclose(data[0, 10, 10],
-0.0004053528990466237,
rtol=limit,
atol=0.)
assert np.allclose(np.mean(data),
-4.056978234798532e-07,
rtol=limit,
atol=0.)
storage.close_connection()
def create_restore(cls, filename, pypline_working_place=None):
head, tail = os.path.split(filename)
if pypline_working_place is None:
working_place = head
else:
working_place = pypline_working_place
tmp_pypeline = Pypeline(working_place, head, head)
obj = cls(tmp_pypeline)
reading = Hdf5ReadingModule("reading",
input_filename=tail,
input_dir=head,
tag_dictionary=obj._m_restore_tag_dict)
obj._pypeline.add_module(reading)
obj._pypeline.run_module("reading")
return obj
def test_remove_module(self):
pipeline = Pypeline(self.test_dir, self.test_dir, self.test_dir)
# --- Reading Modules ---
reading = FitsReadingModule(name_in="reading")
pipeline.add_module(reading)
# --- Processing Module ---
process = BadPixelSigmaFilterModule(name_in="filter", image_in_tag="im_list")
pipeline.add_module(process)
assert pipeline.get_module_names() == ["reading", "filter"]
pipeline.remove_module("reading")
assert pipeline.get_module_names() == ["filter"]
assert pipeline.remove_module("filter") is True
os.remove(self.test_data)
class TestPSFSubtractionPCA(object):
def setup(self):
self.test_dir = os.path.dirname(__file__) + "/"
self.pipeline = Pypeline(self.test_dir, self.test_dir, self.test_dir)
def test_psf_subtraction_pca(self):
read = FitsReadingModule(name_in="read",
image_tag="read")
self.pipeline.add_module(read)
angle = AngleInterpolationModule(name_in="angle",
data_tag="read")
self.pipeline.add_module(angle)
pca = PcaPsfSubtractionModule(pca_numbers=(5, ),
name_in="pca",
images_in_tag="read",
reference_in_tag="read",
res_mean_tag="res_mean",
res_median_tag="res_median",
res_arr_out_tag="res_arr",
res_rot_mean_clip_tag="res_clip",
basis_out_tag="basis",
extra_rot=-15.,
verbose=True)
self.pipeline.add_module(pca)
self.pipeline.run()
data = self.pipeline.get_data("read")
assert np.allclose(data[0, 50, 50], 0.09798413502193708, rtol=limit, atol=0.)
assert np.allclose(np.mean(data), 0.00010063896953157961, rtol=limit, atol=0.)
assert data.shape == (80, 100, 100)
data = self.pipeline.get_data("res_mean")
assert np.allclose(data[0, 50, 50], 1.947810457180298e-06, rtol=limit, atol=0.)
assert np.allclose(data[0, 59, 46], 0.00016087655925993273, rtol=limit, atol=0.)
assert np.allclose(np.mean(data), 3.184676024912574e-08, rtol=limit, atol=0.)
assert data.shape == (1, 100, 100)
data = self.pipeline.get_data("res_median")
assert np.allclose(data[0, 50, 50], -2.223389676715259e-06, rtol=limit, atol=0.)
assert np.allclose(data[0, 59, 46], 0.00015493570876347953, rtol=limit, atol=0.)
assert np.allclose(np.mean(data), 1.250907785757355e-07, rtol=limit, atol=0.)
assert data.shape == (1, 100, 100)
data = self.pipeline.get_data("res_clip")
assert np.allclose(data[0, 50, 50], 2.2828813434810948e-06, rtol=limit, atol=0.)
assert np.allclose(data[0, 59, 46], 1.0816254290076103e-05, rtol=limit, atol=0.)
assert np.allclose(np.mean(data), 2.052077475694807e-06, rtol=limit, atol=0.)
assert data.shape == (1, 100, 100)
data = self.pipeline.get_data("res_arr5")
assert np.allclose(data[0, 50, 50], -0.00010775091764735749, rtol=limit, atol=0.)
assert np.allclose(data[0, 59, 46], 0.0001732810184783699, rtol=limit, atol=0.)
assert np.allclose(np.mean(data), 3.184676024912564e-08, rtol=limit, atol=0.)
assert data.shape == (80, 100, 100)
data = self.pipeline.get_data("basis")
assert np.allclose(data[0, 50, 50], -0.005866797940467074, rtol=limit, atol=0.)
assert np.allclose(data[0, 59, 46], 0.0010154680995154122, rtol=limit, atol=0.)
assert np.allclose(np.mean(data), -4.708475279640416e-05, rtol=limit, atol=0.)
assert data.shape == (5, 100, 100)
class TestBadPixelCleaning(object):
def setup(self):
self.test_dir = os.path.dirname(__file__) + "/"
self.pipeline = Pypeline(self.test_dir, self.test_dir, self.test_dir)
def test_bad_pixel_sigma_filter(self):
sigma = BadPixelSigmaFilterModule(name_in="sigma",
image_in_tag="images",
image_out_tag="sigma",
box=9,
sigma=5,
iterate=1)
self.pipeline.add_module(sigma)
self.pipeline.run()
storage = DataStorage(self.test_dir + "/PynPoint_database.hdf5")
storage.open_connection()
data = storage.m_data_bank["sigma"]
assert np.allclose(data[0, 0, 0],
0.00032486907273264834,
rtol=limit,
atol=0.)
assert np.allclose(data[0, 10, 10],
0.025022559679385093,
rtol=limit,
atol=0.)
assert np.allclose(data[0, 20, 20],
0.024962143884217046,
rtol=limit,
atol=0.)
assert np.allclose(np.mean(data),
6.721637736047109e-07,
rtol=limit,
atol=0.)
storage.close_connection()
def test_bad_pixel_map(self):
bp_map = BadPixelMapModule(name_in="bp_map",
dark_in_tag="dark",
flat_in_tag="flat",
bp_map_out_tag="bp_map",
dark_threshold=0.99,
flat_threshold=-0.99)
self.pipeline.add_module(bp_map)
self.pipeline.run()
storage = DataStorage(self.test_dir + "/PynPoint_database.hdf5")
storage.open_connection()
data = storage.m_data_bank["bp_map"]
assert data[0, 0] == 1.
assert data[30, 30] == 1.
assert data[10, 10] == 0.
assert data[12, 12] == 0.
assert data[14, 14] == 0.
assert data[20, 20] == 0.
assert data[22, 22] == 0.
assert data[24, 24] == 0.
assert np.mean(data) == 0.9993
storage.close_connection()
def test_bad_pixel_interpolation(self):
interpolation = BadPixelInterpolationModule(
name_in="interpolation",
image_in_tag="images",
bad_pixel_map_tag="bp_map",
image_out_tag="interpolation",
iterations=100)
self.pipeline.add_module(interpolation)
self.pipeline.run()
storage = DataStorage(self.test_dir + "/PynPoint_database.hdf5")
storage.open_connection()
data = storage.m_data_bank["interpolation"]
assert np.allclose(data[0, 0, 0],
0.00032486907273264834,
rtol=limit,
atol=0.)
assert np.allclose(data[0, 10, 10],
1.0139222106683477e-05,
rtol=limit,
atol=0.)
assert np.allclose(data[0, 20, 20],
-4.686852973820094e-05,
rtol=limit,
atol=0.)
assert np.allclose(np.mean(data),
3.0499629451215465e-07,
rtol=limit,
atol=0.)
storage.close_connection()
class TestStarAlignment(object):
def setup(self):
self.test_dir = os.path.dirname(__file__) + "/"
self.pipeline = Pypeline(self.test_dir, self.test_dir, self.test_dir)
def test_star_alignment(self):
read = FitsReadingModule(name_in="read",
image_tag="read")
self.pipeline.add_module(read)
extraction = StarExtractionModule(name_in="extract",
image_in_tag="read",
image_out_tag="extract",
image_size=0.6,
fwhm_star=0.1,
position=None)
self.pipeline.add_module(extraction)
align = StarAlignmentModule(name_in="align",
image_in_tag="extract",
ref_image_in_tag=None,
image_out_tag="align",
accuracy=10,
resize=2)
self.pipeline.add_module(align)
shift = ShiftImagesModule((6., 4.),
name_in="shift",
image_in_tag="align",
image_out_tag="shift")
self.pipeline.add_module(shift)
center = StarCenteringModule(name_in="center",
image_in_tag="shift",
image_out_tag="center",
mask_out_tag=None,
fit_out_tag="center_fit",
method="full",
interpolation="spline",
radius=0.1,
sign="positive",
guess=(6., 4., 1., 1., 1., 0.))
self.pipeline.add_module(center)
self.pipeline.run()
storage = DataStorage(self.test_dir+"/PynPoint_database.hdf5")
storage.open_connection()
data = storage.m_data_bank["read"]
assert np.allclose(data[0, 10, 10], 0.00012958496246258364, rtol=limit, atol=0.)
assert np.allclose(np.mean(data), 9.832838021311831e-05, rtol=limit, atol=0.)
data = storage.m_data_bank["extract"]
assert np.allclose(data[0, 10, 10], 0.05304008435511765, rtol=limit, atol=0.)
assert np.allclose(np.mean(data), 0.0020655767159466613, rtol=limit, atol=0.)
data = storage.m_data_bank["header_extract/STAR_POSITION"]
assert data[10, 0] == data[10, 1] == 75
data = storage.m_data_bank["shift"]
assert np.allclose(data[0, 10, 10], -4.341611534220891e-05, rtol=limit, atol=0.)
assert np.allclose(np.mean(data), 0.0005164420068450968, rtol=limit, atol=0.)
data = storage.m_data_bank["center"]
assert np.allclose(data[0, 10, 10], 4.128859892625027e-05, rtol=1e-4, atol=0.)
assert np.allclose(np.mean(data), 0.0005163806188663894, rtol=1e-7, atol=0.)
storage.close_connection()
class TestBackgroundSubtraction(object):
def setup(self):
self.test_dir = os.path.dirname(__file__) + "/"
self.pipeline = Pypeline(self.test_dir, self.test_dir, self.test_dir)
def test_simple_background_subraction(self):
read = FitsReadingModule(name_in="read",
image_tag="read",
input_dir=self.test_dir+"dither")
self.pipeline.add_module(read)
simple = SimpleBackgroundSubtractionModule(shift=20,
name_in="simple",
image_in_tag="read",
image_out_tag="simple")
self.pipeline.add_module(simple)
self.pipeline.run()
data = self.pipeline.get_data("read")
assert np.allclose(data[0, 74, 24], 0.05304008435511765, rtol=limit, atol=0.)
assert np.allclose(np.mean(data), 0.00010033896953157959, rtol=limit, atol=0.)
data = self.pipeline.get_data("simple")
assert np.allclose(data[0, 74, 74], -0.05288064325101517, rtol=limit, atol=0.)
assert np.allclose(np.mean(data), 2.7755575615628916e-22, rtol=limit, atol=0.)
def test_mean_background_subraction(self):
mean1 = MeanBackgroundSubtractionModule(shift=None,
cubes=1,
name_in="mean1",
image_in_tag="read",
image_out_tag="mean1")
self.pipeline.add_module(mean1)
mean2 = MeanBackgroundSubtractionModule(shift=20,
cubes=1,
name_in="mean2",
image_in_tag="read",
image_out_tag="mean2")
self.pipeline.add_module(mean2)
self.pipeline.run()
data = self.pipeline.get_data("mean1")
assert np.allclose(data[0, 74, 24], 0.0530465391626132, rtol=limit, atol=0.)
assert np.allclose(np.mean(data), 1.3970872216676808e-07, rtol=limit, atol=0.)
data = self.pipeline.get_data("mean2")
assert np.allclose(data[0, 74, 24], 0.0530465391626132, rtol=limit, atol=0.)
assert np.allclose(np.mean(data), 1.3970872216676808e-07, rtol=limit, atol=0.)
def test_dithering_background(self):
pca_dither1 = DitheringBackgroundModule(name_in="pca_dither1",
image_in_tag="read",
image_out_tag="pca_dither1",
center=None,
cubes=None,
size=0.8,
gaussian=0.1,
subframe=0.1,
pca_number=5,
mask_star=0.1,
mask_planet=None,
bad_pixel=None,
crop=True,
prepare=True,
pca_background=True,
combine="pca")
self.pipeline.add_module(pca_dither1)
pca_dither2 = DitheringBackgroundModule(name_in="pca_dither2",
image_in_tag="read",
image_out_tag="pca_dither2",
center=((25., 75.), (75., 75.), (75., 25.), (25., 25.)),
cubes=1,
size=0.8,
gaussian=0.1,
pca_number=5,
mask_star=0.1,
mask_planet=None,
bad_pixel=None,
crop=True,
prepare=True,
pca_background=True,
combine="pca")
self.pipeline.add_module(pca_dither2)
self.pipeline.run()
data = self.pipeline.get_data("dither_crop1")
assert np.allclose(data[0, 13, 13], 0.05304008435511765, rtol=1e-6, atol=0.)
assert np.allclose(np.mean(data), 0.0003195300604219455, rtol=1e-6, atol=0.)
data = self.pipeline.get_data("dither_star1")
assert np.allclose(data[0, 13, 13], 0.05304008435511765, rtol=1e-6, atol=0.)
assert np.allclose(np.mean(data), 0.0012762877089355176, rtol=1e-6, atol=0.)
data = self.pipeline.get_data("dither_mean1")
assert np.allclose(data[0, 13, 13], 0.0530465391626132, rtol=1e-6, atol=0.)
assert np.allclose(np.mean(data), 0.0012768562655335774, rtol=1e-6, atol=0.)
data = self.pipeline.get_data("dither_background1")
assert np.allclose(data[0, 13, 13], -0.00010629310882411674, rtol=1e-6, atol=0.)
assert np.allclose(np.mean(data), 6.108442507548571e-07, rtol=1e-6, atol=0.)
data = self.pipeline.get_data("dither_pca_fit1")
assert np.allclose(data[0, 13, 13], 3.8003879389296064e-05, rtol=1e-6, atol=0.)
assert np.allclose(np.mean(data), -2.012779401124373e-08, rtol=1e-4, atol=0.)
data = self.pipeline.get_data("dither_pca_res1")
assert np.allclose(data[0, 13, 13], 0.05300208047572835, rtol=1e-6, atol=0.)
assert np.allclose(np.mean(data), 0.0012763078355621557, rtol=1e-6, atol=0.)
data = self.pipeline.get_data("dither_pca_mask1")
assert np.allclose(data[0, 13, 13], 0., rtol=1e-6, atol=0.)
assert np.allclose(np.mean(data), 0.9426020408163265, rtol=1e-6, atol=0.)
data = self.pipeline.get_data("pca_dither1")
assert np.allclose(data[0, 13, 13], 0.05300208047572835, rtol=1e-6, atol=0.)
assert np.allclose(np.mean(data), 0.0012755430390138146, rtol=1e-6, atol=0.)
data = self.pipeline.get_data("pca_dither2")
assert np.allclose(data[0, 13, 13], 0.05300208049366906, rtol=1e-6, atol=0.)
assert np.allclose(np.mean(data), 0.0012755430394817146, rtol=1e-3, atol=0.)
def test_nodding_background(self):
read1 = FitsReadingModule(name_in="read1",
image_tag="star",
input_dir=self.test_dir+"star")
self.pipeline.add_module(read1)
read2 = FitsReadingModule(name_in="read2",
image_tag="sky",
input_dir=self.test_dir+"sky")
self.pipeline.add_module(read2)
nodding = NoddingBackgroundModule(name_in="nodding",
sky_in_tag="sky",
science_in_tag="star",
image_out_tag="nodding",
mode="both")
self.pipeline.add_module(nodding)
self.pipeline.run()
data = self.pipeline.get_data("star")
assert np.allclose(data[0, 50, 50], 0.09798413502193704, rtol=limit, atol=0.)
assert np.allclose(np.mean(data), 0.00010029494781738066, rtol=limit, atol=0.)
data = self.pipeline.get_data("sky")
assert np.allclose(data[0, 50, 50], -7.613171257478652e-05, rtol=limit, atol=0.)
assert np.allclose(np.mean(data), 8.937360237872607e-07, rtol=limit, atol=0.)
data = self.pipeline.get_data("nodding")
assert np.allclose(data[0, 50, 50], 0.09806026673451182, rtol=limit, atol=0.)
assert np.allclose(np.mean(data), 9.942251790089106e-05, rtol=limit, atol=0.)
class TestDetectionLimits(object):
def setup(self):
self.test_dir = os.path.dirname(__file__) + "/"
self.pipeline = Pypeline(self.test_dir, self.test_dir, self.test_dir)
def test_contrast_curve(self):
read = FitsReadingModule(name_in="read", image_tag="read")
self.pipeline.add_module(read)
angle = AngleInterpolationModule(name_in="angle", data_tag="read")
self.pipeline.add_module(angle)
contrast = ContrastCurveModule(name_in="contrast",
image_in_tag="read",
psf_in_tag="read",
pca_out_tag="pca",
contrast_out_tag="limits",
separation=(0.5, 0.6, 0.1),
angle=(0., 360., 180.),
magnitude=(7.5, 1.),
sigma=5.,
accuracy=1e-1,
psf_scaling=1.,
aperture=0.1,
ignore=True,
pca_number=15,
norm=False,
cent_size=None,
edge_size=None,
extra_rot=0.)
self.pipeline.add_module(contrast)
self.pipeline.run()
storage = DataStorage(self.test_dir + "/PynPoint_database.hdf5")
storage.open_connection()
data = storage.m_data_bank["read"]
assert np.allclose(data[0, 10, 10],
0.00012958496246258364,
rtol=limit,
atol=0.)
assert np.allclose(np.mean(data),
0.00010029494781738066,
rtol=limit,
atol=0.)
data = storage.m_data_bank["header_read/PARANG"]
assert data[5] == 2.7777777777777777
data = storage.m_data_bank["pca"]
assert np.allclose(data[9, 68, 49],
5.707647718560735e-05,
rtol=limit,
atol=0.)
assert np.allclose(np.mean(data),
-3.66890878538392e-08,
rtol=limit,
atol=0.)
data = storage.m_data_bank["pca"]
assert np.allclose(data[21, 31, 50],
5.4392925807364694e-05,
rtol=limit,
atol=0.)
assert np.allclose(np.mean(data),
-3.668908785383954e-08,
rtol=limit,
atol=0.)
storage.close_connection()
class TestDocumentation(object):
def setup(self):
self.test_dir = os.path.dirname(__file__) + "/"
self.pipeline = Pypeline(self.test_dir, self.test_dir, self.test_dir)
def test_docs(self):
read_science = FitsReadingModule(name_in="read_science",
input_dir=self.test_dir + "adi/",
image_tag="im_arr")
self.pipeline.add_module(read_science)
read_dark = FitsReadingModule(name_in="read_dark",
input_dir=self.test_dir + "dark/",
image_tag="dark_arr")
self.pipeline.add_module(read_dark)
read_flat = FitsReadingModule(name_in="read_flat",
input_dir=self.test_dir + "flat/",
image_tag="flat_arr")
self.pipeline.add_module(read_flat)
remove_last = RemoveLastFrameModule(name_in="last_frame",
image_in_tag="im_arr",
image_out_tag="im_arr_last")
self.pipeline.add_module(remove_last)
cutting = RemoveLinesModule(lines=(0, 0, 0, 2),
name_in="cut_lines",
image_in_tag="im_arr_last",
image_out_tag="im_arr_cut")
self.pipeline.add_module(cutting)
dark_sub = DarkCalibrationModule(name_in="dark_subtraction",
image_in_tag="im_arr_cut",
dark_in_tag="dark_arr",
image_out_tag="dark_sub_arr")
flat_sub = FlatCalibrationModule(name_in="flat_subtraction",
image_in_tag="dark_sub_arr",
flat_in_tag="flat_arr",
image_out_tag="flat_sub_arr")
self.pipeline.add_module(dark_sub)
self.pipeline.add_module(flat_sub)
bg_subtraction = MeanBackgroundSubtractionModule(
shift=None,
cubes=1,
name_in="background_subtraction",
image_in_tag="flat_sub_arr",
image_out_tag="bg_cleaned_arr")
self.pipeline.add_module(bg_subtraction)
bp_cleaning = BadPixelSigmaFilterModule(name_in="sigma_filtering",
image_in_tag="bg_cleaned_arr",
image_out_tag="bp_cleaned_arr")
self.pipeline.add_module(bp_cleaning)
extraction = StarExtractionModule(name_in="star_cutting",
image_in_tag="bp_cleaned_arr",
image_out_tag="im_arr_extract",
image_size=0.6,
fwhm_star=0.1,
position=None)
# Required for ref_image_in_tag in StarAlignmentModule, otherwise a random frame is used
ref_extract = StarExtractionModule(name_in="star_cut_ref",
image_in_tag="bp_cleaned_arr",
image_out_tag="im_arr_ref",
image_size=0.6,
fwhm_star=0.1,
position=None)
alignment = StarAlignmentModule(name_in="star_align",
image_in_tag="im_arr_extract",
ref_image_in_tag="im_arr_ref",
image_out_tag="im_arr_aligned",
accuracy=10,
resize=2)
self.pipeline.add_module(extraction)
self.pipeline.add_module(ref_extract)
self.pipeline.add_module(alignment)
angle_calc = AngleInterpolationModule(name_in="angle_calculation",
data_tag="im_arr_aligned")
self.pipeline.add_module(angle_calc)
subset = StackAndSubsetModule(name_in="stacking_subset",
image_in_tag="im_arr_aligned",
image_out_tag="im_arr_stacked",
random=None,
stacking=4)
self.pipeline.add_module(subset)
psf_sub = PSFSubtractionModule(pca_number=5,
name_in="PSF_subtraction",
images_in_tag="im_arr_stacked",
reference_in_tag="im_arr_stacked",
res_mean_tag="res_mean")
self.pipeline.add_module(psf_sub)
writing = FitsWritingModule(name_in="Fits_writing",
file_name="test.fits",
data_tag="res_mean")
self.pipeline.add_module(writing)
self.pipeline.run()
data = self.pipeline.get_data("im_arr")
assert np.allclose(data[0, 61, 39],
-0.00022889163546536875,
rtol=limit,
atol=0.)
data = self.pipeline.get_data("dark_arr")
assert np.allclose(data[0, 61, 39],
2.368170995592123e-05,
rtol=limit,
atol=0.)
data = self.pipeline.get_data("flat_arr")
assert np.allclose(data[0, 61, 39],
0.98703416941301647,
rtol=limit,
atol=0.)
data = self.pipeline.get_data("im_arr_last")
assert np.allclose(data[0, 61, 39],
-0.00022889163546536875,
rtol=limit,
atol=0.)
data = self.pipeline.get_data("im_arr_cut")
assert np.allclose(data[0, 61, 39],
-0.00022889163546536875,
rtol=limit,
atol=0.)
data = self.pipeline.get_data("dark_sub_arr")
assert np.allclose(data[0, 61, 39],
-0.00021601281733413911,
rtol=limit,
atol=0.)
data = self.pipeline.get_data("flat_sub_arr")
assert np.allclose(data[0, 61, 39],
-0.00021647987125847178,
rtol=limit,
atol=0.)
data = self.pipeline.get_data("bg_cleaned_arr")
assert np.allclose(data[0, 61, 39],
-0.00013095662386792948,
rtol=limit,
atol=0.)
data = self.pipeline.get_data("bp_cleaned_arr")
assert np.allclose(data[0, 61, 39],
-0.00013095662386792948,
rtol=limit,
atol=0.)
data = self.pipeline.get_data("im_arr_extract")
assert np.allclose(data[0, 10, 10],
0.052958146579313935,
rtol=limit,
atol=0.)
data = self.pipeline.get_data("im_arr_aligned")
assert np.allclose(data[0, 10, 10],
1.1307471842831197e-05,
rtol=limit,
atol=0.)
data = self.pipeline.get_data("im_arr_stacked")
assert np.allclose(data[0, 10, 10],
2.5572805947810986e-05,
rtol=limit,
atol=0.)
data = self.pipeline.get_data("res_mean")
assert np.allclose(data[38, 22],
0.00018312083384477404,
rtol=limit,
atol=0.)
assert np.allclose(np.mean(data),
-1.598348168584834e-07,
rtol=limit,
atol=0.)
assert data.shape == (44, 44)
def test_add_modules(self):
pipeline = Pypeline(self.test_dir, self.test_dir, self.test_dir)
# --- Reading Modules ---
# default location
reading = FitsReadingModule(name_in="reading",
input_dir=None,
image_tag="im_arr")
pipeline.add_module(reading)
# no default location
reading2 = FitsReadingModule(name_in="reading2",
input_dir=self.test_dir,
image_tag="im_arr2")
pipeline.add_module(reading2)
# --- Processing Module ---
process = BadPixelSigmaFilterModule(image_in_tag="im_arr")
pipeline.add_module(process)
# --- Writing Module ---
# default location
write = FitsWritingModule(name_in="writing",
file_name="result.fits",
data_tag="im_arr")
pipeline.add_module(write)
# no default location
write2 = FitsWritingModule(name_in="writing2",
file_name="result.fits",
data_tag="im_arr",
output_dir=self.test_dir)
pipeline.add_module(write2)
with pytest.warns(UserWarning):
pipeline.add_module(write2)
pipeline.run()
os.remove(self.test_dir+"result.fits")
os.remove(self.test_data)
# --- Reading Module ---
# run_module
pipeline = Pypeline(self.test_dir, self.test_dir, self.test_dir)
reading = FitsReadingModule(name_in="reading",
input_dir=None,
image_tag="im_arr")
pipeline.add_module(reading)
pipeline.run_module("reading")
os.remove(self.test_data)
class TestEndToEnd(object):
def setup(self):
self.test_dir = os.path.dirname(__file__) + "/"
self.pipeline = Pypeline(self.test_dir, self.test_dir, self.test_dir)
def test_read(self):
read_fits = FitsReadingModule(name_in="read_fits",
image_tag="im",
overwrite=True)
self.pipeline.add_module(read_fits)
self.pipeline.run_module("read_fits")
data = self.pipeline.get_data("im")
assert np.allclose(data[0, 0, 0],
0.00032486907273264834,
rtol=limit,
atol=0.)
assert np.allclose(np.mean(data),
9.4518306864680034e-05,
rtol=limit,
atol=0.)
assert data.shape == (82, 102, 100)
def test_remove_last(self):
remove_last = RemoveLastFrameModule(name_in="remove_last",
image_in_tag="im",
image_out_tag="im_last")
self.pipeline.add_module(remove_last)
self.pipeline.run_module("remove_last")
data = self.pipeline.get_data("im_last")
assert np.allclose(data[0, 0, 0],
0.00032486907273264834,
rtol=limit,
atol=0.)
assert np.allclose(np.mean(data),
9.9365399524407205e-05,
rtol=limit,
atol=0.)
assert data.shape == (78, 102, 100)
def test_parang(self):
angle = AngleInterpolationModule(name_in="angle", data_tag="im_last")
self.pipeline.add_module(angle)
self.pipeline.run_module("angle")
files = self.pipeline.get_attribute("im_last", "FILES", static=False)
parang = self.pipeline.get_attribute("im_last", "PARANG", static=False)
assert files[0] == self.test_dir + 'adi01.fits'
assert parang[1] == 1.1904761904761905
def test_cut_lines(self):
cut_lines = RemoveLinesModule(lines=(0, 0, 0, 2),
name_in="cut_lines",
image_in_tag="im_last",
image_out_tag="im_cut")
self.pipeline.add_module(cut_lines)
self.pipeline.run_module("cut_lines")
data = self.pipeline.get_data("im_cut")
assert np.allclose(data[0, 0, 0],
0.00032486907273264834,
rtol=limit,
atol=0.)
assert np.allclose(np.mean(data),
0.00010141595132969683,
rtol=limit,
atol=0.)
assert data.shape == (78, 100, 100)
def test_background(self):
background = MeanBackgroundSubtractionModule(shift=None,
cubes=1,
name_in="background",
image_in_tag="im_cut",
image_out_tag="im_bg")
self.pipeline.add_module(background)
self.pipeline.run_module("background")
data = self.pipeline.get_data("im_bg")
assert np.allclose(data[0, 0, 0],
0.00037132392435389595,
rtol=limit,
atol=0.)
assert np.allclose(np.mean(data),
2.3675404363850964e-07,
rtol=limit,
atol=0.)
assert data.shape == (78, 100, 100)
def test_bad_pixel(self):
bad_pixel = BadPixelSigmaFilterModule(name_in="bad_pixel",
image_in_tag="im_bg",
image_out_tag="im_bp",
box=9,
sigma=8,
iterate=3)
self.pipeline.add_module(bad_pixel)
self.pipeline.run_module("bad_pixel")
data = self.pipeline.get_data("im_bp")
assert np.allclose(data[0, 0, 0],
0.00037132392435389595,
rtol=limit,
atol=0.)
assert np.allclose(np.mean(data),
2.3675404363850964e-07,
rtol=limit,
atol=0.)
assert data.shape == (78, 100, 100)
def test_star(self):
star = StarExtractionModule(name_in="star",
image_in_tag="im_bp",
image_out_tag="im_star",
image_size=1.08,
fwhm_star=0.0108)
self.pipeline.add_module(star)
self.pipeline.run_module("star")
data = self.pipeline.get_data("im_star")
assert np.allclose(data[0, 0, 0],
0.00018025424208141221,
rtol=limit,
atol=0.)
assert np.allclose(np.mean(data),
0.00063151691905138636,
rtol=limit,
atol=0.)
assert data.shape == (78, 40, 40)
def test_center(self):
center = StarAlignmentModule(name_in="center",
image_in_tag="im_star",
ref_image_in_tag=None,
image_out_tag="im_center",
interpolation="spline",
accuracy=10,
resize=5)
self.pipeline.add_module(center)
self.pipeline.run_module("center")
data = self.pipeline.get_data("im_center")
assert np.allclose(data[1, 0, 0],
1.2113798549047296e-06,
rtol=limit,
atol=0.)
assert np.allclose(data[16, 0, 0],
1.0022456564129139e-05,
rtol=limit,
atol=0.)
assert np.allclose(data[50, 0, 0],
1.7024977291686637e-06,
rtol=limit,
atol=0.)
assert np.allclose(data[67, 0, 0],
7.8143774182171561e-07,
rtol=limit,
atol=0.)
assert np.allclose(np.mean(data),
2.5260676762055473e-05,
rtol=limit,
atol=0.)
assert data.shape == (78, 200, 200)
def test_remove_frames(self):
remove_frames = RemoveFramesModule(frames=(0, 15, 49, 66),
name_in="remove_frames",
image_in_tag="im_center",
selected_out_tag="im_remove",
removed_out_tag=None)
self.pipeline.add_module(remove_frames)
self.pipeline.run_module("remove_frames")
data = self.pipeline.get_data("im_remove")
assert np.allclose(data[0, 0, 0],
1.2113798549047296e-06,
rtol=limit,
atol=0.)
assert np.allclose(data[14, 0, 0],
1.0022456564129139e-05,
rtol=limit,
atol=0.)
assert np.allclose(data[47, 0, 0],
1.7024977291686637e-06,
rtol=limit,
atol=0.)
assert np.allclose(data[63, 0, 0],
7.8143774182171561e-07,
rtol=limit,
atol=0.)
assert np.allclose(np.mean(data),
2.5255308248050269e-05,
rtol=limit,
atol=0.)
assert data.shape == (74, 200, 200)
def test_subset(self):
subset = StackAndSubsetModule(name_in="subset",
image_in_tag="im_remove",
image_out_tag="im_subset",
random=37,
stacking=2)
self.pipeline.add_module(subset)
self.pipeline.run_module("subset")
data = self.pipeline.get_data("im_subset")
assert np.allclose(data[0, 0, 0],
-1.9081971570461925e-06,
rtol=limit,
atol=0.)
assert np.allclose(np.mean(data),
2.5255308248050275e-05,
rtol=limit,
atol=0.)
assert data.shape == (37, 200, 200)
def test_pca(self):
pca = PSFSubtractionModule(name_in="pca",
pca_number=2,
images_in_tag="im_subset",
reference_in_tag="im_subset",
res_arr_out_tag="res_arr",
res_arr_rot_out_tag="res_rot",
res_mean_tag="res_mean",
res_median_tag="res_median",
res_var_tag="res_var",
res_rot_mean_clip_tag="res_rot_mean_clip",
extra_rot=0.0,
cent_size=0.1)
self.pipeline.add_module(pca)
self.pipeline.run_module("pca")
data = self.pipeline.get_data("res_mean")
assert np.allclose(data[154, 99],
0.0004308570688425797,
rtol=limit,
atol=0.)
assert np.allclose(np.mean(data),
9.372451154992271e-08,
rtol=limit,
atol=0.)
assert data.shape == (200, 200)
class TestFluxAndPosition(object):
def setup(self):
self.test_dir = os.path.dirname(__file__) + "/"
self.pipeline = Pypeline(self.test_dir, self.test_dir, self.test_dir)
def test_fake_planet(self):
read = FitsReadingModule(name_in="read",
image_tag="read")
self.pipeline.add_module(read)
angle = AngleInterpolationModule(name_in="angle",
data_tag="read")
self.pipeline.add_module(angle)
fake = FakePlanetModule(position=(0.5, 90.),
magnitude=5.,
psf_scaling=1.,
interpolation="spline",
name_in="fake",
image_in_tag="read",
psf_in_tag="read",
image_out_tag="fake",
verbose=True)
self.pipeline.add_module(fake)
simplex = SimplexMinimizationModule(position=(31., 49.),
magnitude=5.,
psf_scaling=-1.,
name_in="simplex",
image_in_tag="fake",
psf_in_tag="read",
res_out_tag="simplex_res",
flux_position_tag="flux_position",
merit="sum",
aperture=0.05,
sigma=0.027,
tolerance=0.1,
pca_number=2,
cent_size=None,
edge_size=None,
extra_rot=0.)
self.pipeline.add_module(simplex)
pca = PcaPsfSubtractionModule(pca_numbers=(2, ),
name_in="pca",
images_in_tag="fake",
reference_in_tag="fake",
res_mean_tag="res_mean",
res_median_tag=None,
res_arr_out_tag=None,
res_rot_mean_clip_tag=None,
extra_rot=0.)
self.pipeline.add_module(pca)
false = FalsePositiveModule(position=(31., 49.),
aperture=0.1,
ignore=True,
name_in="false",
image_in_tag="res_mean",
snr_out_tag="snr_fpf")
self.pipeline.add_module(false)
photometry = AperturePhotometryModule(radius=0.1,
position=None,
name_in="photometry",
image_in_tag="read",
phot_out_tag="photometry")
self.pipeline.add_module(photometry)
self.pipeline.run()
storage = DataStorage(self.test_dir+"/PynPoint_database.hdf5")
storage.open_connection()
data = storage.m_data_bank["read"]
assert np.allclose(data[0, 10, 10], 0.00012958496246258364, rtol=limit, atol=0.)
assert np.allclose(np.mean(data), 0.00010029494781738066, rtol=limit, atol=0.)
data = storage.m_data_bank["header_read/PARANG"]
assert data[5] == 2.7777777777777777
data = storage.m_data_bank["fake"]
assert np.allclose(data[0, 49, 31], 0.00036532633147006946, rtol=limit, atol=0.)
assert np.allclose(np.mean(data), 0.0001012983225928772, rtol=limit, atol=0.)
data = storage.m_data_bank["simplex_res"]
assert np.allclose(data[46, 49, 31], 3.718481593648487e-05, rtol=limit, atol=0.)
assert np.allclose(np.mean(data), -2.8892749617545238e-08, rtol=limit, atol=0.)
data = storage.m_data_bank["flux_position"]
assert np.allclose(data[46, 0], 31.276994533457994, rtol=limit, atol=0.)
assert np.allclose(data[46, 1], 50.10345749706295, rtol=limit, atol=0.)
assert np.allclose(data[46, 2], 0.5055288651354779, rtol=limit, atol=0.)
assert np.allclose(data[46, 3], 89.6834045889695, rtol=limit, atol=0.)
assert np.allclose(data[46, 4], 4.997674024675655, rtol=limit, atol=0.)
data = storage.m_data_bank["res_mean"]
assert np.allclose(data[0, 49, 31], 9.258255068620805e-05, rtol=limit, atol=0.)
assert np.allclose(np.mean(data), -2.610863424405134e-08, rtol=limit, atol=0.)
data = storage.m_data_bank["snr_fpf"]
assert np.allclose(data[0, 2], 0.513710034941892, rtol=limit, atol=0.)
assert np.allclose(data[0, 3], 93.01278750418334, rtol=limit, atol=0.)
assert np.allclose(data[0, 4], 11.775360946367874, rtol=limit, atol=0.)
assert np.allclose(data[0, 5], 2.9838031156970146e-08, rtol=limit, atol=0.)
data = storage.m_data_bank["photometry"]
assert np.allclose(data[0][0], 0.983374353660573, rtol=limit, atol=0.)
assert np.allclose(data[39][0], 0.9841484973083519, rtol=limit, atol=0.)
assert np.allclose(np.mean(data), 0.9835085649488583, rtol=limit, atol=0.)
storage.close_connection()
def test_run_non_valid_module_list(self):
pipeline = Pypeline(self.test_dir, self.test_dir, self.test_dir)
# --- Reading Modules ---
reading = FitsReadingModule(name_in="reading")
pipeline.add_module(reading)
# --- Processing Module ---
process = BadPixelSigmaFilterModule(name_in="filter", image_in_tag="im_list")
pipeline.add_module(process)
# --- Writing Module ---
write = FitsWritingModule(name_in="writing",
file_name="result.fits",
data_tag="im_list")
pipeline.add_module(write)
with pytest.raises(AttributeError):
pipeline.run()
with pytest.raises(AttributeError):
pipeline.run_module("filter")
with pytest.raises(AttributeError):
pipeline.run_module("writing")
assert pipeline.validate_pipeline_module("test") is None
os.remove(self.test_data)
def setup(self):
self.test_dir = os.path.dirname(__file__) + "/"
self.pipeline = Pypeline(self.test_dir, self.test_dir, self.test_dir)
class TestPSFpreparation(object):
def setup(self):
self.test_dir = os.path.dirname(__file__) + "/"
self.pipeline = Pypeline(self.test_dir, self.test_dir, self.test_dir)
def test_psf_preparation(self):
read = FitsReadingModule(name_in="read", image_tag="read")
self.pipeline.add_module(read)
angle = AngleInterpolationModule(name_in="angle", data_tag="read")
self.pipeline.add_module(angle)
prep = PSFpreparationModule(name_in="prep",
image_in_tag="read",
image_out_tag="prep",
mask_out_tag="mask",
norm=True,
resize=2.,
cent_size=0.1,
edge_size=1.0,
verbose=True)
self.pipeline.add_module(prep)
sdi = SDIpreparationModule(name_in="sdi",
wavelength=(0.65, 0.6),
width=(0.1, 0.5),
image_in_tag="read",
image_out_tag="sdi")
self.pipeline.add_module(sdi)
self.pipeline.run()
data = self.pipeline.get_data("read")
assert np.allclose(data[0, 25, 25],
2.0926464668090656e-05,
rtol=limit,
atol=0.)
assert np.allclose(np.mean(data),
0.00010029494781738066,
rtol=limit,
atol=0.)
assert data.shape == (40, 100, 100)
data = self.pipeline.get_data("prep")
assert np.allclose(data[0, 25, 25], 0., rtol=limit, atol=0.)
assert np.allclose(data[0, 99, 99], 0., rtol=limit, atol=0.)
assert np.allclose(np.mean(data),
0.0001818623671899089,
rtol=limit,
atol=0.)
assert data.shape == (40, 200, 200)
data = self.pipeline.get_data("sdi")
assert np.allclose(data[0, 25, 25],
-2.6648118007008814e-05,
rtol=limit,
atol=0.)
assert np.allclose(np.mean(data),
2.0042892634995876e-05,
rtol=limit,
atol=0.)
assert data.shape == (40, 100, 100)
