def test_create_instance_using_existing_database(self): np.random.seed(1) images = np.random.normal(loc=0, scale=2e-4, size=(10, 100, 100)) h5f = h5py.File(self.test_data, "w") dset = h5f.create_dataset("images", data=images) dset.attrs['PIXSCALE'] = 0.01 h5f.close() pipeline = Pypeline(self.test_dir, self.test_dir, self.test_dir) data = pipeline.get_data("images") assert np.allclose(data[0, 0, 0], 0.00032486907273264834, rtol=limit, atol=0.) assert np.allclose(np.mean(data), 1.0506056979365338e-06, rtol=limit, atol=0.) assert pipeline.get_attribute("images", "PIXSCALE") == 0.01 os.remove(self.test_data)
class TestEndToEnd(object): def setup(self): self.test_dir = os.path.dirname(__file__) + "/" self.pipeline = Pypeline(self.test_dir, self.test_dir, self.test_dir) def test_read(self): read_fits = FitsReadingModule(name_in="read_fits", image_tag="im", overwrite=True) self.pipeline.add_module(read_fits) self.pipeline.run_module("read_fits") data = self.pipeline.get_data("im") assert np.allclose(data[0, 0, 0], 0.00032486907273264834, rtol=limit, atol=0.) assert np.allclose(np.mean(data), 9.4518306864680034e-05, rtol=limit, atol=0.) assert data.shape == (82, 102, 100) def test_remove_last(self): remove_last = RemoveLastFrameModule(name_in="remove_last", image_in_tag="im", image_out_tag="im_last") self.pipeline.add_module(remove_last) self.pipeline.run_module("remove_last") data = self.pipeline.get_data("im_last") assert np.allclose(data[0, 0, 0], 0.00032486907273264834, rtol=limit, atol=0.) assert np.allclose(np.mean(data), 9.9365399524407205e-05, rtol=limit, atol=0.) assert data.shape == (78, 102, 100) def test_parang(self): angle = AngleInterpolationModule(name_in="angle", data_tag="im_last") self.pipeline.add_module(angle) self.pipeline.run_module("angle") files = self.pipeline.get_attribute("im_last", "FILES", static=False) parang = self.pipeline.get_attribute("im_last", "PARANG", static=False) assert files[0] == self.test_dir + 'adi01.fits' assert parang[1] == 1.1904761904761905 def test_cut_lines(self): cut_lines = RemoveLinesModule(lines=(0, 0, 0, 2), name_in="cut_lines", image_in_tag="im_last", image_out_tag="im_cut") self.pipeline.add_module(cut_lines) self.pipeline.run_module("cut_lines") data = self.pipeline.get_data("im_cut") assert np.allclose(data[0, 0, 0], 0.00032486907273264834, rtol=limit, atol=0.) assert np.allclose(np.mean(data), 0.00010141595132969683, rtol=limit, atol=0.) assert data.shape == (78, 100, 100) def test_background(self): background = MeanBackgroundSubtractionModule(shift=None, cubes=1, name_in="background", image_in_tag="im_cut", image_out_tag="im_bg") self.pipeline.add_module(background) self.pipeline.run_module("background") data = self.pipeline.get_data("im_bg") assert np.allclose(data[0, 0, 0], 0.00037132392435389595, rtol=limit, atol=0.) assert np.allclose(np.mean(data), 2.3675404363850964e-07, rtol=limit, atol=0.) assert data.shape == (78, 100, 100) def test_bad_pixel(self): bad_pixel = BadPixelSigmaFilterModule(name_in="bad_pixel", image_in_tag="im_bg", image_out_tag="im_bp", box=9, sigma=8, iterate=3) self.pipeline.add_module(bad_pixel) self.pipeline.run_module("bad_pixel") data = self.pipeline.get_data("im_bp") assert np.allclose(data[0, 0, 0], 0.00037132392435389595, rtol=limit, atol=0.) assert np.allclose(np.mean(data), 2.3675404363850964e-07, rtol=limit, atol=0.) assert data.shape == (78, 100, 100) def test_star(self): star = StarExtractionModule(name_in="star", image_in_tag="im_bp", image_out_tag="im_star", image_size=1.08, fwhm_star=0.0108) self.pipeline.add_module(star) self.pipeline.run_module("star") data = self.pipeline.get_data("im_star") assert np.allclose(data[0, 0, 0], 0.00018025424208141221, rtol=limit, atol=0.) assert np.allclose(np.mean(data), 0.00063151691905138636, rtol=limit, atol=0.) assert data.shape == (78, 40, 40) def test_center(self): center = StarAlignmentModule(name_in="center", image_in_tag="im_star", ref_image_in_tag=None, image_out_tag="im_center", interpolation="spline", accuracy=10, resize=5) self.pipeline.add_module(center) self.pipeline.run_module("center") data = self.pipeline.get_data("im_center") assert np.allclose(data[1, 0, 0], 1.2113798549047296e-06, rtol=limit, atol=0.) assert np.allclose(data[16, 0, 0], 1.0022456564129139e-05, rtol=limit, atol=0.) assert np.allclose(data[50, 0, 0], 1.7024977291686637e-06, rtol=limit, atol=0.) assert np.allclose(data[67, 0, 0], 7.8143774182171561e-07, rtol=limit, atol=0.) assert np.allclose(np.mean(data), 2.5260676762055473e-05, rtol=limit, atol=0.) assert data.shape == (78, 200, 200) def test_remove_frames(self): remove_frames = RemoveFramesModule(frames=(0, 15, 49, 66), name_in="remove_frames", image_in_tag="im_center", selected_out_tag="im_remove", removed_out_tag=None) self.pipeline.add_module(remove_frames) self.pipeline.run_module("remove_frames") data = self.pipeline.get_data("im_remove") assert np.allclose(data[0, 0, 0], 1.2113798549047296e-06, rtol=limit, atol=0.) assert np.allclose(data[14, 0, 0], 1.0022456564129139e-05, rtol=limit, atol=0.) assert np.allclose(data[47, 0, 0], 1.7024977291686637e-06, rtol=limit, atol=0.) assert np.allclose(data[63, 0, 0], 7.8143774182171561e-07, rtol=limit, atol=0.) assert np.allclose(np.mean(data), 2.5255308248050269e-05, rtol=limit, atol=0.) assert data.shape == (74, 200, 200) def test_subset(self): subset = StackAndSubsetModule(name_in="subset", image_in_tag="im_remove", image_out_tag="im_subset", random=37, stacking=2) self.pipeline.add_module(subset) self.pipeline.run_module("subset") data = self.pipeline.get_data("im_subset") assert np.allclose(data[0, 0, 0], -1.9081971570461925e-06, rtol=limit, atol=0.) assert np.allclose(np.mean(data), 2.5255308248050275e-05, rtol=limit, atol=0.) assert data.shape == (37, 200, 200) def test_pca(self): pca = PSFSubtractionModule(name_in="pca", pca_number=2, images_in_tag="im_subset", reference_in_tag="im_subset", res_arr_out_tag="res_arr", res_arr_rot_out_tag="res_rot", res_mean_tag="res_mean", res_median_tag="res_median", res_var_tag="res_var", res_rot_mean_clip_tag="res_rot_mean_clip", extra_rot=0.0, cent_size=0.1) self.pipeline.add_module(pca) self.pipeline.run_module("pca") data = self.pipeline.get_data("res_mean") assert np.allclose(data[154, 99], 0.0004308570688425797, rtol=limit, atol=0.) assert np.allclose(np.mean(data), 9.372451154992271e-08, rtol=limit, atol=0.) assert data.shape == (200, 200)
class TestBackgroundSubtraction(object): def setup(self): self.test_dir = os.path.dirname(__file__) + "/" self.pipeline = Pypeline(self.test_dir, self.test_dir, self.test_dir) def test_simple_background_subraction(self): read = FitsReadingModule(name_in="read", image_tag="read", input_dir=self.test_dir+"dither") self.pipeline.add_module(read) simple = SimpleBackgroundSubtractionModule(shift=20, name_in="simple", image_in_tag="read", image_out_tag="simple") self.pipeline.add_module(simple) self.pipeline.run() data = self.pipeline.get_data("read") assert np.allclose(data[0, 74, 24], 0.05304008435511765, rtol=limit, atol=0.) assert np.allclose(np.mean(data), 0.00010033896953157959, rtol=limit, atol=0.) data = self.pipeline.get_data("simple") assert np.allclose(data[0, 74, 74], -0.05288064325101517, rtol=limit, atol=0.) assert np.allclose(np.mean(data), 2.7755575615628916e-22, rtol=limit, atol=0.) def test_mean_background_subraction(self): mean1 = MeanBackgroundSubtractionModule(shift=None, cubes=1, name_in="mean1", image_in_tag="read", image_out_tag="mean1") self.pipeline.add_module(mean1) mean2 = MeanBackgroundSubtractionModule(shift=20, cubes=1, name_in="mean2", image_in_tag="read", image_out_tag="mean2") self.pipeline.add_module(mean2) self.pipeline.run() data = self.pipeline.get_data("mean1") assert np.allclose(data[0, 74, 24], 0.0530465391626132, rtol=limit, atol=0.) assert np.allclose(np.mean(data), 1.3970872216676808e-07, rtol=limit, atol=0.) data = self.pipeline.get_data("mean2") assert np.allclose(data[0, 74, 24], 0.0530465391626132, rtol=limit, atol=0.) assert np.allclose(np.mean(data), 1.3970872216676808e-07, rtol=limit, atol=0.) def test_dithering_background(self): pca_dither1 = DitheringBackgroundModule(name_in="pca_dither1", image_in_tag="read", image_out_tag="pca_dither1", center=None, cubes=None, size=0.8, gaussian=0.1, subframe=0.1, pca_number=5, mask_star=0.1, mask_planet=None, bad_pixel=None, crop=True, prepare=True, pca_background=True, combine="pca") self.pipeline.add_module(pca_dither1) pca_dither2 = DitheringBackgroundModule(name_in="pca_dither2", image_in_tag="read", image_out_tag="pca_dither2", center=((25., 75.), (75., 75.), (75., 25.), (25., 25.)), cubes=1, size=0.8, gaussian=0.1, pca_number=5, mask_star=0.1, mask_planet=None, bad_pixel=None, crop=True, prepare=True, pca_background=True, combine="pca") self.pipeline.add_module(pca_dither2) self.pipeline.run() data = self.pipeline.get_data("dither_crop1") assert np.allclose(data[0, 13, 13], 0.05304008435511765, rtol=1e-6, atol=0.) assert np.allclose(np.mean(data), 0.0003195300604219455, rtol=1e-6, atol=0.) data = self.pipeline.get_data("dither_star1") assert np.allclose(data[0, 13, 13], 0.05304008435511765, rtol=1e-6, atol=0.) assert np.allclose(np.mean(data), 0.0012762877089355176, rtol=1e-6, atol=0.) data = self.pipeline.get_data("dither_mean1") assert np.allclose(data[0, 13, 13], 0.0530465391626132, rtol=1e-6, atol=0.) assert np.allclose(np.mean(data), 0.0012768562655335774, rtol=1e-6, atol=0.) data = self.pipeline.get_data("dither_background1") assert np.allclose(data[0, 13, 13], -0.00010629310882411674, rtol=1e-6, atol=0.) assert np.allclose(np.mean(data), 6.108442507548571e-07, rtol=1e-6, atol=0.) data = self.pipeline.get_data("dither_pca_fit1") assert np.allclose(data[0, 13, 13], 3.8003879389296064e-05, rtol=1e-6, atol=0.) assert np.allclose(np.mean(data), -2.012779401124373e-08, rtol=1e-4, atol=0.) data = self.pipeline.get_data("dither_pca_res1") assert np.allclose(data[0, 13, 13], 0.05300208047572835, rtol=1e-6, atol=0.) assert np.allclose(np.mean(data), 0.0012763078355621557, rtol=1e-6, atol=0.) data = self.pipeline.get_data("dither_pca_mask1") assert np.allclose(data[0, 13, 13], 0., rtol=1e-6, atol=0.) assert np.allclose(np.mean(data), 0.9426020408163265, rtol=1e-6, atol=0.) data = self.pipeline.get_data("pca_dither1") assert np.allclose(data[0, 13, 13], 0.05300208047572835, rtol=1e-6, atol=0.) assert np.allclose(np.mean(data), 0.0012755430390138146, rtol=1e-6, atol=0.) data = self.pipeline.get_data("pca_dither2") assert np.allclose(data[0, 13, 13], 0.05300208049366906, rtol=1e-6, atol=0.) assert np.allclose(np.mean(data), 0.0012755430394817146, rtol=1e-3, atol=0.) def test_nodding_background(self): read1 = FitsReadingModule(name_in="read1", image_tag="star", input_dir=self.test_dir+"star") self.pipeline.add_module(read1) read2 = FitsReadingModule(name_in="read2", image_tag="sky", input_dir=self.test_dir+"sky") self.pipeline.add_module(read2) nodding = NoddingBackgroundModule(name_in="nodding", sky_in_tag="sky", science_in_tag="star", image_out_tag="nodding", mode="both") self.pipeline.add_module(nodding) self.pipeline.run() data = self.pipeline.get_data("star") assert np.allclose(data[0, 50, 50], 0.09798413502193704, rtol=limit, atol=0.) assert np.allclose(np.mean(data), 0.00010029494781738066, rtol=limit, atol=0.) data = self.pipeline.get_data("sky") assert np.allclose(data[0, 50, 50], -7.613171257478652e-05, rtol=limit, atol=0.) assert np.allclose(np.mean(data), 8.937360237872607e-07, rtol=limit, atol=0.) data = self.pipeline.get_data("nodding") assert np.allclose(data[0, 50, 50], 0.09806026673451182, rtol=limit, atol=0.) assert np.allclose(np.mean(data), 9.942251790089106e-05, rtol=limit, atol=0.)
class TestDocumentation(object): def setup(self): self.test_dir = os.path.dirname(__file__) + "/" self.pipeline = Pypeline(self.test_dir, self.test_dir, self.test_dir) def test_docs(self): read_science = FitsReadingModule(name_in="read_science", input_dir=self.test_dir + "adi/", image_tag="im_arr") self.pipeline.add_module(read_science) read_dark = FitsReadingModule(name_in="read_dark", input_dir=self.test_dir + "dark/", image_tag="dark_arr") self.pipeline.add_module(read_dark) read_flat = FitsReadingModule(name_in="read_flat", input_dir=self.test_dir + "flat/", image_tag="flat_arr") self.pipeline.add_module(read_flat) remove_last = RemoveLastFrameModule(name_in="last_frame", image_in_tag="im_arr", image_out_tag="im_arr_last") self.pipeline.add_module(remove_last) cutting = RemoveLinesModule(lines=(0, 0, 0, 2), name_in="cut_lines", image_in_tag="im_arr_last", image_out_tag="im_arr_cut") self.pipeline.add_module(cutting) dark_sub = DarkCalibrationModule(name_in="dark_subtraction", image_in_tag="im_arr_cut", dark_in_tag="dark_arr", image_out_tag="dark_sub_arr") flat_sub = FlatCalibrationModule(name_in="flat_subtraction", image_in_tag="dark_sub_arr", flat_in_tag="flat_arr", image_out_tag="flat_sub_arr") self.pipeline.add_module(dark_sub) self.pipeline.add_module(flat_sub) bg_subtraction = MeanBackgroundSubtractionModule( shift=None, cubes=1, name_in="background_subtraction", image_in_tag="flat_sub_arr", image_out_tag="bg_cleaned_arr") self.pipeline.add_module(bg_subtraction) bp_cleaning = BadPixelSigmaFilterModule(name_in="sigma_filtering", image_in_tag="bg_cleaned_arr", image_out_tag="bp_cleaned_arr") self.pipeline.add_module(bp_cleaning) extraction = StarExtractionModule(name_in="star_cutting", image_in_tag="bp_cleaned_arr", image_out_tag="im_arr_extract", image_size=0.6, fwhm_star=0.1, position=None) # Required for ref_image_in_tag in StarAlignmentModule, otherwise a random frame is used ref_extract = StarExtractionModule(name_in="star_cut_ref", image_in_tag="bp_cleaned_arr", image_out_tag="im_arr_ref", image_size=0.6, fwhm_star=0.1, position=None) alignment = StarAlignmentModule(name_in="star_align", image_in_tag="im_arr_extract", ref_image_in_tag="im_arr_ref", image_out_tag="im_arr_aligned", accuracy=10, resize=2) self.pipeline.add_module(extraction) self.pipeline.add_module(ref_extract) self.pipeline.add_module(alignment) angle_calc = AngleInterpolationModule(name_in="angle_calculation", data_tag="im_arr_aligned") self.pipeline.add_module(angle_calc) subset = StackAndSubsetModule(name_in="stacking_subset", image_in_tag="im_arr_aligned", image_out_tag="im_arr_stacked", random=None, stacking=4) self.pipeline.add_module(subset) psf_sub = PSFSubtractionModule(pca_number=5, name_in="PSF_subtraction", images_in_tag="im_arr_stacked", reference_in_tag="im_arr_stacked", res_mean_tag="res_mean") self.pipeline.add_module(psf_sub) writing = FitsWritingModule(name_in="Fits_writing", file_name="test.fits", data_tag="res_mean") self.pipeline.add_module(writing) self.pipeline.run() data = self.pipeline.get_data("im_arr") assert np.allclose(data[0, 61, 39], -0.00022889163546536875, rtol=limit, atol=0.) data = self.pipeline.get_data("dark_arr") assert np.allclose(data[0, 61, 39], 2.368170995592123e-05, rtol=limit, atol=0.) data = self.pipeline.get_data("flat_arr") assert np.allclose(data[0, 61, 39], 0.98703416941301647, rtol=limit, atol=0.) data = self.pipeline.get_data("im_arr_last") assert np.allclose(data[0, 61, 39], -0.00022889163546536875, rtol=limit, atol=0.) data = self.pipeline.get_data("im_arr_cut") assert np.allclose(data[0, 61, 39], -0.00022889163546536875, rtol=limit, atol=0.) data = self.pipeline.get_data("dark_sub_arr") assert np.allclose(data[0, 61, 39], -0.00021601281733413911, rtol=limit, atol=0.) data = self.pipeline.get_data("flat_sub_arr") assert np.allclose(data[0, 61, 39], -0.00021647987125847178, rtol=limit, atol=0.) data = self.pipeline.get_data("bg_cleaned_arr") assert np.allclose(data[0, 61, 39], -0.00013095662386792948, rtol=limit, atol=0.) data = self.pipeline.get_data("bp_cleaned_arr") assert np.allclose(data[0, 61, 39], -0.00013095662386792948, rtol=limit, atol=0.) data = self.pipeline.get_data("im_arr_extract") assert np.allclose(data[0, 10, 10], 0.052958146579313935, rtol=limit, atol=0.) data = self.pipeline.get_data("im_arr_aligned") assert np.allclose(data[0, 10, 10], 1.1307471842831197e-05, rtol=limit, atol=0.) data = self.pipeline.get_data("im_arr_stacked") assert np.allclose(data[0, 10, 10], 2.5572805947810986e-05, rtol=limit, atol=0.) data = self.pipeline.get_data("res_mean") assert np.allclose(data[38, 22], 0.00018312083384477404, rtol=limit, atol=0.) assert np.allclose(np.mean(data), -1.598348168584834e-07, rtol=limit, atol=0.) assert data.shape == (44, 44)
class TestPSFSubtractionPCA(object): def setup(self): self.test_dir = os.path.dirname(__file__) + "/" self.pipeline = Pypeline(self.test_dir, self.test_dir, self.test_dir) def test_psf_subtraction_pca(self): read = FitsReadingModule(name_in="read", image_tag="read") self.pipeline.add_module(read) angle = AngleInterpolationModule(name_in="angle", data_tag="read") self.pipeline.add_module(angle) pca = PcaPsfSubtractionModule(pca_numbers=(5, ), name_in="pca", images_in_tag="read", reference_in_tag="read", res_mean_tag="res_mean", res_median_tag="res_median", res_arr_out_tag="res_arr", res_rot_mean_clip_tag="res_clip", basis_out_tag="basis", extra_rot=-15., verbose=True) self.pipeline.add_module(pca) self.pipeline.run() data = self.pipeline.get_data("read") assert np.allclose(data[0, 50, 50], 0.09798413502193708, rtol=limit, atol=0.) assert np.allclose(np.mean(data), 0.00010063896953157961, rtol=limit, atol=0.) assert data.shape == (80, 100, 100) data = self.pipeline.get_data("res_mean") assert np.allclose(data[0, 50, 50], 1.947810457180298e-06, rtol=limit, atol=0.) assert np.allclose(data[0, 59, 46], 0.00016087655925993273, rtol=limit, atol=0.) assert np.allclose(np.mean(data), 3.184676024912574e-08, rtol=limit, atol=0.) assert data.shape == (1, 100, 100) data = self.pipeline.get_data("res_median") assert np.allclose(data[0, 50, 50], -2.223389676715259e-06, rtol=limit, atol=0.) assert np.allclose(data[0, 59, 46], 0.00015493570876347953, rtol=limit, atol=0.) assert np.allclose(np.mean(data), 1.250907785757355e-07, rtol=limit, atol=0.) assert data.shape == (1, 100, 100) data = self.pipeline.get_data("res_clip") assert np.allclose(data[0, 50, 50], 2.2828813434810948e-06, rtol=limit, atol=0.) assert np.allclose(data[0, 59, 46], 1.0816254290076103e-05, rtol=limit, atol=0.) assert np.allclose(np.mean(data), 2.052077475694807e-06, rtol=limit, atol=0.) assert data.shape == (1, 100, 100) data = self.pipeline.get_data("res_arr5") assert np.allclose(data[0, 50, 50], -0.00010775091764735749, rtol=limit, atol=0.) assert np.allclose(data[0, 59, 46], 0.0001732810184783699, rtol=limit, atol=0.) assert np.allclose(np.mean(data), 3.184676024912564e-08, rtol=limit, atol=0.) assert data.shape == (80, 100, 100) data = self.pipeline.get_data("basis") assert np.allclose(data[0, 50, 50], -0.005866797940467074, rtol=limit, atol=0.) assert np.allclose(data[0, 59, 46], 0.0010154680995154122, rtol=limit, atol=0.) assert np.allclose(np.mean(data), -4.708475279640416e-05, rtol=limit, atol=0.) assert data.shape == (5, 100, 100)
class TestPSFpreparation(object): def setup(self): self.test_dir = os.path.dirname(__file__) + "/" self.pipeline = Pypeline(self.test_dir, self.test_dir, self.test_dir) def test_psf_preparation(self): read = FitsReadingModule(name_in="read", image_tag="read") self.pipeline.add_module(read) angle = AngleInterpolationModule(name_in="angle", data_tag="read") self.pipeline.add_module(angle) prep = PSFpreparationModule(name_in="prep", image_in_tag="read", image_out_tag="prep", mask_out_tag="mask", norm=True, resize=2., cent_size=0.1, edge_size=1.0, verbose=True) self.pipeline.add_module(prep) sdi = SDIpreparationModule(name_in="sdi", wavelength=(0.65, 0.6), width=(0.1, 0.5), image_in_tag="read", image_out_tag="sdi") self.pipeline.add_module(sdi) self.pipeline.run() data = self.pipeline.get_data("read") assert np.allclose(data[0, 25, 25], 2.0926464668090656e-05, rtol=limit, atol=0.) assert np.allclose(np.mean(data), 0.00010029494781738066, rtol=limit, atol=0.) assert data.shape == (40, 100, 100) data = self.pipeline.get_data("prep") assert np.allclose(data[0, 25, 25], 0., rtol=limit, atol=0.) assert np.allclose(data[0, 99, 99], 0., rtol=limit, atol=0.) assert np.allclose(np.mean(data), 0.0001818623671899089, rtol=limit, atol=0.) assert data.shape == (40, 200, 200) data = self.pipeline.get_data("sdi") assert np.allclose(data[0, 25, 25], -2.6648118007008814e-05, rtol=limit, atol=0.) assert np.allclose(np.mean(data), 2.0042892634995876e-05, rtol=limit, atol=0.) assert data.shape == (40, 100, 100)