def test_no_background(self): diff, ruined_image, optKernel, bkg = ois.optimal_system( self.image, self.ref_img, kernelshape=(11, 11), bkgdegree=None, method="Alard-Lupton", gausslist=self.mygausslist) self.assertEqual(np.linalg.norm(bkg.flatten(), ord=np.inf), 0.0) norm_diff = np.linalg.norm(diff) / np.linalg.norm(self.image) self.assertLess(norm_diff, 1E-2) diff, ruined_image, optKernel, bkg = ois.optimal_system( self.image, self.ref_img, kernelshape=(11, 11), bkgdegree=None, method="Bramich") self.assertEqual(np.linalg.norm(bkg.flatten(), ord=np.inf), 0.0) norm_diff = np.linalg.norm(diff) / np.linalg.norm(self.image) self.assertLess(norm_diff, 1E-2) diff, ruined_image, optKernel, bkg = ois.optimal_system( self.image, self.ref_img, kernelshape=(11, 11), bkgdegree=None, method="AdaptiveBramich", poly_degree=1) self.assertEqual(np.linalg.norm(bkg.flatten(), ord=np.inf), 0.0) norm_diff = np.linalg.norm(diff) / np.linalg.norm(self.image) self.assertLess(norm_diff, 1E-2)
def test_image_dims(self): with self.assertRaises(ValueError): diff, opt_image, krn, bkg = ois.optimal_system( self.img, np.random.random((10, 10, 100))) with self.assertRaises(ValueError): diff, opt_image, krn, bkg = ois.optimal_system( np.random.random((10, 10, 100)), self.ref) with self.assertRaises(ValueError): diff, opt_image, krn, bkg = ois.optimal_system( np.zeros((5, 5)), np.zeros((7, 7)))
def subtract(hduls, name="SCI"): """ Returns differences of a set of images from a template image Arguments: hduls -- list of fits hdul's where the last image is the template name -- name of the HDU to use image that the other images will be subtracted from """ hduls = [h for h in hduls] output = [] outputs = [] template = combine(hduls, name)["PRIMARY"].data for hdu in hduls: diff = ois.optimal_system(image=hdu[name].data, refimage=template, method='Bramich')[0] output.append(diff) for array_set in output: # FIXME this is ragingly wrong, multiple items should be associated for item in array_set: hdu = fits.PrimaryHDU(item) outputs.append(fits.HDUList([hdu])) return (hdul for hdul in outputs)
def subtract(sources, template): output = [] if type(sources) == list: for element in sources: diff = ois.optimal_system(image=element.data, refimage=template, method='Bramich') output.append(diff) return output
def the_grid_test(self, method_name, **kwargs): gh, gw = 2, 2 diff, opt, krn, bkg = ois.optimal_system( self.img, self.ref, method=method_name, gridshape=(gh, gw), kernelshape=(11, 11), **kwargs) # Assert it's not returning masked arrays self.assertFalse(isinstance(diff, np.ma.MaskedArray)) self.assertFalse(isinstance(opt, np.ma.MaskedArray)) h, w = self.img.shape diff_grid = diff[:h // gh, :w // gw] k_spill = 5 img_crop = self.img[:h // gh + k_spill, :w // gw + k_spill] ref_crop = self.ref[:h // gh + k_spill, :w // gw + k_spill] diff_crop, opt, krn, bkg = ois.optimal_system( img_crop, ref_crop, method=method_name, gridshape=None, kernelshape=(11, 11), **kwargs) diff_crop = diff_crop[:h // gh, :w // gw] norm_diff = np.linalg.norm(diff_grid - diff_crop) # Assert it does the same on grid or not self.assertLess(norm_diff, 1E-10) h, w = self.img.shape diff_grid = diff[:h // gh, w // gw:] k_spill = 5 img_crop = self.img[:h // gh + k_spill, w // gw - k_spill:] ref_crop = self.ref[:h // gh + k_spill, w // gw - k_spill:] diff_crop, opt, krn, bkg = ois.optimal_system( img_crop, ref_crop, method=method_name, gridshape=None, kernelshape=(11, 11), **kwargs) diff_crop = diff_crop[:h // gh, k_spill:] norm_diff = np.linalg.norm(diff_grid - diff_crop) # Assert it does the same on grid or not self.assertLess(norm_diff, 1E-10)
def test_optimal_system_bramich(self): # Test Bramich diff, ruined_image, optKernel, bkg = ois.optimal_system( self.image, self.ref_img, kernelshape=(11, 11), bkgdegree=self.bkgdeg, method="Bramich") norm_diff = np.linalg.norm(diff) / np.linalg.norm(self.image) self.assertLess(norm_diff, 1E-10) self.assertFalse(isinstance(diff, np.ma.MaskedArray)) self.assertFalse(isinstance(ruined_image, np.ma.MaskedArray)) self.assertFalse(isinstance(bkg, np.ma.MaskedArray))
def test_both_bramich_consistency(self): k_side = 3 image = np.random.random((10, 10)) refimage = np.random.random((10, 10)) k_shape = (k_side, k_side) diff, opt_img, opt_k, bkg = ois.optimal_system( image, refimage, kernelshape=k_shape, bkgdegree=None, method="Bramich") diff, opt_img, opt_vark, bkg = ois.optimal_system( image, refimage, kernelshape=k_shape, bkgdegree=None, method="AdaptiveBramich", poly_degree=0) self.assertEqual(opt_vark.shape, (k_side, k_side, 1)) opt_vark = opt_vark.reshape((k_side, k_side)) diff_norm = np.linalg.norm((opt_k - opt_vark).flatten(), ord=np.inf) kernel_norm = np.linalg.norm(opt_k.flatten(), ord=np.inf) self.assertLess(diff_norm / kernel_norm, 1E-8)
def test_Bramich_align(self): diff, opt, krn, bkg = ois.optimal_system( self.img, self.ref, method="Bramich") norm_diff = np.linalg.norm(diff) / np.linalg.norm(self.ref) # Assert it's a good subtraction self.assertLess(norm_diff, 1E-3) # Assert it's not returning masked arrays self.assertFalse(isinstance(diff, np.ma.MaskedArray)) self.assertFalse(isinstance(opt, np.ma.MaskedArray))
def test_AlardLupton_diffPSF(self): diff, opt, krn, bkg = ois.optimal_system( self.img, self.ref, method="Alard-Lupton", gausslist=[{'sx': 1.5, 'sy': 1.5}]) norm_diff = np.linalg.norm(diff) / np.linalg.norm(self.ref) # Assert it's a good subtraction self.assertLess(norm_diff, 1E-3) # Assert it's not returning masked arrays self.assertFalse(isinstance(diff, np.ma.MaskedArray)) self.assertFalse(isinstance(opt, np.ma.MaskedArray))
def test_AdaptiveBramich_mask(self): diff, opt, krn, bkg = ois.optimal_system( self.img_masked, self.ref_masked, method="AdaptiveBramich") norm_diff = np.linalg.norm(diff.compressed()) \ / np.linalg.norm(self.ref_masked.compressed()) # Assert it's a good subtraction self.assertLess(norm_diff, 1E-3) # Assert it's returning masked arrays self.assertTrue(isinstance(diff, np.ma.MaskedArray)) self.assertTrue(isinstance(opt, np.ma.MaskedArray))
def test_AlardLupton_align(self): diff, opt, krn, bkg = ois.optimal_system( self.img, self.ref, method="Alard-Lupton", kernelshape=(5, 5), gausslist=[{'sx': 0.01, 'sy':0.01, 'center': (2 - self.x0, 2 - self.y0), 'modPolyDeg':0}]) norm_diff = np.linalg.norm(diff) / np.linalg.norm(self.ref) # Assert it's a good subtraction self.assertLess(norm_diff, 1E-3) # Assert it's not returning masked arrays self.assertFalse(isinstance(diff, np.ma.MaskedArray)) self.assertFalse(isinstance(opt, np.ma.MaskedArray))
def test_AlardLupton_mask(self): diff, opt, krn, bkg = ois.optimal_system( self.img_masked, self.ref_masked, method="Alard-Lupton", gausslist=[{'sx': 1.5, 'sy': 1.5}]) norm_diff = np.linalg.norm(diff.compressed()) \ / np.linalg.norm(self.ref_masked.compressed()) # Assert it's a good subtraction self.assertLess(norm_diff, 1E-3) # Assert it's returning masked arrays self.assertTrue(isinstance(diff, np.ma.MaskedArray)) self.assertTrue(isinstance(opt, np.ma.MaskedArray))
def test_both_bramich_consistency(self): deg = 0 bkg_deg = 0 k_side = 3 image = np.random.random((10, 10)) refimage = np.random.random((10, 10)) k_shape = (k_side, k_side) diff, opt_img, opt_k, bkg = ois.optimal_system( image, refimage, kernelshape=k_shape, bkgdegree=bkg_deg, method="Bramich") diff, opt_img, opt_vark, bkg = ois.optimal_system( image, refimage, kernelshape=k_shape, bkgdegree=bkg_deg, method="AdaptiveBramich", poly_degree=deg) self.assertEqual(opt_vark.shape, (k_side, k_side, 1)) opt_vark = opt_vark.reshape((k_side, k_side)) diff_norm = np.linalg.norm((opt_k - opt_vark).flatten(), ord=np.inf) kernel_norm = np.linalg.norm(opt_k.flatten(), ord=np.inf) self.assertLess(diff_norm / kernel_norm, 1E-8)
def test_optimal_system_alardlp(self): # Test Alard & Lupton diff, ruined_image, optKernel, bkg = ois.optimal_system( self.image, self.ref_img, kernelshape=(11, 11), bkgdegree=self.bkgdeg, method="Alard-Lupton", gausslist=self.mygausslist) norm_diff = np.linalg.norm(diff) / np.linalg.norm(self.image) self.assertLess(norm_diff, 1E-10) self.assertFalse(isinstance(diff, np.ma.MaskedArray)) self.assertFalse(isinstance(ruined_image, np.ma.MaskedArray)) self.assertFalse(isinstance(bkg, np.ma.MaskedArray))
def test_background_AdaptiveBramich(self): diff, opt, krn, bkg = ois.optimal_system( self.img, self.ref, method="AdaptiveBramich", bkgdegree=2) # Assert it's a good subtraction norm_diff = np.linalg.norm(diff) / np.linalg.norm(self.ref) self.assertLess(norm_diff, 1E-3) # Assert we had a good background estimation norm_bkg_diff = np.linalg.norm(bkg - self.bkg) / np.linalg.norm(bkg) self.assertLess(norm_bkg_diff, 1E-3) # Assert it's not returning masked arrays self.assertFalse(isinstance(diff, np.ma.MaskedArray)) self.assertFalse(isinstance(opt, np.ma.MaskedArray))
def test_background_AlardLupton(self): diff, opt, krn, bkg = ois.optimal_system( self.img, self.ref, method="Alard-Lupton", gausslist=[{'sx': 1.5, 'sy': 1.5}], bkgdegree=2) # Assert it's a good subtraction norm_diff = np.linalg.norm(diff) / np.linalg.norm(self.ref) self.assertLess(norm_diff, 1E-3) # Assert we had a good background estimation norm_bkg_diff = np.linalg.norm(bkg - self.bkg) / np.linalg.norm(bkg) self.assertLess(norm_bkg_diff, 1E-3) # Assert it's not returning masked arrays self.assertFalse(isinstance(diff, np.ma.MaskedArray)) self.assertFalse(isinstance(opt, np.ma.MaskedArray))
def subtract(hduls): """ Returns differences of a set of images from a template image Arguments: hduls --list of fits hdul's where the last image is the template image that the other images will be subtracted from """ hduls_list = [hdul for hdul in hduls] output = [] outputs = [] # FIXME this is not a combined image template = hduls_list[-1][0] try: template = hduls_list[-1][0].data except AttributeError: pass return (fits.HDUList(fits.PrimaryHDU(ois.optimal_system( \ image=hdu["SCI"].data, refimage=template, method="Bramich")[0])) \ for hdu in hduls_list)
def subtract(hduls): """ Returns differences of a set of images from a template image Arguments: hduls --list of fits hdul's where the last image is the template image that the other images will be subtracted from """ hduls_list = [hdul for hdul in hduls] output = [] outputs = [] template = to_np(hduls_list[-1][0]) for hdu in hduls[:-1]: diff = ois.optimal_system(image=hdu["SCI"].data, refimage=template, method='Bramich') output.append(diff) for array_set in output: for item in array_set: hdu = fits.PrimaryHDU(item) outputs.append(fits.HDUList([hdu])) return (hdul for hdul in outputs)
def main(imgs_dir, ref_path, new_path, details): if not os.path.isdir(imgs_dir): os.makedirs(imgs_dir) ref_dest = os.path.join(imgs_dir, 'ref.fits') new_dest = os.path.join(imgs_dir, 'new.fits') os.rename(ref_path, ref_dest) os.rename(new_path, new_dest) ref = si.SingleImage(ref_dest, borders=False) #, crop=((150,150), (150, 150))) new = si.SingleImage(new_dest, borders=False) #, crop=((150,150), (150, 150))) #~ ## Adding stars foo = new.cov_matrix srcs = new.best_sources rows = [] for i in range(15): j = np.random.choice(len(srcs), 1, replace=False) flux = srcs['flux'][j][0] xs = srcs['x'][j][0] ys = srcs['y'][j][0] position = (ys, xs) sx, sy = new.stamp_shape sx += 3 sy += 3 star = extract_array(new.pixeldata.data, (sx, sy), position, mode='partial', fill_value=new._bkg.globalrms) #print flux #~ star = flux*new.db.load(j)[0] x = np.random.choice(new.pixeldata.shape[0]-3*sx, 1)[0] + sx#* np.random.random()) y = np.random.choice(new.pixeldata.shape[1]-3*sy, 1)[0] + sy # np.int((new.pixeldata.shape[1]-star.shape[1]) * np.random.random()) #~ if new.pixeldata.data[x:x+sx, y:y+sy].shape != star.shape: #~ import ipdb; ipdb.set_trace() new.pixeldata.data[x:x+sx, y:y+sy] = star #~ except: #~ continue xc = x+sx/2. yc = y+sy/2. app_mag = -2.5*np.log10(flux)+25. rows.append([yc, xc, app_mag, flux]) newcat = Table(rows=rows, names=['x', 'y', 'app_mag', 'flux']) fits.writeto(filename=new_dest, header=fits.getheader(new_dest), data=new.pixeldata.data, overwrite=True) new._clean() new = si.SingleImage(new_dest, borders=False) #, crop=((150,150), (150, 150))) newcat.write(os.path.join(imgs_dir, 'transient.list'), format='ascii.fast_no_header', overwrite=True) fits.writeto(filename=os.path.join(imgs_dir, 'interp_ref.fits'), data=ref.interped, overwrite=True) fits.writeto(filename=os.path.join(imgs_dir, 'interp_new.fits'), data=new.interped, overwrite=True) try: print 'Images to be subtracted: {} {}'.format(ref_dest, new_dest) import time t0 = time.time() D, P, S, mask = ps.diff(ref, new, align=False, iterative=False, shift=False, beta=True) dt_z = time.time() - t0 new._clean() ref._clean() mea, med, std = sigma_clipped_stats(D.real) D = np.ma.MaskedArray(D.real, mask).filled(mea) fits.writeto(os.path.join(imgs_dir,'diff.fits'), D, overwrite=True) #~ utils.encapsule_R(D, path=os.path.join(imgs_dir, 'diff.fits')) utils.encapsule_R(P, path=os.path.join(imgs_dir, 'psf_d.fits')) utils.encapsule_R(S, path=os.path.join(imgs_dir, 's_diff.fits')) scorrdetected = utils.find_S_local_maxima(S, threshold=3.5) print 'S_corr found thath {} transients were above 3.5 sigmas'.format(len(scorrdetected)) ascii.write(table=np.asarray(scorrdetected), output=os.path.join(imgs_dir, 's_corr_detected.csv'), names=['X_IMAGE', 'Y_IMAGE', 'SIGNIFICANCE'], format='csv') S = np.ascontiguousarray(S) #~ s_bkg = sep.Background(S) mean, median, std = sigma_clipped_stats(S) sdetected = sep.extract(S-median, 3.5*std, filter_kernel=None) print 'S_corr with sep found thath {} transients were above 3.5 sigmas'.format(len(sdetected)) ascii.write(table=sdetected, output=os.path.join(imgs_dir, 'sdetected.csv'), format='csv') ## With OIS t0 = time.time() ois_d = ois.optimal_system(fits.getdata(new_dest), fits.getdata(ref_dest))[0] dt_o = time.time() - t0 utils.encapsule_R(ois_d, path=os.path.join(imgs_dir, 'diff_ois.fits')) ## With HOTPANTS t0 = time.time() os.system('hotpants -v 0 -inim {} -tmplim {} -outim {} -r 15 -tu 40000 -tl -100 -il -100 -iu 40000'.format(new_dest, ref_dest, os.path.join(imgs_dir, 'diff_hot.fits'))) dt_h = time.time() - t0 return [newcat.to_pandas(), [dt_z, dt_o, dt_h]] except: raise
def test_wrong_method_name(self): with self.assertRaises(ValueError): diff, opt_image, krn, bkg = ois.optimal_system( self.img, self.ref, method="WrongName")
def test_even_side_kernel(self): for bad_shape in ((8, 9), (9, 8), (8, 8)): with self.assertRaises(ois.EvenSideKernelError): ois.optimal_system(self.img, self.ref, bad_shape)
start = time.time() # # this scirpt gets all the files in a directoty and appends them to a list to be used stating from all_files[0] as 1st file # image_list_prelim = [] for frame in glob.glob("*.fit"): image_list_prelim.append(frame) image_list = [fits.getdata(image) for image in image_list_prelim] compressed_image = np.sum(image_list, axis=0) hdu_diff = fits.PrimaryHDU(diff_data, header=fits.getdata(image_list[0])) # Read FITS data ref_data = fits.getdata(image_list[0]) targ_data = fits.getdata(compressed_image) # # Read FITS headers ref_hdr = ref_fits[0].header targ_hdr = targ_fits[0].header # aligned_array = [] # #subtaction of compressed image to reference image # diff_data, optimal_image, kernel, background = ois.optimal_system( image_list, ref_data, kernelshape=(11, 11), method="Bramich") hdu_diff = fits.PrimaryHDU(diff_data, header=ref_hdr) hdu_diff.writeto("final_sub.fit", overwrite=True) # time = end - start print("Ended in", "%.1f" % time, "seconds!")
#Difference Imaging ref_fits = fits.open( "/home/linkmaster/Documents/CTMO/Projects/Pipeline/AGN_Project/Reference/dss_search" ) targ_fits = fits.open("final_stacked_image.fit") # stack image FITS image # Read FITS data ref_data = fits.getdata( "/home/linkmaster/Documents/CTMO/Projects/Pipeline/AGN_Project/Reference/dss_search" ) targ_data = fits.getdata("final_stacked_image.fit") # Read FITS headers ref_hdr = ref_fits[0].header targ_hdr = targ_fits[0].header # Alignment aligned_array, footprint = reproject_interp( targ_fits, ref_hdr) #aligned array is the new fits file aligned print(type(aligned_array)) aligned_hdu = fits.PrimaryHDU(aligned_array, header=ref_hdr) aligned_hdu.writeto("aligned_to.fit", overwrite=True) # Subtraction diff_data, optimal_image, kernel, background = ois.optimal_system( aligned_array, ref_data, kernelshape=(11, 11), method="Bramich") hdu_diff = fits.PrimaryHDU(diff_data, header=ref_hdr) hdu_diff.writeto("sub.fit", overwrite=True)