def test_no_background(self):
diff, ruined_image, optKernel, bkg = ois.optimal_system(
self.image, self.ref_img,
kernelshape=(11, 11),
bkgdegree=None,
method="Alard-Lupton",
gausslist=self.mygausslist)
self.assertEqual(np.linalg.norm(bkg.flatten(), ord=np.inf), 0.0)
norm_diff = np.linalg.norm(diff) / np.linalg.norm(self.image)
self.assertLess(norm_diff, 1E-2)
diff, ruined_image, optKernel, bkg = ois.optimal_system(
self.image, self.ref_img,
kernelshape=(11, 11),
bkgdegree=None,
method="Bramich")
self.assertEqual(np.linalg.norm(bkg.flatten(), ord=np.inf), 0.0)
norm_diff = np.linalg.norm(diff) / np.linalg.norm(self.image)
self.assertLess(norm_diff, 1E-2)
diff, ruined_image, optKernel, bkg = ois.optimal_system(
self.image, self.ref_img,
kernelshape=(11, 11),
bkgdegree=None,
method="AdaptiveBramich",
poly_degree=1)
self.assertEqual(np.linalg.norm(bkg.flatten(), ord=np.inf), 0.0)
norm_diff = np.linalg.norm(diff) / np.linalg.norm(self.image)
self.assertLess(norm_diff, 1E-2)
def test_image_dims(self):
with self.assertRaises(ValueError):
diff, opt_image, krn, bkg = ois.optimal_system(
self.img, np.random.random((10, 10, 100)))
with self.assertRaises(ValueError):
diff, opt_image, krn, bkg = ois.optimal_system(
np.random.random((10, 10, 100)), self.ref)
with self.assertRaises(ValueError):
diff, opt_image, krn, bkg = ois.optimal_system(
np.zeros((5, 5)), np.zeros((7, 7)))
def subtract(hduls, name="SCI"):
"""
Returns differences of a set of images from a template image
Arguments:
hduls -- list of fits hdul's where the last image is the template
name -- name of the HDU to use
image that the other images will be subtracted from
"""
hduls = [h for h in hduls]
output = []
outputs = []
template = combine(hduls, name)["PRIMARY"].data
for hdu in hduls:
diff = ois.optimal_system(image=hdu[name].data,
refimage=template,
method='Bramich')[0]
output.append(diff)
for array_set in output:
# FIXME this is ragingly wrong, multiple items should be associated
for item in array_set:
hdu = fits.PrimaryHDU(item)
outputs.append(fits.HDUList([hdu]))
return (hdul for hdul in outputs)
def subtract(sources, template):
output = []
if type(sources) == list:
for element in sources:
diff = ois.optimal_system(image=element.data, refimage=template, method='Bramich')
output.append(diff)
return output
def the_grid_test(self, method_name, **kwargs):
gh, gw = 2, 2
diff, opt, krn, bkg = ois.optimal_system(
self.img, self.ref,
method=method_name,
gridshape=(gh, gw),
kernelshape=(11, 11),
**kwargs)
# Assert it's not returning masked arrays
self.assertFalse(isinstance(diff, np.ma.MaskedArray))
self.assertFalse(isinstance(opt, np.ma.MaskedArray))
h, w = self.img.shape
diff_grid = diff[:h // gh, :w // gw]
k_spill = 5
img_crop = self.img[:h // gh + k_spill, :w // gw + k_spill]
ref_crop = self.ref[:h // gh + k_spill, :w // gw + k_spill]
diff_crop, opt, krn, bkg = ois.optimal_system(
img_crop, ref_crop,
method=method_name,
gridshape=None,
kernelshape=(11, 11),
**kwargs)
diff_crop = diff_crop[:h // gh, :w // gw]
norm_diff = np.linalg.norm(diff_grid - diff_crop)
# Assert it does the same on grid or not
self.assertLess(norm_diff, 1E-10)
h, w = self.img.shape
diff_grid = diff[:h // gh, w // gw:]
k_spill = 5
img_crop = self.img[:h // gh + k_spill, w // gw - k_spill:]
ref_crop = self.ref[:h // gh + k_spill, w // gw - k_spill:]
diff_crop, opt, krn, bkg = ois.optimal_system(
img_crop, ref_crop,
method=method_name,
gridshape=None,
kernelshape=(11, 11),
**kwargs)
diff_crop = diff_crop[:h // gh, k_spill:]
norm_diff = np.linalg.norm(diff_grid - diff_crop)
# Assert it does the same on grid or not
self.assertLess(norm_diff, 1E-10)
def test_optimal_system_bramich(self):
# Test Bramich
diff, ruined_image, optKernel, bkg = ois.optimal_system(
self.image, self.ref_img, kernelshape=(11, 11),
bkgdegree=self.bkgdeg, method="Bramich")
norm_diff = np.linalg.norm(diff) / np.linalg.norm(self.image)
self.assertLess(norm_diff, 1E-10)
self.assertFalse(isinstance(diff, np.ma.MaskedArray))
self.assertFalse(isinstance(ruined_image, np.ma.MaskedArray))
self.assertFalse(isinstance(bkg, np.ma.MaskedArray))
def test_both_bramich_consistency(self):
k_side = 3
image = np.random.random((10, 10))
refimage = np.random.random((10, 10))
k_shape = (k_side, k_side)
diff, opt_img, opt_k, bkg = ois.optimal_system(
image, refimage, kernelshape=k_shape, bkgdegree=None,
method="Bramich")
diff, opt_img, opt_vark, bkg = ois.optimal_system(
image, refimage, kernelshape=k_shape, bkgdegree=None,
method="AdaptiveBramich", poly_degree=0)
self.assertEqual(opt_vark.shape, (k_side, k_side, 1))
opt_vark = opt_vark.reshape((k_side, k_side))
diff_norm = np.linalg.norm((opt_k - opt_vark).flatten(), ord=np.inf)
kernel_norm = np.linalg.norm(opt_k.flatten(), ord=np.inf)
self.assertLess(diff_norm / kernel_norm, 1E-8)
def test_Bramich_align(self):
diff, opt, krn, bkg = ois.optimal_system(
self.img, self.ref,
method="Bramich")
norm_diff = np.linalg.norm(diff) / np.linalg.norm(self.ref)
# Assert it's a good subtraction
self.assertLess(norm_diff, 1E-3)
# Assert it's not returning masked arrays
self.assertFalse(isinstance(diff, np.ma.MaskedArray))
self.assertFalse(isinstance(opt, np.ma.MaskedArray))
def test_AlardLupton_diffPSF(self):
diff, opt, krn, bkg = ois.optimal_system(
self.img, self.ref,
method="Alard-Lupton",
gausslist=[{'sx': 1.5, 'sy': 1.5}])
norm_diff = np.linalg.norm(diff) / np.linalg.norm(self.ref)
# Assert it's a good subtraction
self.assertLess(norm_diff, 1E-3)
# Assert it's not returning masked arrays
self.assertFalse(isinstance(diff, np.ma.MaskedArray))
self.assertFalse(isinstance(opt, np.ma.MaskedArray))
def test_AdaptiveBramich_mask(self):
diff, opt, krn, bkg = ois.optimal_system(
self.img_masked, self.ref_masked,
method="AdaptiveBramich")
norm_diff = np.linalg.norm(diff.compressed()) \
/ np.linalg.norm(self.ref_masked.compressed())
# Assert it's a good subtraction
self.assertLess(norm_diff, 1E-3)
# Assert it's returning masked arrays
self.assertTrue(isinstance(diff, np.ma.MaskedArray))
self.assertTrue(isinstance(opt, np.ma.MaskedArray))
def test_AlardLupton_align(self):
diff, opt, krn, bkg = ois.optimal_system(
self.img, self.ref,
method="Alard-Lupton",
kernelshape=(5, 5),
gausslist=[{'sx': 0.01, 'sy':0.01, 'center': (2 - self.x0, 2 - self.y0), 'modPolyDeg':0}])
norm_diff = np.linalg.norm(diff) / np.linalg.norm(self.ref)
# Assert it's a good subtraction
self.assertLess(norm_diff, 1E-3)
# Assert it's not returning masked arrays
self.assertFalse(isinstance(diff, np.ma.MaskedArray))
self.assertFalse(isinstance(opt, np.ma.MaskedArray))
def test_AlardLupton_mask(self):
diff, opt, krn, bkg = ois.optimal_system(
self.img_masked, self.ref_masked,
method="Alard-Lupton",
gausslist=[{'sx': 1.5, 'sy': 1.5}])
norm_diff = np.linalg.norm(diff.compressed()) \
/ np.linalg.norm(self.ref_masked.compressed())
# Assert it's a good subtraction
self.assertLess(norm_diff, 1E-3)
# Assert it's returning masked arrays
self.assertTrue(isinstance(diff, np.ma.MaskedArray))
self.assertTrue(isinstance(opt, np.ma.MaskedArray))
def test_both_bramich_consistency(self):
deg = 0
bkg_deg = 0
k_side = 3
image = np.random.random((10, 10))
refimage = np.random.random((10, 10))
k_shape = (k_side, k_side)
diff, opt_img, opt_k, bkg = ois.optimal_system(
image, refimage, kernelshape=k_shape, bkgdegree=bkg_deg,
method="Bramich")
diff, opt_img, opt_vark, bkg = ois.optimal_system(
image, refimage, kernelshape=k_shape, bkgdegree=bkg_deg,
method="AdaptiveBramich", poly_degree=deg)
self.assertEqual(opt_vark.shape, (k_side, k_side, 1))
opt_vark = opt_vark.reshape((k_side, k_side))
diff_norm = np.linalg.norm((opt_k - opt_vark).flatten(), ord=np.inf)
kernel_norm = np.linalg.norm(opt_k.flatten(), ord=np.inf)
self.assertLess(diff_norm / kernel_norm, 1E-8)
def test_optimal_system_alardlp(self):
# Test Alard & Lupton
diff, ruined_image, optKernel, bkg = ois.optimal_system(
self.image, self.ref_img,
kernelshape=(11, 11),
bkgdegree=self.bkgdeg,
method="Alard-Lupton",
gausslist=self.mygausslist)
norm_diff = np.linalg.norm(diff) / np.linalg.norm(self.image)
self.assertLess(norm_diff, 1E-10)
self.assertFalse(isinstance(diff, np.ma.MaskedArray))
self.assertFalse(isinstance(ruined_image, np.ma.MaskedArray))
self.assertFalse(isinstance(bkg, np.ma.MaskedArray))
def test_background_AdaptiveBramich(self):
diff, opt, krn, bkg = ois.optimal_system(
self.img, self.ref,
method="AdaptiveBramich",
bkgdegree=2)
# Assert it's a good subtraction
norm_diff = np.linalg.norm(diff) / np.linalg.norm(self.ref)
self.assertLess(norm_diff, 1E-3)
# Assert we had a good background estimation
norm_bkg_diff = np.linalg.norm(bkg - self.bkg) / np.linalg.norm(bkg)
self.assertLess(norm_bkg_diff, 1E-3)
# Assert it's not returning masked arrays
self.assertFalse(isinstance(diff, np.ma.MaskedArray))
self.assertFalse(isinstance(opt, np.ma.MaskedArray))
def test_background_AlardLupton(self):
diff, opt, krn, bkg = ois.optimal_system(
self.img, self.ref,
method="Alard-Lupton",
gausslist=[{'sx': 1.5, 'sy': 1.5}],
bkgdegree=2)
# Assert it's a good subtraction
norm_diff = np.linalg.norm(diff) / np.linalg.norm(self.ref)
self.assertLess(norm_diff, 1E-3)
# Assert we had a good background estimation
norm_bkg_diff = np.linalg.norm(bkg - self.bkg) / np.linalg.norm(bkg)
self.assertLess(norm_bkg_diff, 1E-3)
# Assert it's not returning masked arrays
self.assertFalse(isinstance(diff, np.ma.MaskedArray))
self.assertFalse(isinstance(opt, np.ma.MaskedArray))
def subtract(hduls):
"""
Returns differences of a set of images from a template image
Arguments:
hduls --list of fits hdul's where the last image is the template
image that the other images will be subtracted from
"""
hduls_list = [hdul for hdul in hduls]
output = []
outputs = []
# FIXME this is not a combined image
template = hduls_list[-1][0]
try:
template = hduls_list[-1][0].data
except AttributeError:
pass
return (fits.HDUList(fits.PrimaryHDU(ois.optimal_system( \
image=hdu["SCI"].data, refimage=template, method="Bramich")[0])) \
for hdu in hduls_list)
def subtract(hduls):
"""
Returns differences of a set of images from a template image
Arguments:
hduls --list of fits hdul's where the last image is the template
image that the other images will be subtracted from
"""
hduls_list = [hdul for hdul in hduls]
output = []
outputs = []
template = to_np(hduls_list[-1][0])
for hdu in hduls[:-1]:
diff = ois.optimal_system(image=hdu["SCI"].data,
refimage=template,
method='Bramich')
output.append(diff)
for array_set in output:
for item in array_set:
hdu = fits.PrimaryHDU(item)
outputs.append(fits.HDUList([hdu]))
return (hdul for hdul in outputs)
def main(imgs_dir, ref_path, new_path, details):
if not os.path.isdir(imgs_dir):
os.makedirs(imgs_dir)
ref_dest = os.path.join(imgs_dir, 'ref.fits')
new_dest = os.path.join(imgs_dir, 'new.fits')
os.rename(ref_path, ref_dest)
os.rename(new_path, new_dest)
ref = si.SingleImage(ref_dest, borders=False) #, crop=((150,150), (150, 150)))
new = si.SingleImage(new_dest, borders=False) #, crop=((150,150), (150, 150)))
#~ ## Adding stars
foo = new.cov_matrix
srcs = new.best_sources
rows = []
for i in range(15):
j = np.random.choice(len(srcs), 1, replace=False)
flux = srcs['flux'][j][0]
xs = srcs['x'][j][0]
ys = srcs['y'][j][0]
position = (ys, xs)
sx, sy = new.stamp_shape
sx += 3
sy += 3
star = extract_array(new.pixeldata.data,
(sx, sy), position,
mode='partial',
fill_value=new._bkg.globalrms)
#print flux
#~ star = flux*new.db.load(j)[0]
x = np.random.choice(new.pixeldata.shape[0]-3*sx, 1)[0] + sx#* np.random.random())
y = np.random.choice(new.pixeldata.shape[1]-3*sy, 1)[0] + sy
# np.int((new.pixeldata.shape[1]-star.shape[1]) * np.random.random())
#~ if new.pixeldata.data[x:x+sx, y:y+sy].shape != star.shape:
#~ import ipdb; ipdb.set_trace()
new.pixeldata.data[x:x+sx, y:y+sy] = star
#~ except:
#~ continue
xc = x+sx/2.
yc = y+sy/2.
app_mag = -2.5*np.log10(flux)+25.
rows.append([yc, xc, app_mag, flux])
newcat = Table(rows=rows, names=['x', 'y', 'app_mag', 'flux'])
fits.writeto(filename=new_dest, header=fits.getheader(new_dest),
data=new.pixeldata.data, overwrite=True)
new._clean()
new = si.SingleImage(new_dest, borders=False) #, crop=((150,150), (150, 150)))
newcat.write(os.path.join(imgs_dir, 'transient.list'),
format='ascii.fast_no_header',
overwrite=True)
fits.writeto(filename=os.path.join(imgs_dir, 'interp_ref.fits'), data=ref.interped, overwrite=True)
fits.writeto(filename=os.path.join(imgs_dir, 'interp_new.fits'), data=new.interped, overwrite=True)
try:
print 'Images to be subtracted: {} {}'.format(ref_dest, new_dest)
import time
t0 = time.time()
D, P, S, mask = ps.diff(ref, new, align=False,
iterative=False, shift=False, beta=True)
dt_z = time.time() - t0
new._clean()
ref._clean()
mea, med, std = sigma_clipped_stats(D.real)
D = np.ma.MaskedArray(D.real, mask).filled(mea)
fits.writeto(os.path.join(imgs_dir,'diff.fits'), D, overwrite=True)
#~ utils.encapsule_R(D, path=os.path.join(imgs_dir, 'diff.fits'))
utils.encapsule_R(P, path=os.path.join(imgs_dir, 'psf_d.fits'))
utils.encapsule_R(S, path=os.path.join(imgs_dir, 's_diff.fits'))
scorrdetected = utils.find_S_local_maxima(S, threshold=3.5)
print 'S_corr found thath {} transients were above 3.5 sigmas'.format(len(scorrdetected))
ascii.write(table=np.asarray(scorrdetected),
output=os.path.join(imgs_dir, 's_corr_detected.csv'),
names=['X_IMAGE', 'Y_IMAGE', 'SIGNIFICANCE'],
format='csv')
S = np.ascontiguousarray(S)
#~ s_bkg = sep.Background(S)
mean, median, std = sigma_clipped_stats(S)
sdetected = sep.extract(S-median, 3.5*std,
filter_kernel=None)
print 'S_corr with sep found thath {} transients were above 3.5 sigmas'.format(len(sdetected))
ascii.write(table=sdetected,
output=os.path.join(imgs_dir, 'sdetected.csv'),
format='csv')
## With OIS
t0 = time.time()
ois_d = ois.optimal_system(fits.getdata(new_dest), fits.getdata(ref_dest))[0]
dt_o = time.time() - t0
utils.encapsule_R(ois_d, path=os.path.join(imgs_dir, 'diff_ois.fits'))
## With HOTPANTS
t0 = time.time()
os.system('hotpants -v 0 -inim {} -tmplim {} -outim {} -r 15 -tu 40000 -tl -100 -il -100 -iu 40000'.format(new_dest, ref_dest,
os.path.join(imgs_dir, 'diff_hot.fits')))
dt_h = time.time() - t0
return [newcat.to_pandas(), [dt_z, dt_o, dt_h]]
except:
raise
def test_wrong_method_name(self):
with self.assertRaises(ValueError):
diff, opt_image, krn, bkg = ois.optimal_system(
self.img, self.ref, method="WrongName")
def test_even_side_kernel(self):
for bad_shape in ((8, 9), (9, 8), (8, 8)):
with self.assertRaises(ois.EvenSideKernelError):
ois.optimal_system(self.img, self.ref, bad_shape)
start = time.time()
#
# this scirpt gets all the files in a directoty and appends them to a list to be used stating from all_files[0] as 1st file
#
image_list_prelim = []
for frame in glob.glob("*.fit"):
image_list_prelim.append(frame)
image_list = [fits.getdata(image) for image in image_list_prelim]
compressed_image = np.sum(image_list, axis=0)
hdu_diff = fits.PrimaryHDU(diff_data, header=fits.getdata(image_list[0]))
# Read FITS data
ref_data = fits.getdata(image_list[0])
targ_data = fits.getdata(compressed_image)
#
# Read FITS headers
ref_hdr = ref_fits[0].header
targ_hdr = targ_fits[0].header
# aligned_array = []
#
#subtaction of compressed image to reference image
#
diff_data, optimal_image, kernel, background = ois.optimal_system(
image_list, ref_data, kernelshape=(11, 11), method="Bramich")
hdu_diff = fits.PrimaryHDU(diff_data, header=ref_hdr)
hdu_diff.writeto("final_sub.fit", overwrite=True)
#
time = end - start
print("Ended in", "%.1f" % time, "seconds!")
#Difference Imaging
ref_fits = fits.open(
"/home/linkmaster/Documents/CTMO/Projects/Pipeline/AGN_Project/Reference/dss_search"
)
targ_fits = fits.open("final_stacked_image.fit") # stack image FITS image
# Read FITS data
ref_data = fits.getdata(
"/home/linkmaster/Documents/CTMO/Projects/Pipeline/AGN_Project/Reference/dss_search"
)
targ_data = fits.getdata("final_stacked_image.fit")
# Read FITS headers
ref_hdr = ref_fits[0].header
targ_hdr = targ_fits[0].header
# Alignment
aligned_array, footprint = reproject_interp(
targ_fits, ref_hdr) #aligned array is the new fits file aligned
print(type(aligned_array))
aligned_hdu = fits.PrimaryHDU(aligned_array, header=ref_hdr)
aligned_hdu.writeto("aligned_to.fit", overwrite=True)
# Subtraction
diff_data, optimal_image, kernel, background = ois.optimal_system(
aligned_array, ref_data, kernelshape=(11, 11), method="Bramich")
hdu_diff = fits.PrimaryHDU(diff_data, header=ref_hdr)
hdu_diff.writeto("sub.fit", overwrite=True)
