def main(args):
imgsdir = '/home/bruno/Documentos/Data/SNiPTF/imgs'
imgsdir = '/home/bruno/Data/SNiPTF/imgs'
#imgsdir = '/home/bruno/Documentos/Data/LIGO_O2/20171116/ESO202-009'
dest_dir = './test/test_images/test_sub_sniptf'
imgs = glob.glob(imgsdir + '/*sci*.fits')
mask = glob.glob(imgsdir + '/*mask*.fits')
imgs.sort()
mask.sort()
images = [
s.SingleImage(animg, mask=amask) for animg, amask in zip(imgs, mask)
]
#images = [s.SingleImage(animg) for animg in imgs]
for i, animg in enumerate(images[1:]):
#~ t, _ = aa.find_transform(animg.data, images[0].data)
#~ if abs(t.rotation)>5e-4:
#~ k = t.__class__
#~ t = k(np.round(t.params, decimals=5))
#~ reg = aa.apply_transform(t, animg.data, images[0].data)
reg = aa.register(animg.data, images[0].data)
new = s.SingleImage(reg.data, mask=reg.mask)
fits.writeto('/home/bruno/aligned_{}.fits'.format(i),
reg.data,
overwrite=True)
D, P, S_corr, mask = ps.diff(images[0],
new,
align=True,
iterative=False,
shift=True,
beta=True)
D = np.ma.MaskedArray(D.real, mask).filled(np.median(D.real))
fits.writeto(os.path.join(dest_dir, 'Diff_{}.fits'.format(i)),
D.real,
overwrite=True)
fits.writeto(os.path.join(dest_dir, 'P_{}.fits'.format(i)),
P.real,
overwrite=True)
fits.writeto(os.path.join(dest_dir, 'Scorr_{}.fits'.format(i)),
S_corr,
overwrite=True)
new._clean()
for an_img in images:
an_img._clean()
return
def testChunkIt(self):
imgs = [si.SingleImage(img) for img in self.paths]
for i in range(len(imgs)):
chunks = utils.chunk_it(imgs, i + 1)
self.assertIsInstance(chunks, list)
flat_list = [item for sublist in chunks for item in sublist]
self.assertCountEqual(flat_list, imgs)
def random_simage():
pixel = random.random((128, 128)) * 5.0
# Add some stars to it
star = [[35, 38, 35], [38, 90, 39], [35, 39, 34]]
for i in range(25):
x, y = random.randint(120, size=2)
pixel[x : x + 3, y : y + 3] = star
mask = random.randint(2, size=(128, 128))
for i in range(10):
mask = mask & random.randint(2, size=(128, 128))
img = s.SingleImage(pixel, mask)
return img
def random_4psf_simage():
frames = []
for theta in [0, 45, 105, 150]:
image_seed = int(random.rand() * 1000)
N = 512 # side
X_FWHM = 5 + 5.5 * theta / 180
Y_FWHM = 5
t_exp = 5
max_fw = max(X_FWHM, Y_FWHM)
x = random.randint(low=6 * max_fw, high=N - 6 * max_fw, size=80)
y = random.randint(low=6 * max_fw, high=N - 6 * max_fw, size=80)
xy = [(x[i], y[i]) for i in range(80)]
SN = 30.0 # SN para poder medir psf
weights = list(np.linspace(10, 1000.0, len(xy)))
m = simtools.delta_point(N, center=False, xy=xy, weights=weights)
im = simtools.image(
m,
N,
t_exp,
X_FWHM,
Y_FWHM=Y_FWHM,
theta=theta,
SN=SN,
bkg_pdf="gaussian",
seed=image_seed,
)
frames.append(im + 100.0)
frame = np.zeros((1024, 1024))
for j in range(2):
for i in range(2):
frame[i * 512 : (i + 1) * 512, j * 512 : (j + 1) * 512] = frames[
i + 2 * j
]
return s.SingleImage(frame)
def setUp(self):
super(TestHDUList, self).setUp()
self.si = s.SingleImage(self.mock_masked_hdu)
def setUp(self):
super(TestHDU, self).setUp()
self.si = s.SingleImage(self.mockimageHdu)
def setUp(self):
super(TestFitsMask, self).setUp()
self.si = s.SingleImage(self.mockfits_path, mask=self.mockmask_path)
def testNoSources(self):
with self.assertRaises(ValueError):
s.SingleImage(self.no_sources_image)
def setUp(self):
super(TestHDUListPicky, self).setUp()
self.si = s.SingleImage(self.mock_masked_hdu, strict_star_pick=True)
def setUp(self):
super(TestFitsMaskPicky, self).setUp()
self.si = s.SingleImage(self.mockfits_path,
mask=self.mockmask_path,
strict_star_pick=True)
def main():
imgsdir = '/home/bruno/Documentos/Data/SNiPTF/imgs'
imgsdir = '/home/bruno/Data/SNiPTF/imgs'
imgsdir = '/home/bruno/Data/LIGO_O2/Jan04/20171119/PGC073926/calibrated'
dest_dir = './test/test_images/test_sniptf'
imgs = glob.glob(imgsdir + '/aligned_PGC073926-*.fits')
mask = glob.glob(imgsdir + '/*mask*.fits')
imgs.sort()
mask.sort()
#~ print imgs
#~ print mask
#~ for animg in imgs:
#~ img = fits.open(animg, 'update')
#~ img[0].data = img[0].data[:495, :495]
#~ img[0].header['NAXIS1'] = 495
#~ img[0].header['NAXIS2'] = 495
#~ img.flush()
#~ img.close()
#images = [s.SingleImage(animg, mask=amask) for animg, amask in zip(imgs, mask)]
images = [s.SingleImage(animg) for animg in imgs]
for j, an_img in enumerate(images):
an_img.inf_loss = 0.18
plot.plot_psfbasis(an_img.kl_basis,
path=os.path.join(dest_dir,
'psf_basis_{}.png'.format(j)),
nbook=False)
x, y = an_img.get_afield_domain()
plot.plot_afields(an_img.kl_afields,
x,
y,
path=os.path.join(dest_dir,
'afields_{}.png'.format(j)),
nbook=False,
size=4)
fits.writeto(os.path.join(dest_dir, 'mask_{}.fits'.format(j)),
an_img.mask.astype('uint8'),
overwrite=True)
fits.writeto(os.path.join(dest_dir, 'S_comp_{}.fits'.format(j)),
an_img.s_component,
overwrite=True)
fits.writeto(os.path.join(dest_dir, 'interped_{}.fits'.format(j)),
an_img.interped,
overwrite=True)
plot.plt.imshow(an_img.psf_hat_sqnorm().real)
plot.plt.colorbar()
plot.plt.savefig(
os.path.join(dest_dir, 'psf_hat_sqnorm_{}.png'.format(j)))
plot.plt.close()
R, P_r = pc.stack_R(images, align=False, n_procs=4, inf_loss=0.25)
fits.writeto(os.path.join(dest_dir, 'R.fits'), R.real, overwrite=True)
fits.writeto(os.path.join(dest_dir, 'P.fits'), P_r.real, overwrite=True)
for an_img in images:
an_img._clean()
return
def setUp(self):
super(TestSubtract, self).setUp()
self.si_ref = si.SingleImage(self.paths[0])
self.si_new = si.SingleImage(self.paths[1])
def main(args):
imgsdir = '/home/bruno/Data/LIGO_O2/Jan04/newstacks/PGC073926'
dest_dir = './test/test_images/test_propersubtract'
imgs = glob.glob(imgsdir + '/pgc073926_17*.fits')
#mask = glob.glob(imgsdir+'/*mask*.fits')
imgs.sort()
#mask.sort()
#images = [s.SingleImage(animg, mask=amask) for animg, amask in zip(imgs, mask)]
images = []
for animg in imgs:
print(animg)
try:
o = s.SingleImage(animg,
borders=True,
crop=((150, 150), (150, 150)))
images.append(o)
except:
pass
#~ hdu = fits.open(animg)
#~ data = hdu[0].data
#~ images = [s.SingleImage(animg, borders=True, crop=((250,250), (250, 250)))
#~ for animg in imgs]
fits.writeto(os.path.join(dest_dir, 'InterpedRef.fits'),
images[0].interped,
overwrite=True)
for i, animg in enumerate(images[1:]):
#~ ## Erasing stars
srcs = animg.best_sources
f60 = np.percentile(srcs['cflux'], q=60)
jj = np.random.choice(len(srcs), len(srcs) / 3, replace=False)
mea, med, std = sigma_clipped_stats(animg.data)
sx, sy = animg.stamp_shape
st = animg._bkg.globalrms
for aj in jj:
star = srcs[aj]
x = star['x']
y = star['y']
if x < sx or animg.data.shape[0] - x < sx:
continue
if y < sy or animg.data.shape[1] - y < sy:
continue
if star['cflux'] < f60:
continue
print(x, y)
noise = np.random.normal(loc=med, scale=st, size=animg.stamp_shape)
animg.data.data[np.int(x - sx / 2.):np.int(x + sx / 2.),
np.int(y - sy / 2.):np.int(y + sy / 2.)] = noise
## Adding stars
#~ foo = animg.cov_matrix
#~ srcs = animg.best_sources
#~ jj = np.random.choice(len(srcs), 12, replace=False)
#~ for aj in jj:
#~ star = animg.db.load(aj)[0]
#~ x, y = np.random.choice(np.min(animg.data.shape)-np.max(star.shape),
#~ 2, replace=True)
#~ print star.shape
#~ animg.data.data[x:x+star.shape[0], y:y+star.shape[1]] = star
#~ xc, yc = x+star.shape[0]/2., y+star.shape[1]/2.
#~ print xc, yc
fits.writeto(os.path.join(dest_dir, 'InterpedNew_{}.fits'.format(i)),
animg.interped,
overwrite=True)
try:
D, P, S_corr, mask = ps.diff(animg,
images[0],
align=True,
iterative=False,
shift=False,
beta=True)
mea, med, std = sigma_clipped_stats(D.real, mask)
D = np.ma.MaskedArray(D.real, mask).filled(mea)
fits.writeto(os.path.join(dest_dir, 'Diff_{}.fits'.format(i)),
D,
overwrite=True)
fits.writeto(os.path.join(dest_dir, 'P_{}.fits'.format(i)),
P.real,
overwrite=True)
fits.writeto(os.path.join(dest_dir, 'Scorr_{}.fits'.format(i)),
S_corr,
overwrite=True)
except:
print('subtraction failed')
print('ref: ', images[0])
print('new: ', animg)
for an_img in images:
an_img._clean()
return
def main(ref_path, new_path, objname):
# alineo las imagenes
refdata = fits.getdata(os.path.join(STACK_PATH, ref_path))[250:-250,
250:-250]
newdata = fits.getdata(os.path.join(STACK_PATH, new_path))[250:-250,
250:-250]
try:
trf, _ = aa.find_transform(newdata.astype('<f8'),
refdata.astype('<f8'))
new_aligned = aa.apply_transform(t, newdata.astype('<f8'),
refdata.astype('<f8'))
from skimage.transform import warp
init_mask = np.zeros_like(newdata)
outside_px_mask = warp(init_mask,
inverse_map=trf.inverse,
output_shape=refdata.shape,
order=3,
mode='constant',
cval=1.,
clip=False,
preserve_range=False)
useful = np.where(outside_px_mask == 0)
max_x = np.max(useful[0])
min_x = np.min(useful[0])
max_y = np.max(useful[1])
min_y = np.min(useful[1])
new_cropped = new_aligned[min_x:max_x, min_y:max_y]
new_mask = outside_px_mask[min_x:max_x, min_y:max_y]
ref_cropped = refdata[min_x:max_x, min_y:max_y]
except:
try:
aa.MIN_MATCHES_FRACTION = 0.01
ref = si.SingleImage(refdata.astype('<f8'))
new = si.SingleImage(newdata.astype('<f8'))
ref.best_sources.sort(order='flux')
new.best_sources.sort(order='flux')
#~ import ipdb; ipdb.set_trace()
rs = np.empty((len(ref.best_sources), 2))
j = 0
for x, y in ref.best_sources[['x', 'y']]:
rs[j] = x, y
j += 1
ns = np.empty((len(new.best_sources), 2))
j = 0
for x, y in new.best_sources[['x', 'y']]:
ns[j] = x, y
j += 1
if abs(len(ns) - len(rs)) > 50:
max_l = np.max([len(ns), len(rs)])
ns = ns[:max_l]
rs = rs[:max_l]
trf, _ = aa.find_transform(ns, rs)
new_aligned = aa.apply_transform(trf, newdata.astype('<f8'),
refdata.astype('<f8'))
from skimage.transform import warp
init_mask = np.zeros_like(newdata)
outside_px_mask = warp(init_mask,
inverse_map=trf.inverse,
output_shape=refdata.shape,
order=3,
mode='constant',
cval=1.,
clip=False,
preserve_range=False)
useful = np.where(outside_px_mask == 0)
max_x = np.max(useful[0])
min_x = np.min(useful[0])
max_y = np.max(useful[1])
min_y = np.min(useful[1])
new_cropped = new_aligned[min_x:max_x, min_y:max_y]
new_mask = outside_px_mask[min_x:max_x, min_y:max_y]
ref_cropped = refdata[min_x:max_x, min_y:max_y]
except:
#~ import ipdb; ipdb.set_trace()
raise
# las copio al lugar designado en la pipeline
ref_h = fits.getheader(os.path.join(STACK_PATH, ref_path))
fits.writeto(data=ref_cropped.astype('<f4'),
header=ref_h,
filename=REFERENCE_IMAGE,
overwrite=True)
new_h = fits.getheader(os.path.join(STACK_PATH, new_path))
fits.writeto(data=new_cropped.astype('<f4'),
header=new_h,
filename=NEW_IMAGE,
overwrite=True)
# creo un file con los detalles
meta = {}
meta['object'] = objname
meta['orig_ref_path'] = ref_path
meta['orig_new_path'] = new_path
with open(DETAILS_FILE, 'w') as fp:
json.dump(meta, fp)
return 0
def setUp(self):
super(TestNpArrayPicky, self).setUp()
print(self.mock_image_data.shape)
self.si = s.SingleImage(self.mock_image_data, strict_star_pick=True)
# =============================================================================
# PSF measure test by propercoadd
# =============================================================================
datapath = os.path.abspath('/home/bruno/Data/Tolar2017/20170624/NGC_6544')
frame = os.path.join(datapath, 'NGC_6544-010.fit')
d = fits.getdata(frame)
plot.plot_S(d, path='./test/test_images/real_image_test/frame.png')
# =============================================================================
# PSF spatially variant
# =============================================================================
with si.SingleImage(frame) as sim:
a_f, psf_b = sim.get_variable_psf(inf_loss=0.1)
x, y = sim.get_afield_domain()
normal_image = sim.normal_image
interp = sim.interped
print((sim.n_sources))
S = sim.s_component
plot.plot_psfbasis(psf_b,
path='./test/test_images/real_image_test/psf_basis.png')
plot.plot_afields(a_f,
x,
y,
path='./test/test_images/real_image_test/a_fields.png')
def setUp(self):
super(TestNpArrayMaskPicky, self).setUp()
self.si = s.SingleImage(self.mock_image_data,
self.mock_image_mask,
strict_star_pick=True)
def setUp(self):
super(TestNpArray, self).setUp()
print(self.mock_image_data.shape)
self.si = s.SingleImage(self.mock_image_data)
def setUp(self):
super(TestHDUPicky, self).setUp()
self.si = s.SingleImage(self.mockimageHdu, strict_star_pick=True)
def setUp(self):
super(TestNpArrayMask, self).setUp()
self.si = s.SingleImage(self.mock_image_data, self.mock_image_mask)
def setUp(self):
super(TestFitsExtensionPicky, self).setUp()
self.si = s.SingleImage(self.masked_hdu_path)
def setUp(self):
super(TestFitsFile, self).setUp()
self.si = s.SingleImage(self.mockfits_path)
def main(imgs_dir, ref_path, new_path, details):
if not os.path.isdir(imgs_dir):
os.makedirs(imgs_dir)
ref_dest = os.path.join(imgs_dir, 'ref.fits')
new_dest = os.path.join(imgs_dir, 'new.fits')
os.rename(ref_path, ref_dest)
os.rename(new_path, new_dest)
ref = si.SingleImage(ref_dest, borders=False) #, crop=((150,150), (150, 150)))
new = si.SingleImage(new_dest, borders=False) #, crop=((150,150), (150, 150)))
#~ ## Adding stars
foo = new.cov_matrix
srcs = new.best_sources
rows = []
for i in range(15):
j = np.random.choice(len(srcs), 1, replace=False)
flux = srcs['flux'][j][0]
xs = srcs['x'][j][0]
ys = srcs['y'][j][0]
position = (ys, xs)
sx, sy = new.stamp_shape
sx += 3
sy += 3
star = extract_array(new.pixeldata.data,
(sx, sy), position,
mode='partial',
fill_value=new._bkg.globalrms)
#print flux
#~ star = flux*new.db.load(j)[0]
x = np.random.choice(new.pixeldata.shape[0]-3*sx, 1)[0] + sx#* np.random.random())
y = np.random.choice(new.pixeldata.shape[1]-3*sy, 1)[0] + sy
# np.int((new.pixeldata.shape[1]-star.shape[1]) * np.random.random())
#~ if new.pixeldata.data[x:x+sx, y:y+sy].shape != star.shape:
#~ import ipdb; ipdb.set_trace()
new.pixeldata.data[x:x+sx, y:y+sy] = star
#~ except:
#~ continue
xc = x+sx/2.
yc = y+sy/2.
app_mag = -2.5*np.log10(flux)+25.
rows.append([yc, xc, app_mag, flux])
newcat = Table(rows=rows, names=['x', 'y', 'app_mag', 'flux'])
fits.writeto(filename=new_dest, header=fits.getheader(new_dest),
data=new.pixeldata.data, overwrite=True)
new._clean()
new = si.SingleImage(new_dest, borders=False) #, crop=((150,150), (150, 150)))
newcat.write(os.path.join(imgs_dir, 'transient.list'),
format='ascii.fast_no_header',
overwrite=True)
fits.writeto(filename=os.path.join(imgs_dir, 'interp_ref.fits'), data=ref.interped, overwrite=True)
fits.writeto(filename=os.path.join(imgs_dir, 'interp_new.fits'), data=new.interped, overwrite=True)
try:
print 'Images to be subtracted: {} {}'.format(ref_dest, new_dest)
import time
t0 = time.time()
D, P, S, mask = ps.diff(ref, new, align=False,
iterative=False, shift=False, beta=True)
dt_z = time.time() - t0
new._clean()
ref._clean()
mea, med, std = sigma_clipped_stats(D.real)
D = np.ma.MaskedArray(D.real, mask).filled(mea)
fits.writeto(os.path.join(imgs_dir,'diff.fits'), D, overwrite=True)
#~ utils.encapsule_R(D, path=os.path.join(imgs_dir, 'diff.fits'))
utils.encapsule_R(P, path=os.path.join(imgs_dir, 'psf_d.fits'))
utils.encapsule_R(S, path=os.path.join(imgs_dir, 's_diff.fits'))
scorrdetected = utils.find_S_local_maxima(S, threshold=3.5)
print 'S_corr found thath {} transients were above 3.5 sigmas'.format(len(scorrdetected))
ascii.write(table=np.asarray(scorrdetected),
output=os.path.join(imgs_dir, 's_corr_detected.csv'),
names=['X_IMAGE', 'Y_IMAGE', 'SIGNIFICANCE'],
format='csv')
S = np.ascontiguousarray(S)
#~ s_bkg = sep.Background(S)
mean, median, std = sigma_clipped_stats(S)
sdetected = sep.extract(S-median, 3.5*std,
filter_kernel=None)
print 'S_corr with sep found thath {} transients were above 3.5 sigmas'.format(len(sdetected))
ascii.write(table=sdetected,
output=os.path.join(imgs_dir, 'sdetected.csv'),
format='csv')
## With OIS
t0 = time.time()
ois_d = ois.optimal_system(fits.getdata(new_dest), fits.getdata(ref_dest))[0]
dt_o = time.time() - t0
utils.encapsule_R(ois_d, path=os.path.join(imgs_dir, 'diff_ois.fits'))
## With HOTPANTS
t0 = time.time()
os.system('hotpants -v 0 -inim {} -tmplim {} -outim {} -r 15 -tu 40000 -tl -100 -il -100 -iu 40000'.format(new_dest, ref_dest,
os.path.join(imgs_dir, 'diff_hot.fits')))
dt_h = time.time() - t0
return [newcat.to_pandas(), [dt_z, dt_o, dt_h]]
except:
raise
def setUp(self):
super(TestCoaddMultCore2, self).setUp()
self.imgs = [si.SingleImage(img) for img in self.paths]
