def main(args): imgsdir = '/home/bruno/Documentos/Data/SNiPTF/imgs' imgsdir = '/home/bruno/Data/SNiPTF/imgs' #imgsdir = '/home/bruno/Documentos/Data/LIGO_O2/20171116/ESO202-009' dest_dir = './test/test_images/test_sub_sniptf' imgs = glob.glob(imgsdir + '/*sci*.fits') mask = glob.glob(imgsdir + '/*mask*.fits') imgs.sort() mask.sort() images = [ s.SingleImage(animg, mask=amask) for animg, amask in zip(imgs, mask) ] #images = [s.SingleImage(animg) for animg in imgs] for i, animg in enumerate(images[1:]): #~ t, _ = aa.find_transform(animg.data, images[0].data) #~ if abs(t.rotation)>5e-4: #~ k = t.__class__ #~ t = k(np.round(t.params, decimals=5)) #~ reg = aa.apply_transform(t, animg.data, images[0].data) reg = aa.register(animg.data, images[0].data) new = s.SingleImage(reg.data, mask=reg.mask) fits.writeto('/home/bruno/aligned_{}.fits'.format(i), reg.data, overwrite=True) D, P, S_corr, mask = ps.diff(images[0], new, align=True, iterative=False, shift=True, beta=True) D = np.ma.MaskedArray(D.real, mask).filled(np.median(D.real)) fits.writeto(os.path.join(dest_dir, 'Diff_{}.fits'.format(i)), D.real, overwrite=True) fits.writeto(os.path.join(dest_dir, 'P_{}.fits'.format(i)), P.real, overwrite=True) fits.writeto(os.path.join(dest_dir, 'Scorr_{}.fits'.format(i)), S_corr, overwrite=True) new._clean() for an_img in images: an_img._clean() return
def testChunkIt(self): imgs = [si.SingleImage(img) for img in self.paths] for i in range(len(imgs)): chunks = utils.chunk_it(imgs, i + 1) self.assertIsInstance(chunks, list) flat_list = [item for sublist in chunks for item in sublist] self.assertCountEqual(flat_list, imgs)
def random_simage(): pixel = random.random((128, 128)) * 5.0 # Add some stars to it star = [[35, 38, 35], [38, 90, 39], [35, 39, 34]] for i in range(25): x, y = random.randint(120, size=2) pixel[x : x + 3, y : y + 3] = star mask = random.randint(2, size=(128, 128)) for i in range(10): mask = mask & random.randint(2, size=(128, 128)) img = s.SingleImage(pixel, mask) return img
def random_4psf_simage(): frames = [] for theta in [0, 45, 105, 150]: image_seed = int(random.rand() * 1000) N = 512 # side X_FWHM = 5 + 5.5 * theta / 180 Y_FWHM = 5 t_exp = 5 max_fw = max(X_FWHM, Y_FWHM) x = random.randint(low=6 * max_fw, high=N - 6 * max_fw, size=80) y = random.randint(low=6 * max_fw, high=N - 6 * max_fw, size=80) xy = [(x[i], y[i]) for i in range(80)] SN = 30.0 # SN para poder medir psf weights = list(np.linspace(10, 1000.0, len(xy))) m = simtools.delta_point(N, center=False, xy=xy, weights=weights) im = simtools.image( m, N, t_exp, X_FWHM, Y_FWHM=Y_FWHM, theta=theta, SN=SN, bkg_pdf="gaussian", seed=image_seed, ) frames.append(im + 100.0) frame = np.zeros((1024, 1024)) for j in range(2): for i in range(2): frame[i * 512 : (i + 1) * 512, j * 512 : (j + 1) * 512] = frames[ i + 2 * j ] return s.SingleImage(frame)
def setUp(self): super(TestHDUList, self).setUp() self.si = s.SingleImage(self.mock_masked_hdu)
def setUp(self): super(TestHDU, self).setUp() self.si = s.SingleImage(self.mockimageHdu)
def setUp(self): super(TestFitsMask, self).setUp() self.si = s.SingleImage(self.mockfits_path, mask=self.mockmask_path)
def testNoSources(self): with self.assertRaises(ValueError): s.SingleImage(self.no_sources_image)
def setUp(self): super(TestHDUListPicky, self).setUp() self.si = s.SingleImage(self.mock_masked_hdu, strict_star_pick=True)
def setUp(self): super(TestFitsMaskPicky, self).setUp() self.si = s.SingleImage(self.mockfits_path, mask=self.mockmask_path, strict_star_pick=True)
def main(): imgsdir = '/home/bruno/Documentos/Data/SNiPTF/imgs' imgsdir = '/home/bruno/Data/SNiPTF/imgs' imgsdir = '/home/bruno/Data/LIGO_O2/Jan04/20171119/PGC073926/calibrated' dest_dir = './test/test_images/test_sniptf' imgs = glob.glob(imgsdir + '/aligned_PGC073926-*.fits') mask = glob.glob(imgsdir + '/*mask*.fits') imgs.sort() mask.sort() #~ print imgs #~ print mask #~ for animg in imgs: #~ img = fits.open(animg, 'update') #~ img[0].data = img[0].data[:495, :495] #~ img[0].header['NAXIS1'] = 495 #~ img[0].header['NAXIS2'] = 495 #~ img.flush() #~ img.close() #images = [s.SingleImage(animg, mask=amask) for animg, amask in zip(imgs, mask)] images = [s.SingleImage(animg) for animg in imgs] for j, an_img in enumerate(images): an_img.inf_loss = 0.18 plot.plot_psfbasis(an_img.kl_basis, path=os.path.join(dest_dir, 'psf_basis_{}.png'.format(j)), nbook=False) x, y = an_img.get_afield_domain() plot.plot_afields(an_img.kl_afields, x, y, path=os.path.join(dest_dir, 'afields_{}.png'.format(j)), nbook=False, size=4) fits.writeto(os.path.join(dest_dir, 'mask_{}.fits'.format(j)), an_img.mask.astype('uint8'), overwrite=True) fits.writeto(os.path.join(dest_dir, 'S_comp_{}.fits'.format(j)), an_img.s_component, overwrite=True) fits.writeto(os.path.join(dest_dir, 'interped_{}.fits'.format(j)), an_img.interped, overwrite=True) plot.plt.imshow(an_img.psf_hat_sqnorm().real) plot.plt.colorbar() plot.plt.savefig( os.path.join(dest_dir, 'psf_hat_sqnorm_{}.png'.format(j))) plot.plt.close() R, P_r = pc.stack_R(images, align=False, n_procs=4, inf_loss=0.25) fits.writeto(os.path.join(dest_dir, 'R.fits'), R.real, overwrite=True) fits.writeto(os.path.join(dest_dir, 'P.fits'), P_r.real, overwrite=True) for an_img in images: an_img._clean() return
def setUp(self): super(TestSubtract, self).setUp() self.si_ref = si.SingleImage(self.paths[0]) self.si_new = si.SingleImage(self.paths[1])
def main(args): imgsdir = '/home/bruno/Data/LIGO_O2/Jan04/newstacks/PGC073926' dest_dir = './test/test_images/test_propersubtract' imgs = glob.glob(imgsdir + '/pgc073926_17*.fits') #mask = glob.glob(imgsdir+'/*mask*.fits') imgs.sort() #mask.sort() #images = [s.SingleImage(animg, mask=amask) for animg, amask in zip(imgs, mask)] images = [] for animg in imgs: print(animg) try: o = s.SingleImage(animg, borders=True, crop=((150, 150), (150, 150))) images.append(o) except: pass #~ hdu = fits.open(animg) #~ data = hdu[0].data #~ images = [s.SingleImage(animg, borders=True, crop=((250,250), (250, 250))) #~ for animg in imgs] fits.writeto(os.path.join(dest_dir, 'InterpedRef.fits'), images[0].interped, overwrite=True) for i, animg in enumerate(images[1:]): #~ ## Erasing stars srcs = animg.best_sources f60 = np.percentile(srcs['cflux'], q=60) jj = np.random.choice(len(srcs), len(srcs) / 3, replace=False) mea, med, std = sigma_clipped_stats(animg.data) sx, sy = animg.stamp_shape st = animg._bkg.globalrms for aj in jj: star = srcs[aj] x = star['x'] y = star['y'] if x < sx or animg.data.shape[0] - x < sx: continue if y < sy or animg.data.shape[1] - y < sy: continue if star['cflux'] < f60: continue print(x, y) noise = np.random.normal(loc=med, scale=st, size=animg.stamp_shape) animg.data.data[np.int(x - sx / 2.):np.int(x + sx / 2.), np.int(y - sy / 2.):np.int(y + sy / 2.)] = noise ## Adding stars #~ foo = animg.cov_matrix #~ srcs = animg.best_sources #~ jj = np.random.choice(len(srcs), 12, replace=False) #~ for aj in jj: #~ star = animg.db.load(aj)[0] #~ x, y = np.random.choice(np.min(animg.data.shape)-np.max(star.shape), #~ 2, replace=True) #~ print star.shape #~ animg.data.data[x:x+star.shape[0], y:y+star.shape[1]] = star #~ xc, yc = x+star.shape[0]/2., y+star.shape[1]/2. #~ print xc, yc fits.writeto(os.path.join(dest_dir, 'InterpedNew_{}.fits'.format(i)), animg.interped, overwrite=True) try: D, P, S_corr, mask = ps.diff(animg, images[0], align=True, iterative=False, shift=False, beta=True) mea, med, std = sigma_clipped_stats(D.real, mask) D = np.ma.MaskedArray(D.real, mask).filled(mea) fits.writeto(os.path.join(dest_dir, 'Diff_{}.fits'.format(i)), D, overwrite=True) fits.writeto(os.path.join(dest_dir, 'P_{}.fits'.format(i)), P.real, overwrite=True) fits.writeto(os.path.join(dest_dir, 'Scorr_{}.fits'.format(i)), S_corr, overwrite=True) except: print('subtraction failed') print('ref: ', images[0]) print('new: ', animg) for an_img in images: an_img._clean() return
def main(ref_path, new_path, objname): # alineo las imagenes refdata = fits.getdata(os.path.join(STACK_PATH, ref_path))[250:-250, 250:-250] newdata = fits.getdata(os.path.join(STACK_PATH, new_path))[250:-250, 250:-250] try: trf, _ = aa.find_transform(newdata.astype('<f8'), refdata.astype('<f8')) new_aligned = aa.apply_transform(t, newdata.astype('<f8'), refdata.astype('<f8')) from skimage.transform import warp init_mask = np.zeros_like(newdata) outside_px_mask = warp(init_mask, inverse_map=trf.inverse, output_shape=refdata.shape, order=3, mode='constant', cval=1., clip=False, preserve_range=False) useful = np.where(outside_px_mask == 0) max_x = np.max(useful[0]) min_x = np.min(useful[0]) max_y = np.max(useful[1]) min_y = np.min(useful[1]) new_cropped = new_aligned[min_x:max_x, min_y:max_y] new_mask = outside_px_mask[min_x:max_x, min_y:max_y] ref_cropped = refdata[min_x:max_x, min_y:max_y] except: try: aa.MIN_MATCHES_FRACTION = 0.01 ref = si.SingleImage(refdata.astype('<f8')) new = si.SingleImage(newdata.astype('<f8')) ref.best_sources.sort(order='flux') new.best_sources.sort(order='flux') #~ import ipdb; ipdb.set_trace() rs = np.empty((len(ref.best_sources), 2)) j = 0 for x, y in ref.best_sources[['x', 'y']]: rs[j] = x, y j += 1 ns = np.empty((len(new.best_sources), 2)) j = 0 for x, y in new.best_sources[['x', 'y']]: ns[j] = x, y j += 1 if abs(len(ns) - len(rs)) > 50: max_l = np.max([len(ns), len(rs)]) ns = ns[:max_l] rs = rs[:max_l] trf, _ = aa.find_transform(ns, rs) new_aligned = aa.apply_transform(trf, newdata.astype('<f8'), refdata.astype('<f8')) from skimage.transform import warp init_mask = np.zeros_like(newdata) outside_px_mask = warp(init_mask, inverse_map=trf.inverse, output_shape=refdata.shape, order=3, mode='constant', cval=1., clip=False, preserve_range=False) useful = np.where(outside_px_mask == 0) max_x = np.max(useful[0]) min_x = np.min(useful[0]) max_y = np.max(useful[1]) min_y = np.min(useful[1]) new_cropped = new_aligned[min_x:max_x, min_y:max_y] new_mask = outside_px_mask[min_x:max_x, min_y:max_y] ref_cropped = refdata[min_x:max_x, min_y:max_y] except: #~ import ipdb; ipdb.set_trace() raise # las copio al lugar designado en la pipeline ref_h = fits.getheader(os.path.join(STACK_PATH, ref_path)) fits.writeto(data=ref_cropped.astype('<f4'), header=ref_h, filename=REFERENCE_IMAGE, overwrite=True) new_h = fits.getheader(os.path.join(STACK_PATH, new_path)) fits.writeto(data=new_cropped.astype('<f4'), header=new_h, filename=NEW_IMAGE, overwrite=True) # creo un file con los detalles meta = {} meta['object'] = objname meta['orig_ref_path'] = ref_path meta['orig_new_path'] = new_path with open(DETAILS_FILE, 'w') as fp: json.dump(meta, fp) return 0
def setUp(self): super(TestNpArrayPicky, self).setUp() print(self.mock_image_data.shape) self.si = s.SingleImage(self.mock_image_data, strict_star_pick=True)
# ============================================================================= # PSF measure test by propercoadd # ============================================================================= datapath = os.path.abspath('/home/bruno/Data/Tolar2017/20170624/NGC_6544') frame = os.path.join(datapath, 'NGC_6544-010.fit') d = fits.getdata(frame) plot.plot_S(d, path='./test/test_images/real_image_test/frame.png') # ============================================================================= # PSF spatially variant # ============================================================================= with si.SingleImage(frame) as sim: a_f, psf_b = sim.get_variable_psf(inf_loss=0.1) x, y = sim.get_afield_domain() normal_image = sim.normal_image interp = sim.interped print((sim.n_sources)) S = sim.s_component plot.plot_psfbasis(psf_b, path='./test/test_images/real_image_test/psf_basis.png') plot.plot_afields(a_f, x, y, path='./test/test_images/real_image_test/a_fields.png')
def setUp(self): super(TestNpArrayMaskPicky, self).setUp() self.si = s.SingleImage(self.mock_image_data, self.mock_image_mask, strict_star_pick=True)
def setUp(self): super(TestNpArray, self).setUp() print(self.mock_image_data.shape) self.si = s.SingleImage(self.mock_image_data)
def setUp(self): super(TestHDUPicky, self).setUp() self.si = s.SingleImage(self.mockimageHdu, strict_star_pick=True)
def setUp(self): super(TestNpArrayMask, self).setUp() self.si = s.SingleImage(self.mock_image_data, self.mock_image_mask)
def setUp(self): super(TestFitsExtensionPicky, self).setUp() self.si = s.SingleImage(self.masked_hdu_path)
def setUp(self): super(TestFitsFile, self).setUp() self.si = s.SingleImage(self.mockfits_path)
def main(imgs_dir, ref_path, new_path, details): if not os.path.isdir(imgs_dir): os.makedirs(imgs_dir) ref_dest = os.path.join(imgs_dir, 'ref.fits') new_dest = os.path.join(imgs_dir, 'new.fits') os.rename(ref_path, ref_dest) os.rename(new_path, new_dest) ref = si.SingleImage(ref_dest, borders=False) #, crop=((150,150), (150, 150))) new = si.SingleImage(new_dest, borders=False) #, crop=((150,150), (150, 150))) #~ ## Adding stars foo = new.cov_matrix srcs = new.best_sources rows = [] for i in range(15): j = np.random.choice(len(srcs), 1, replace=False) flux = srcs['flux'][j][0] xs = srcs['x'][j][0] ys = srcs['y'][j][0] position = (ys, xs) sx, sy = new.stamp_shape sx += 3 sy += 3 star = extract_array(new.pixeldata.data, (sx, sy), position, mode='partial', fill_value=new._bkg.globalrms) #print flux #~ star = flux*new.db.load(j)[0] x = np.random.choice(new.pixeldata.shape[0]-3*sx, 1)[0] + sx#* np.random.random()) y = np.random.choice(new.pixeldata.shape[1]-3*sy, 1)[0] + sy # np.int((new.pixeldata.shape[1]-star.shape[1]) * np.random.random()) #~ if new.pixeldata.data[x:x+sx, y:y+sy].shape != star.shape: #~ import ipdb; ipdb.set_trace() new.pixeldata.data[x:x+sx, y:y+sy] = star #~ except: #~ continue xc = x+sx/2. yc = y+sy/2. app_mag = -2.5*np.log10(flux)+25. rows.append([yc, xc, app_mag, flux]) newcat = Table(rows=rows, names=['x', 'y', 'app_mag', 'flux']) fits.writeto(filename=new_dest, header=fits.getheader(new_dest), data=new.pixeldata.data, overwrite=True) new._clean() new = si.SingleImage(new_dest, borders=False) #, crop=((150,150), (150, 150))) newcat.write(os.path.join(imgs_dir, 'transient.list'), format='ascii.fast_no_header', overwrite=True) fits.writeto(filename=os.path.join(imgs_dir, 'interp_ref.fits'), data=ref.interped, overwrite=True) fits.writeto(filename=os.path.join(imgs_dir, 'interp_new.fits'), data=new.interped, overwrite=True) try: print 'Images to be subtracted: {} {}'.format(ref_dest, new_dest) import time t0 = time.time() D, P, S, mask = ps.diff(ref, new, align=False, iterative=False, shift=False, beta=True) dt_z = time.time() - t0 new._clean() ref._clean() mea, med, std = sigma_clipped_stats(D.real) D = np.ma.MaskedArray(D.real, mask).filled(mea) fits.writeto(os.path.join(imgs_dir,'diff.fits'), D, overwrite=True) #~ utils.encapsule_R(D, path=os.path.join(imgs_dir, 'diff.fits')) utils.encapsule_R(P, path=os.path.join(imgs_dir, 'psf_d.fits')) utils.encapsule_R(S, path=os.path.join(imgs_dir, 's_diff.fits')) scorrdetected = utils.find_S_local_maxima(S, threshold=3.5) print 'S_corr found thath {} transients were above 3.5 sigmas'.format(len(scorrdetected)) ascii.write(table=np.asarray(scorrdetected), output=os.path.join(imgs_dir, 's_corr_detected.csv'), names=['X_IMAGE', 'Y_IMAGE', 'SIGNIFICANCE'], format='csv') S = np.ascontiguousarray(S) #~ s_bkg = sep.Background(S) mean, median, std = sigma_clipped_stats(S) sdetected = sep.extract(S-median, 3.5*std, filter_kernel=None) print 'S_corr with sep found thath {} transients were above 3.5 sigmas'.format(len(sdetected)) ascii.write(table=sdetected, output=os.path.join(imgs_dir, 'sdetected.csv'), format='csv') ## With OIS t0 = time.time() ois_d = ois.optimal_system(fits.getdata(new_dest), fits.getdata(ref_dest))[0] dt_o = time.time() - t0 utils.encapsule_R(ois_d, path=os.path.join(imgs_dir, 'diff_ois.fits')) ## With HOTPANTS t0 = time.time() os.system('hotpants -v 0 -inim {} -tmplim {} -outim {} -r 15 -tu 40000 -tl -100 -il -100 -iu 40000'.format(new_dest, ref_dest, os.path.join(imgs_dir, 'diff_hot.fits'))) dt_h = time.time() - t0 return [newcat.to_pandas(), [dt_z, dt_o, dt_h]] except: raise
def setUp(self): super(TestCoaddMultCore2, self).setUp() self.imgs = [si.SingleImage(img) for img in self.paths]