def filter_outside_extract(ds9region, infilename, catalogue):
hdu = fits.open(infilename)
hduflat = flatten(hdu)
map = hdu[0].data
w = WCS(flatten(hdu).header)
r = pyregion.open(ds9region)
manualmask = r.get_mask(hdu=hduflat)
inregion = []
for element in catalogue:
i, j = w.wcs_world2pix(element['RA'], element['DEC'], 0)
print i, j, manualmask[int(i), int(j)]
inregion.append(manualmask[int(i), int(j)])
return catalogue[inregion]
def hextile(image,radius):
pos=[]
hs=radius*np.sqrt(3)
hdus=fits.open(image)
hdu=flatten(hdus)
maxy,maxx=hdu.data.shape
w=WCS(hdu.header)
print 'Hex tiling image'
# co-ords of bottom left of image
ra_c,dec_c=w.wcs_pix2world(maxx/2,maxy/2,0)
ra_factor=np.cos(dec_c*np.pi/180.0)
ra_ll,dec_ll=w.wcs_pix2world(0,0,0)
ra_lr,dec_lr=w.wcs_pix2world(maxx,0,0)
ra_ul,dec_ul=w.wcs_pix2world(0,maxy,0)
c_c=SkyCoord(ra_c*u.degree,dec_c*u.degree,frame='icrs')
c_ll=SkyCoord(ra_ll*u.degree,dec_ll*u.degree,frame='icrs')
c_lr=SkyCoord(ra_lr*u.degree,dec_lr*u.degree,frame='icrs')
dra,ddec=[v.value for v in c_c.spherical_offsets_to(c_ll)]
nha=dra*2/hs
print 'Number of hexes across',nha
c_ul=SkyCoord(ra_ul*u.degree,dec_ul*u.degree,frame='icrs')
dra,ddec=[v.value for v in c_c.spherical_offsets_to(c_ul)]
nhu=2*ddec/hs
print 'Number of hexes up',nhu
nha=int(0.5+nha)
nhu=int(0.5+nhu)
for j in range(nhu):
for i in range(nha):
xc=(1.0*maxx*(i+(j % 2)*0.5))/nha
yc=(maxy*(j+0.5))/nhu
ra_p,dec_p=w.wcs_pix2world(xc,yc,0)
pos.append((float(ra_p),float(dec_p)))
return ra_factor,pos
def hextile(image,radius):
pos=[]
hs=radius*np.sqrt(3)
hdus=fits.open(image)
hdu=flatten(hdus)
maxy,maxx=hdu.data.shape
w=WCS(hdu.header)
print 'Hex tiling image'
# co-ords of bottom left of image
ra_c,dec_c=w.wcs_pix2world(maxx/2,maxy/2,0)
ra_factor=np.cos(dec_c*np.pi/180.0)
ra_ll,dec_ll=w.wcs_pix2world(0,0,0)
ra_lr,_=w.wcs_pix2world(maxx,0,0)
_,dec_ul=w.wcs_pix2world(0,maxy,0)
nha=(ra_ll-ra_lr)*ra_factor/hs
print 'Number of hexes across',nha
nhu=(dec_ul-dec_ll)/hs
print 'Number of hexes up',nhu
nha=int(0.5+nha)
nhu=int(0.5+nhu)
for j in range(nhu):
for i in range(nha):
xp=ra_lr+(i*hs+(j % 2)*0.5*hs)/ra_factor
yp=dec_ll+((j+0.25)*hs)
pos.append((xp,yp))
return ra_factor,pos
def add_manual_mask(infile, ds9region, outfile):
hdu = fits.open(infile)
hduflat = flatten(hdu)
map = hdu[0].data
r = pyregion.open(ds9region)
manualmask = r.get_mask(hdu=hduflat)
hdu[0].data = (map.astype(int) | manualmask).astype(np.float32)
hdu.writeto(outfile, clobber=True)
def add_manual_mask(infile,ds9region,outfile):
hdu=fits.open(infile)
hduflat = flatten(hdu)
map=hdu[0].data
r = pyregion.open(ds9region)
manualmask = r.get_mask(hdu=hduflat)
hdu[0].data=(map.astype(int) | manualmask).astype(np.float32)
hdu.writeto(outfile,clobber=True)
def filter_inside_extract(ds9region, infilename, catalogue):
hdu = fits.open(infilename)
hduflat = flatten(hdu)
map = hdu[0].data
w = WCS(flatten(hdu).header)
r = pyregion.open(ds9region)
manualmask = r.get_mask(hdu=hduflat)
inregion = []
for element in catalogue:
i, j = w.wcs_world2pix(element['RA'], element['DEC'], 0)
if i < 0.0 or j < 0.0:
# outside map
inregion.append(False)
continue
try:
inregion.append(~manualmask[int(j), int(i)])
except IndexError:
#print 'Going into exception'
inregion.append(False)
return catalogue[inregion]
def make_mosaic(args):
if args.scale is not None:
if len(args.scale) != len(args.directories):
die('Scales provided must match directories')
if args.noise is not None:
if len(args.noise) != len(args.directories):
die('Noises provided must match directories')
if args.rootname:
rootname=args.rootname+'-'
else:
rootname=''
if args.exact:
reproj=reproject_exact
else:
reproj=reproject_interp_chunk_2d
if args.do_lowres:
intname='image_full_low_m.int.restored.fits'
appname='image_full_low_m.app.restored.fits'
elif args.use_shifted:
intname='image_full_ampphase_di_m.NS_shift.int.facetRestored.fits'
appname='image_full_ampphase_di_m.NS_shift.app.facetRestored.fits'
else:
intname='image_full_ampphase_di_m.NS.int.restored.fits'
appname='image_full_ampphase_di_m.NS.app.restored.fits'
# astromap blanking if required
bth=None
try:
bth=float(args.astromap_blank)
except:
pass
threshold=float(args.beamcut)
hdus=[]
app=[]
astromaps=[]
wcs=[]
print 'Reading files...'
noise=[]
name=[]
for d in args.directories:
name.append(d.split('/')[-1])
hdu=fits.open(d+'/'+intname)
if args.find_noise:
print 'Estimating noise for', d+'/' + intname
if args.do_lowres:
noise.append(get_rms(hdu,boxsize=1500))
else:
noise.append(get_rms(hdu))
hdus.append(flatten(hdu))
app.append(flatten(fits.open(d+'/'+appname)))
if bth:
astromaps.append(flatten(fits.open(d+'/astromap.fits')))
if args.find_noise:
args.noise=noise
print 'Noise values are:'
for t,n in zip(name,noise):
print t,n
print 'Computing noise/beam factors...'
for i in range(len(app)):
np.seterr(divide='ignore')
app[i].data=np.divide(app[i].data,hdus[i].data)
app[i].data[app[i].data<threshold]=0
# at this point this is the beam factor: we want 1/sigma**2.0, so divide by central noise and square
if args.noise is not None:
if args.scale is not None:
app[i].data/=args.noise[i]*args.scale[i]
else:
app[i].data/=args.noise[i]
app[i].data=app[i].data**2.0
if args.scale is not None:
hdus[i].data*=args.scale[i]
if args.shift:
print 'Finding shifts (NOTE THIS CODE IS OBSOLETE)...'
# shift according to the FIRST delta ra/dec from quality pipeline
dras=[]
ddecs=[]
for d in args.directories:
t=Table.read(d+'/image_full_ampphase1m.cat.fits_FIRST_match_filtered.fits')
dras.append(np.mean(t['FIRST_dRA']))
ddecs.append(np.mean(t['FIRST_dDEC']))
print 'Applying shifts:',dras,ddecs
for i in range(len(app)):
for hdu in [hdus[i],app[i]]:
ra=hdu.header['CRVAL1']
dec=hdu.header['CRVAL2']
hdu.header['CRVAL1']-=dras[i]/(3600.0*np.cos(np.pi*dec/180.0))
hdu.header['CRVAL2']-=ddecs[i]/3600.0
for i in range(len(app)):
wcs.append(WCS(hdus[i].header))
# astromap blanking
if bth:
print 'Blanking using astrometry quality maps with threshold',bth,'arcsec'
for i in range(len(app)):
outname=rootname+'astroblank-'+name[i]+'.fits'
if args.load and os.path.isfile(outname):
print 'Loading previously blanked image'
hdu=fits.open(outname)
hdus[i].data=hdu[0].data
else:
print 'Blanking image',i
dmaxy,dmaxx=hdus[i].data.shape
count=0
am=astromaps[i]
awcs=WCS(am.header)
maxy,maxx=am.data.shape
for y in range(maxy):
for x in range(maxx):
value=am.data[y,x]
if np.isnan(value):
if y<maxy-1:
value=am.data[y+1,x]
if value>bth:
ra,dec=[float(f) for f in awcs.wcs_pix2world(x,y,0)]
rx,ry=[int(p) for p in wcs[i].wcs_world2pix(ra,dec,0)]
rxp=rx+21 # astromap pix size, with margin
ryp=ry+21
if rx<0: rx=0
if ry<0: ry=0
if rxp>dmaxx: rxp=dmaxx
if ryp>dmaxy: ryp=dmaxy
hdus[i].data[ry:ryp,rx:rxp]=np.nan
count+=1
print '... blanked',count*900.0/3600,'square arcmin'
outname=rootname+'astroblank-'+name[i]+'.fits'
if args.save: hdus[i].writeto(outname,clobber=True)
app[i].data[np.isnan(hdus[i].data)]=np.nan
# If the header is directly passed in, use it
try:
header=args.header
xsize=header['NAXIS1']
ysize=header['NAXIS2']
print 'Mosaic using header passed from calling program'
except:
header=None
if header is None:
if args.load_layout:
with open(rootname+'mosaic-header.pickle') as f:
header=pickle.load(f)
xsize=header['NAXIS1']
ysize=header['NAXIS2']
print 'Mosaic using loaded header'
else:
print 'Creating the mosaic header'
ras=np.array([w.wcs.crval[0] for w in wcs])
decs=np.array([w.wcs.crval[1] for w in wcs])
mra=np.mean(ras)
mdec=np.mean(decs)
print 'Will make mosaic at',mra,mdec
# we make a reference WCS and use it to find the extent in pixels
# needed for the combined image
rwcs=WCS(naxis=2)
rwcs.wcs.ctype=wcs[0].wcs.ctype
rwcs.wcs.cdelt=wcs[0].wcs.cdelt
rwcs.wcs.crval=[mra,mdec]
rwcs.wcs.crpix=[1,1]
xmin=0
xmax=0
ymin=0
ymax=0
for a,w in zip(app,wcs):
ys,xs=np.where(a.data)
axmin=xs.min()
aymin=ys.min()
axmax=xs.max()
aymax=ys.max()
del(xs)
del(ys)
print 'non-zero',axmin,aymin,axmax,aymax
for x,y in ((axmin,aymin),(axmax,aymin),(axmin,aymax),(axmax,aymax)):
ra,dec=[float(f) for f in w.wcs_pix2world(x,y,0)]
#print ra,dec
nx,ny=[float (f) for f in rwcs.wcs_world2pix(ra,dec,0)]
print nx,ny
if nx<xmin: xmin=nx
if nx>xmax: xmax=nx
if ny<ymin: ymin=ny
if ny>ymax: ymax=ny
print 'co-ord range:', xmin, xmax, ymin, ymax
xsize=int(xmax-xmin)
ysize=int(ymax-ymin)
rwcs.wcs.crpix=[-int(xmin)+1,-int(ymin)+1]
print 'checking:', rwcs.wcs_world2pix(mra,mdec,0)
print rwcs
header=rwcs.to_header()
header['NAXIS']=2
header['NAXIS1']=xsize
header['NAXIS2']=ysize
with open(rootname+'mosaic-header.pickle','w') as f:
pickle.dump(header,f)
isum=np.zeros([ysize,xsize])
wsum=np.zeros_like(isum)
mask=np.zeros_like(isum,dtype=np.bool)
print 'now making the mosaic'
for i in range(len(hdus)):
print 'image',i,'(',name[i],')'
outname=rootname+'reproject-'+name[i]+'.fits'
if args.load and os.path.exists(outname):
print 'loading...'
hdu=fits.open(outname)
r=hdu[0].data
else:
print 'reprojecting...'
r, footprint = reproj(hdus[i], header, hdu_in=0, parallel=False)
r[np.isnan(r)]=0
hdu = fits.PrimaryHDU(header=header,data=r)
if args.save: hdu.writeto(outname,clobber=True)
print 'weights',i,'(',name[i],')'
outname=rootname+'weight-'+name[i]+'.fits'
if args.load and os.path.exists(outname):
print 'loading...'
hdu=fits.open(outname)
w=hdu[0].data
mask|=(w>0)
else:
print 'reprojecting...'
w, footprint = reproj(app[i], header, hdu_in=0, parallel=False)
mask|=~np.isnan(w)
w[np.isnan(w)]=0
hdu = fits.PrimaryHDU(header=header,data=w)
if args.save: hdu.writeto(outname,clobber=True)
print 'add to mosaic...'
isum+=r*w
wsum+=w
if not(args.no_write):
isum/=wsum
# mask now contains True where a non-nan region was present in either map
isum[~mask]=np.nan
for ch in ('BMAJ', 'BMIN', 'BPA'):
header[ch]=hdus[0].header[ch]
header['ORIGIN']='ddf-pipeline '+version()
hdu = fits.PrimaryHDU(header=header,data=isum)
hdu.writeto(rootname+'mosaic.fits',clobber=True)
hdu = fits.PrimaryHDU(header=header,data=wsum)
hdu.writeto(rootname+'mosaic-weights.fits',clobber=True)
def make_mosaic(args):
if args.scale is not None:
if len(args.scale) != len(args.directories):
die('Scales provided must match directories')
if args.noise is not None:
if len(args.noise) != len(args.directories):
die('Noises provided must match directories')
if args.rootname:
rootname = args.rootname + '-'
else:
rootname = ''
if args.exact:
reproj = reproject_exact
else:
reproj = reproject_interp_chunk_2d
if args.do_lowres:
intname = 'image_full_low_m.int.restored.fits'
appname = 'image_full_low_m.app.restored.fits'
elif args.use_shifted:
intname = 'image_full_ampphase_di_m.NS_shift.int.facetRestored.fits'
appname = 'image_full_ampphase_di_m.NS_shift.app.facetRestored.fits'
else:
intname = 'image_full_ampphase_di_m.NS.int.restored.fits'
appname = 'image_full_ampphase_di_m.NS.app.restored.fits'
# astromap blanking if required
bth = None
try:
bth = float(args.astromap_blank)
except:
pass
threshold = float(args.beamcut)
hdus = []
app = []
astromaps = []
wcs = []
print 'Reading files...'
noise = []
name = []
for d in args.directories:
name.append(d.split('/')[-1])
hdu = fits.open(d + '/' + intname)
if args.find_noise:
print 'Estimating noise for', d + '/' + intname
if args.do_lowres:
noise.append(get_rms(hdu, boxsize=1500))
else:
noise.append(get_rms(hdu))
hdus.append(flatten(hdu))
app.append(flatten(fits.open(d + '/' + appname)))
if bth:
astromaps.append(flatten(fits.open(d + '/astromap.fits')))
if args.find_noise:
args.noise = noise
print 'Noise values are:'
for t, n in zip(name, noise):
print t, n
print 'Computing noise/beam factors...'
for i in range(len(app)):
np.seterr(divide='ignore')
app[i].data = np.divide(app[i].data, hdus[i].data)
app[i].data[app[i].data < threshold] = 0
# at this point this is the beam factor: we want 1/sigma**2.0, so divide by central noise and square
if args.noise is not None:
if args.scale is not None:
app[i].data /= args.noise[i] * args.scale[i]
else:
app[i].data /= args.noise[i]
app[i].data = app[i].data**2.0
if args.scale is not None:
hdus[i].data *= args.scale[i]
if args.shift:
print 'Finding shifts (NOTE THIS CODE IS OBSOLETE)...'
# shift according to the FIRST delta ra/dec from quality pipeline
dras = []
ddecs = []
for d in args.directories:
t = Table.read(
d +
'/image_full_ampphase1m.cat.fits_FIRST_match_filtered.fits')
dras.append(np.mean(t['FIRST_dRA']))
ddecs.append(np.mean(t['FIRST_dDEC']))
print 'Applying shifts:', dras, ddecs
for i in range(len(app)):
for hdu in [hdus[i], app[i]]:
ra = hdu.header['CRVAL1']
dec = hdu.header['CRVAL2']
hdu.header['CRVAL1'] -= dras[i] / (3600.0 *
np.cos(np.pi * dec / 180.0))
hdu.header['CRVAL2'] -= ddecs[i] / 3600.0
for i in range(len(app)):
wcs.append(WCS(hdus[i].header))
# astromap blanking
if bth:
print 'Blanking using astrometry quality maps with threshold', bth, 'arcsec'
for i in range(len(app)):
outname = rootname + 'astroblank-' + name[i] + '.fits'
if args.load and os.path.isfile(outname):
print 'Loading previously blanked image'
hdu = fits.open(outname)
hdus[i].data = hdu[0].data
else:
print 'Blanking image', i
dmaxy, dmaxx = hdus[i].data.shape
count = 0
am = astromaps[i]
awcs = WCS(am.header)
maxy, maxx = am.data.shape
for y in range(maxy):
for x in range(maxx):
value = am.data[y, x]
if np.isnan(value):
if y < maxy - 1:
value = am.data[y + 1, x]
if value > bth:
ra, dec = [
float(f) for f in awcs.wcs_pix2world(x, y, 0)
]
rx, ry = [
int(p)
for p in wcs[i].wcs_world2pix(ra, dec, 0)
]
rxp = rx + 21 # astromap pix size, with margin
ryp = ry + 21
if rx < 0: rx = 0
if ry < 0: ry = 0
if rxp > dmaxx: rxp = dmaxx
if ryp > dmaxy: ryp = dmaxy
hdus[i].data[ry:ryp, rx:rxp] = np.nan
count += 1
print '... blanked', count * 900.0 / 3600, 'square arcmin'
outname = rootname + 'astroblank-' + name[i] + '.fits'
if args.save: hdus[i].writeto(outname, clobber=True)
app[i].data[np.isnan(hdus[i].data)] = np.nan
# If the header is directly passed in, use it
try:
header = args.header
xsize = header['NAXIS1']
ysize = header['NAXIS2']
print 'Mosaic using header passed from calling program'
except:
header = None
if header is None:
if args.load_layout:
with open(rootname + 'mosaic-header.pickle') as f:
header = pickle.load(f)
xsize = header['NAXIS1']
ysize = header['NAXIS2']
print 'Mosaic using loaded header'
else:
print 'Creating the mosaic header'
ras = np.array([w.wcs.crval[0] for w in wcs])
decs = np.array([w.wcs.crval[1] for w in wcs])
mra = np.mean(ras)
mdec = np.mean(decs)
print 'Will make mosaic at', mra, mdec
# we make a reference WCS and use it to find the extent in pixels
# needed for the combined image
rwcs = WCS(naxis=2)
rwcs.wcs.ctype = wcs[0].wcs.ctype
rwcs.wcs.cdelt = wcs[0].wcs.cdelt
rwcs.wcs.crval = [mra, mdec]
rwcs.wcs.crpix = [1, 1]
xmin = 0
xmax = 0
ymin = 0
ymax = 0
for a, w in zip(app, wcs):
ys, xs = np.where(a.data)
axmin = xs.min()
aymin = ys.min()
axmax = xs.max()
aymax = ys.max()
del (xs)
del (ys)
print 'non-zero', axmin, aymin, axmax, aymax
for x, y in ((axmin, aymin), (axmax, aymin), (axmin, aymax),
(axmax, aymax)):
ra, dec = [float(f) for f in w.wcs_pix2world(x, y, 0)]
#print ra,dec
nx, ny = [float(f) for f in rwcs.wcs_world2pix(ra, dec, 0)]
print nx, ny
if nx < xmin: xmin = nx
if nx > xmax: xmax = nx
if ny < ymin: ymin = ny
if ny > ymax: ymax = ny
print 'co-ord range:', xmin, xmax, ymin, ymax
xsize = int(xmax - xmin)
ysize = int(ymax - ymin)
rwcs.wcs.crpix = [-int(xmin) + 1, -int(ymin) + 1]
print 'checking:', rwcs.wcs_world2pix(mra, mdec, 0)
print rwcs
header = rwcs.to_header()
header['NAXIS'] = 2
header['NAXIS1'] = xsize
header['NAXIS2'] = ysize
with open(rootname + 'mosaic-header.pickle', 'w') as f:
pickle.dump(header, f)
isum = np.zeros([ysize, xsize])
wsum = np.zeros_like(isum)
mask = np.zeros_like(isum, dtype=np.bool)
print 'now making the mosaic'
for i in range(len(hdus)):
print 'image', i, '(', name[i], ')'
outname = rootname + 'reproject-' + name[i] + '.fits'
if args.load and os.path.exists(outname):
print 'loading...'
hdu = fits.open(outname)
r = hdu[0].data
else:
print 'reprojecting...'
r, footprint = reproj(hdus[i], header, hdu_in=0, parallel=False)
r[np.isnan(r)] = 0
hdu = fits.PrimaryHDU(header=header, data=r)
if args.save: hdu.writeto(outname, clobber=True)
print 'weights', i, '(', name[i], ')'
outname = rootname + 'weight-' + name[i] + '.fits'
if args.load and os.path.exists(outname):
print 'loading...'
hdu = fits.open(outname)
w = hdu[0].data
mask |= (w > 0)
else:
print 'reprojecting...'
w, footprint = reproj(app[i], header, hdu_in=0, parallel=False)
mask |= ~np.isnan(w)
w[np.isnan(w)] = 0
hdu = fits.PrimaryHDU(header=header, data=w)
if args.save: hdu.writeto(outname, clobber=True)
print 'add to mosaic...'
isum += r * w
wsum += w
if not (args.no_write):
isum /= wsum
# mask now contains True where a non-nan region was present in either map
isum[~mask] = np.nan
for ch in ('BMAJ', 'BMIN', 'BPA'):
header[ch] = hdus[0].header[ch]
header['ORIGIN'] = 'ddf-pipeline ' + version()
hdu = fits.PrimaryHDU(header=header, data=isum)
hdu.writeto(rootname + 'mosaic.fits', clobber=True)
hdu = fits.PrimaryHDU(header=header, data=wsum)
hdu.writeto(rootname + 'mosaic-weights.fits', clobber=True)
#!/usr/bin/env python
# make a header for a mosaic using an input image as a template.
# then we can run
# mosaic.py --save --use_shifted --load_layout --find_noise --directories ...
from __future__ import division
from past.utils import old_div
from auxcodes import flatten
import sys
from astropy.io import fits
import pickle
template=sys.argv[1]
hdus=fits.open(template)
hdu=flatten(hdus)
size=2.5
cellsize=1.5/3600.0
himsize=int(old_div(size,cellsize))
hdu.header['NAXIS1']=2*himsize
hdu.header['NAXIS2']=2*himsize
hdu.header['CRPIX1']=himsize
hdu.header['CRPIX2']=himsize
# fix up headers
hdu.header['TELESCOP']='LOFAR'
hdu.header['RESTFRQ']=143.65e6
hdu.header['OBSERVER']='LoTSS'
with open('mosaic-header.pickle','w') as f:
pickle.dump(hdu.header,f)
#!/usr/bin/env python
# make a header for a mosaic using an input image as a template.
# then we can run
# mosaic.py --save --use_shifted --load_layout --find_noise --directories ...
from auxcodes import flatten
import sys
from astropy.io import fits
import pickle
template=sys.argv[1]
hdus=fits.open(template)
hdu=flatten(hdus)
size=2.5
cellsize=1.5/3600.0
himsize=int(size/cellsize)
hdu.header['NAXIS1']=2*himsize
hdu.header['NAXIS2']=2*himsize
hdu.header['CRPIX1']=himsize
hdu.header['CRPIX2']=himsize
# fix up headers
hdu.header['TELESCOP']='LOFAR'
hdu.header['RESTFRQ']=143.65e6
hdu.header['OBSERVER']='LoTSS'
with open('mosaic-header.pickle','w') as f:
pickle.dump(hdu.header,f)
