def find_pointings_to_mosaic(pointingdict,mospointingname): seps = np.array([]) tomospointings = np.array([]) ra1,dec1 = pointingdict[mospointingname][1],pointingdict[mospointingname][2] for j in pointingdict: ra2,dec2 = pointingdict[j][1],pointingdict[j][2] seps = np.append(seps,(sepn(ra1*deg2rad,dec1*deg2rad,ra2*deg2rad,dec2*deg2rad)*rad2deg)) tomospointings = np.append(tomospointings,j) sortedseps = np.sort(seps) # For now include the 6 closest pointings plus keep including others until we have all that are within the distance of 1.1 times the 6th closest pointing. At later stage try to take into account e.g. elongation of beam for low declination observations. endmosindex = 7 for i in range(7,20): if (sortedseps[i])/(sortedseps[6]) < 1.1: endmosindex = i mosaicpointings = tomospointings[np.where(seps < sortedseps[endmosindex])] mosseps = seps[np.where(seps < sortedseps[endmosindex])] return mosaicpointings,mosseps
def find_median_astrometry(astromaps,pointingra,pointingdec):
foundpointing = False
for astromap in astromaps:
ra,dec=getposim('%s/astromap.fits'%astromap)
if sepn(ra*deg2rad,dec*deg2rad,pointingra,pointingdec)*rad2deg < 0.6:
foundpointing = True
f = fits.open('%s/astromap.fits'%astromap)
_,_,ys,xs=f[0].data.shape
# Use central 20% of the image
subim=f[0].data[0][0][ys/2-ys/5:ys/2+ys/5,xs/2-xs/5:xs/2+xs/5].flatten()
if foundpointing:
subim = np.nan_to_num(subim)
return np.median(subim)
else:
print 'Cant find astrometry near %s, %s'%(pointingra*rad2deg,pointingdec*rad2deg)
sys.exit(0)
def find_pointings_to_mosaic(pointingdict, mospointingname):
seps = np.array([])
tomospointings = np.array([])
ra1, dec1 = pointingdict[mospointingname][1], pointingdict[
mospointingname][2]
for j in pointingdict:
ra2, dec2 = pointingdict[j][1], pointingdict[j][2]
seps = np.append(seps, (sepn(ra1 * deg2rad, dec1 * deg2rad,
ra2 * deg2rad, dec2 * deg2rad) * rad2deg))
tomospointings = np.append(tomospointings, j)
sortedseps = np.sort(seps)
# For now include the 6 closest pointings plus keep including others until we have all that are within the distance of 1.1 times the 6th closest pointing. At later stage try to take into account e.g. elongation of beam for low declination observations.
endmosindex = 7
for i in range(7, 20):
if (sortedseps[i]) / (sortedseps[6]) < 1.1:
endmosindex = i
mosaicpointings = tomospointings[np.where(seps < sortedseps[endmosindex])]
mosseps = seps[np.where(seps < sortedseps[endmosindex])]
return mosaicpointings, mosseps
def find_median_astrometry(astromaps,pointingra,pointingdec):
foundpointing = False
for astromap in astromaps:
amname='%s/astromap.fits'%astromap
if not os.path.isfile(amname):
continue
ra,dec=getposim(amname)
if sepn(ra*deg2rad,dec*deg2rad,pointingra,pointingdec)*rad2deg < 0.6:
foundpointing = True
f = fits.open('%s/astromap.fits'%astromap)
_,_,ys,xs=f[0].data.shape
# Use central 20% of the image
subim=f[0].data[0][0][old_div(ys,2)-old_div(ys,5):old_div(ys,2)+old_div(ys,5),old_div(xs,2)-old_div(xs,5):old_div(xs,2)+old_div(xs,5)].flatten()
if foundpointing:
subim = np.nan_to_num(subim)
return np.median(subim)
else:
print('Cant find astrometry near %s, %s'%(pointingra*rad2deg,pointingdec*rad2deg))
return None
def filter_catalogs(args, pointingras, pointingdecs, mosaiccat, outname,
dessourcenums, cattype):
pointdirectories = args.pointdirectories
sourceids = np.array([])
sourceresolved = np.array([])
sourcera = np.array([])
e_sourcera = np.array([])
e_sourcera_tot = np.array([])
sourcedec = np.array([])
e_sourcedec = np.array([])
e_sourcedec_tot = np.array([])
sint = np.array([])
e_sint = np.array([])
e_sint_tot = np.array([])
speak = np.array([])
e_speak = np.array([])
e_speak_tot = np.array([])
maj = np.array([])
e_maj = np.array([])
smin = np.array([])
e_smin = np.array([])
dcmaj = np.array([])
e_dcmaj = np.array([])
dcsmin = np.array([])
e_dcsmin = np.array([])
pa = np.array([])
e_pa = np.array([])
dcpa = np.array([])
e_dcpa = np.array([])
rms_noise = np.array([])
stype = np.array([])
mosaic_identifier = np.array([])
gausid = np.array([])
islid = np.array([])
sourcenum = np.array([])
pointing = mosaiccat.replace('.cat.fits', '-blanked.fits')
if cattype == 'gaus':
sourcecat = fits.open(mosaiccat)
mosaiccat = glob.glob(
mosaiccat.replace(
'mosaic.cat.fits',
'mosaic-blanked_pybdsm/*/catalogues/mosaic-blanked.pybdsm.gaul.FITS'
))[0]
cat = fits.open(mosaiccat)
f = fits.open(pointing)
rapointing = f[0].header['CRVAL1'] * deg2rad
decpointing = f[0].header['CRVAL2'] * deg2rad
f.close()
numsources = len(cat[1].data['RA'])
closepointingindex = np.where(
sepn(pointingras, pointingdecs, rapointing, decpointing) *
rad2deg < 5.0)
astromed = find_median_astrometry(pointdirectories, rapointing,
decpointing)
if astromed is None:
astromed = 5.0 # missing data, assume bad e.g. hole
keepindices = []
time1 = time.time()
for i in range(0, numsources):
if dessourcenums == None:
allsep = sepn(pointingras[closepointingindex],
pointingdecs[closepointingindex],
cat[1].data['RA'][i] * deg2rad,
cat[1].data['DEC'][i] * deg2rad)
centsep = sepn(rapointing, decpointing,
cat[1].data['RA'][i] * deg2rad,
cat[1].data['DEC'][i] * deg2rad)
if min(allsep) != centsep:
continue
else:
keepindices.append(i)
else:
if cat[1].data['Source_id'][i] in dessourcenums:
keepindices.append(i)
else:
continue
if cattype == 'srl':
sc = SkyCoord(cat[1].data['RA'][i] * deg2rad * u.rad,
cat[1].data['DEC'][i] * deg2rad * u.rad,
frame='icrs')
if cattype == 'gaus':
sourceindex = cat[1].data['Source_id'][i]
sc = SkyCoord(
sourcecat[1].data['RA'][sourceindex] * deg2rad * u.rad,
sourcecat[1].data['DEC'][sourceindex] * deg2rad * u.rad,
frame='icrs')
s = sc.to_string(style='hmsdms', sep='', precision=3)
s = sc.to_string(style='hmsdms', sep='', precision=2)
identity = str('ILTJ' + s).replace(' ', '')[:-1]
sourceids = np.append(sourceids, identity)
if cattype == 'srl':
mosaic_identifier = np.append(mosaic_identifier,
mosaiccat.split('/')[-2])
if cattype == 'gaus':
mosaic_identifier = np.append(mosaic_identifier,
mosaiccat.split('/')[-5])
fluxratio = cat[1].data['Total_flux'][i] / cat[1].data['Peak_flux'][i]
snr = cat[1].data['Peak_flux'][i] / cat[1].data['Isl_rms'][i]
# Some equation to figure out if the source is resolved -- leave these dummy values for now.
if fluxratio > (1.483 + 1000.4 / (snr**3.94)):
#if fluxratio > (1.50341355 + 1.78467767/(snr**0.78385826)):
sourceresolved = np.append(sourceresolved, 'R')
else:
sourceresolved = np.append(sourceresolved, 'U')
print 'Keeping %s sources for %s -- took %s' % (len(keepindices), pointing,
time.time() - time1)
sourcera = np.append(sourcera, cat[1].data[keepindices]['RA'])
e_sourcera = np.append(e_sourcera, cat[1].data[keepindices]['E_RA'])
e_sourcera_tot = np.append(
e_sourcera_tot,
(cat[1].data[keepindices]['E_RA']**2.0 + (astromed * arcsec2deg)**
2.0)**0.5) #$ ADD SOME ERROR TO THE SOURCE POSITIONS
sourcedec = np.append(sourcedec, cat[1].data[keepindices]['DEC'])
e_sourcedec = np.append(e_sourcedec, cat[1].data[keepindices]['E_DEC'])
e_sourcedec_tot = np.append(
e_sourcedec_tot,
(cat[1].data[keepindices]['E_DEC']**2.0 + (astromed * arcsec2deg)**
2.0)**0.5) # ADD SOME ERROR TO THE SOURCE POSITIONS
sint = np.append(sint, cat[1].data[keepindices]['Total_flux'])
e_sint = np.append(e_sint, cat[1].data[keepindices]['E_Total_flux'])
e_sint_tot = np.append(
e_sint_tot, (cat[1].data[keepindices]['E_Total_flux']**2.0 +
(cat[1].data[keepindices]['Total_flux'] * 0.2)**2.0)**0.5)
speak = np.append(speak, cat[1].data[keepindices]['Peak_flux'])
e_speak = np.append(e_speak, cat[1].data[keepindices]['E_Peak_flux'])
e_speak_tot = np.append(
e_speak_tot, (cat[1].data[keepindices]['E_Peak_flux']**2.0 +
(cat[1].data[keepindices]['Peak_flux'] * 0.2)**2.0)**0.5)
maj = np.append(maj, cat[1].data[keepindices]['Maj'])
e_maj = np.append(e_maj, cat[1].data[keepindices]['E_Maj'])
smin = np.append(smin, cat[1].data[keepindices]['Min'])
e_smin = np.append(e_smin, cat[1].data[keepindices]['E_Min'])
dcmaj = np.append(dcmaj, cat[1].data[keepindices]['DC_Maj'])
e_dcmaj = np.append(e_dcmaj, cat[1].data[keepindices]['E_DC_Maj'])
dcsmin = np.append(dcsmin, cat[1].data[keepindices]['DC_Min'])
e_dcsmin = np.append(e_dcsmin, cat[1].data[keepindices]['E_DC_Min'])
pa = np.append(pa, cat[1].data[keepindices]['PA'])
e_pa = np.append(e_pa, cat[1].data[keepindices]['E_PA'])
dcpa = np.append(dcpa, cat[1].data[keepindices]['DC_PA'])
e_dcpa = np.append(e_dcpa, cat[1].data[keepindices]['E_DC_PA'])
rms_noise = np.append(rms_noise, cat[1].data[keepindices]['Isl_rms'])
stype = np.append(stype, cat[1].data[keepindices]['S_Code'])
islid = np.append(islid, cat[1].data[keepindices]['Isl_id'])
sourcenum = np.append(sourcenum, cat[1].data[keepindices]['Source_id'])
col1 = fits.Column(name='Source_Name',
format='24A',
unit='',
array=sourceids)
col2 = fits.Column(name='RA', format='f8', unit='deg', array=sourcera)
col3 = fits.Column(name='E_RA',
format='f8',
unit='arcsec',
array=e_sourcera * deg2arcsec)
col4 = fits.Column(name='E_RA_tot',
format='f8',
unit='arcsec',
array=e_sourcera_tot * deg2arcsec)
col5 = fits.Column(name='DEC', format='f8', unit='deg', array=sourcedec)
col6 = fits.Column(name='E_DEC',
format='f8',
unit='arcsec',
array=e_sourcedec * deg2arcsec)
col7 = fits.Column(name='E_DEC_tot',
format='f8',
unit='arcsec',
array=e_sourcedec_tot * deg2arcsec)
col8 = fits.Column(name='Peak_flux',
format='f8',
unit='beam-1 mJy',
array=speak * 1000.0)
col9 = fits.Column(name='E_Peak_flux',
format='f8',
unit='beam-1 mJy',
array=e_speak * 1000.0)
col10 = fits.Column(name='E_Peak_flux_tot',
format='f8',
unit='beam-1 mJy',
array=e_speak_tot * 1000.0)
col11 = fits.Column(name='Total_flux',
format='f8',
unit='mJy',
array=sint * 1000.0)
col12 = fits.Column(name='E_Total_flux',
format='f8',
unit='mJy',
array=e_sint * 1000.0)
col13 = fits.Column(name='E_Total_flux_tot',
format='f8',
unit='mJy',
array=e_sint_tot * 1000.0)
#maj[np.where(sourceresolved=='U')] = np.nan
#e_maj[np.where(sourceresolved=='U')] = np.nan
#smin[np.where(sourceresolved=='U')] = np.nan
#e_smin[np.where(sourceresolved=='U')] = np.nan
#pa[np.where(sourceresolved=='U')] = np.nan
#e_pa[np.where(sourceresolved=='U')] = np.nan
col14 = fits.Column(name='Maj',
format='f8',
unit='arcsec',
array=maj * deg2arcsec)
col15 = fits.Column(name='E_Maj',
format='f8',
unit='arcsec',
array=e_maj * deg2arcsec)
col16 = fits.Column(name='Min',
format='f8',
unit='arcsec',
array=smin * deg2arcsec)
col17 = fits.Column(name='E_Min',
format='f8',
unit='arcsec',
array=e_smin * deg2arcsec)
col18 = fits.Column(name='DC_Maj',
format='f8',
unit='arcsec',
array=dcmaj * deg2arcsec)
col19 = fits.Column(name='E_DC_Maj',
format='f8',
unit='arcsec',
array=e_dcmaj * deg2arcsec)
col20 = fits.Column(name='DC_Min',
format='f8',
unit='arcsec',
array=dcsmin * deg2arcsec)
col21 = fits.Column(name='E_DC_Min',
format='f8',
unit='arcsec',
array=e_dcsmin * deg2arcsec)
col22 = fits.Column(name='PA', format='f8', unit='deg', array=pa)
col23 = fits.Column(name='E_PA', format='f8', unit='deg', array=e_pa)
col24 = fits.Column(name='DC_PA', format='f8', unit='deg', array=dcpa)
col25 = fits.Column(name='E_DC_PA', format='f8', unit='deg', array=e_dcpa)
#col20 = fits.Column(name='Resolved',format='1A',unit='',array=sourceresolved)
col26 = fits.Column(name='Isl_rms',
format='f8',
unit='beam-1 mJy',
array=rms_noise * 1000.0)
col27 = fits.Column(name='S_Code', format='1A', unit='', array=stype)
col28 = fits.Column(name='Mosaic_ID',
format='9A',
unit='',
array=mosaic_identifier)
col29 = fits.Column(name='Isl_id', format='I8', unit='', array=islid)
# With unique source names that are matched with source and gaussian catalogs the source_id is not needed.
#col24 = fits.Column(name='Source_id',format='I8',unit='',array=sourcenum)
if cattype == 'gaus':
gausid = np.append(gausid, cat[1].data[keepindices]['Gaus_id'])
col30 = fits.Column(name='Gaus_id', format='I8', unit='', array=gausid)
if cattype == 'srl':
cols = fits.ColDefs([
col1, col2, col3, col4, col5, col6, col7, col8, col9, col10, col11,
col12, col13, col14, col15, col16, col17, col18, col19, col20,
col21, col22, col23, col24, col25, col26, col27, col28, col29
])
if cattype == 'gaus':
cols = fits.ColDefs([
col1, col2, col3, col4, col5, col6, col7, col8, col9, col10, col11,
col12, col13, col14, col15, col16, col17, col18, col19, col20,
col21, col22, col23, col24, col25, col26, col27, col28, col29,
col30
])
tbhdu = fits.BinTableHDU.from_columns(cols)
if cattype == 'gaus':
regionfile = open('%s.gaus.reg' % outname, 'w')
if cattype == 'srl':
regionfile = open('%s.srl.reg' % outname, 'w')
regionfile.write('# Region file format: DS9 version 4.0\n')
regionfile.write(
'global color=green font="helvetica 10 normal" select=1 highlite=1 edit=1 move=1 delete=1 include=1 fixed=0 source\n'
)
regionfile.write('fk5\n')
for i in range(0, len(sourceids)):
if not np.isnan(maj[i]):
regionfile.write(
'ellipse(%s,%s,%s,%s,%s)\n' %
(sourcera[i], sourcedec[i], maj[i], smin[i], pa[i] + 90))
else:
regionfile.write('box(%s,%s,5.0",5.0",0.0)\n' %
(sourcera[i], sourcedec[i]))
regionfile.close()
prihdr = fits.Header()
prihdr['NAME'] = outname
prihdu = fits.PrimaryHDU(header=prihdr)
tbhdulist = fits.HDUList([prihdu, tbhdu])
if cattype == 'srl':
outcat = outname + '.srl.fits'
if cattype == 'gaus':
outcat = outname + '.gaus.fits'
tbhdulist.writeto(outcat)
return sourcenum, outcat
def separation(ra1, dec1, ra2, dec2):
return np.degrees(
sepn(np.radians(ra1), np.radians(dec1), np.radians(ra2),
np.radians(dec2)))
def plot_flux_errors(catalog,fitsimage,outname,auxcatname,options=None):
if options is None:
options = o
scat = Table.read(catalog)
fitsimage = fits.open(fitsimage)
fieldra = fitsimage[0].header['CRVAL1']
fielddec = fitsimage[0].header['CRVAL2']
fitsimage.close()
radialseps = sepn(scat['RA']*deg2rad,scat['DEC']*deg2rad,fieldra*deg2rad,fielddec*deg2rad)*rad2deg
pylab.plot(np.sort(scat['Total_flux']),np.sort(scat['Total_flux'])*np.median(np.array(scat[auxcatname+'_Total_flux']/options[auxcatname+'_fluxfactor'])/np.array(scat['Total_flux'])),'r--')
pylab.plot(scat['Total_flux'],scat[auxcatname+'_Total_flux']/options[auxcatname+'_fluxfactor'],'ko')
pylab.xlabel('Integrated LOFAR flux (Jy)')
pylab.ylabel('Integrated '+auxcatname+' flux (Jy)')
pylab.xlim(xmin=0.01,xmax=0.5)
pylab.ylim(ymin=0.01,ymax=0.5)
pylab.semilogx()
pylab.semilogy()
equality = np.arange(0,10,0.01)
pylab.plot(equality,equality,'k-')
pylab.savefig(outname.replace('.png','_integrated.png'))
pylab.close('all')
pylab.cla()
plt.clf()
pylab.plot(np.sort(scat['Peak_flux']),np.sort(scat['Peak_flux'])*np.median(np.array(scat[auxcatname+'_Peak_flux']/options['%s_fluxfactor'%auxcatname])/np.array(scat['Peak_flux'])),'r--')
pylab.plot(scat['Peak_flux'],scat[auxcatname+'_Peak_flux']/options[auxcatname+'_fluxfactor'],'ko')
pylab.xlabel('Peak LOFAR flux (Jy)')
pylab.ylabel('Peak '+auxcatname+' flux (Jy)')
pylab.xlim(xmin=0.01,xmax=0.5)
pylab.ylim(ymin=0.01,ymax=0.5)
pylab.semilogx()
pylab.semilogy()
equality = np.arange(0,10,0.01)
pylab.plot(equality,equality,'k-')
pylab.savefig(outname.replace('.png','_peak.png'))
pylab.close('all')
pylab.cla()
plt.clf()
pylab.plot(radialseps,scat['Total_flux']/(scat[auxcatname+'_Total_flux']/options[auxcatname+'_fluxfactor']),'bo',alpha=0.4,markersize=3)
equality = np.arange(-0.01,10,0.01)
fractionrange = np.arange(0.99,1.01,0.002)
for i in fractionrange:
pylab.plot(equality,1.0*i+0*equality,'k-')
pylab.plot(equality,1.0+0*equality,'k-')
pylab.xlabel('Distance from pointing centre (deg)')
pylab.ylabel('Integrated LOFAR flux / Integrated '+auxcatname+' flux')
pylab.xlim(xmin=0.0,xmax=2)
pylab.ylim(ymin=0.5,ymax=2.0)
distancerange = np.arange(0,np.max(radialseps)+0.15,0.15)
for i in range(0,len(distancerange)-1):
distancemin = distancerange[i]
distancemax = distancerange[i+1]
binvals = np.array([])
for j in range(0,len(radialseps)):
if distancemin < radialseps[j] < distancemax:
binvals = np.append(binvals,scat['Total_flux'][j]/(scat[auxcatname+'_Total_flux'][j]/options[auxcatname+'_fluxfactor']))
midpoint = (distancemin+distancemax)/2.0
if len(binvals) > 0.0:
booterrl,booterrh = bootstrap(binvals, 100000, np.mean, 0.05)
booterrl,booterrh = bootstrap(binvals, 100000, np.median, 0.05)
# print booterrl,booterrh, binvals
pylab.errorbar(midpoint,np.median(binvals),xerr=(distancemin-distancemax)/2.0,yerr=[[np.median(binvals)-booterrl],[booterrh-np.median(binvals)]],fmt='--o',ecolor='b',color='b',zorder=999999)
pylab.savefig(outname.replace('.png','_total_radial.png'))
pylab.close('all')
pylab.cla()
plt.clf()
def plot_flux_ratios(catalog,fitsimage,outname,options=None):
if options is None:
options = o
scat = Table.read(catalog)
fluxratios = scat['Total_flux']/scat['Peak_flux']
fitsimage = fits.open(fitsimage)
fieldra = fitsimage[0].header['CRVAL1']
fielddec = fitsimage[0].header['CRVAL2']
fitsimage.close()
radialseps = sepn(scat['RA']*deg2rad,scat['DEC']*deg2rad,fieldra*deg2rad,fielddec*deg2rad)*rad2deg
nullfmt = NullFormatter() # no labels
# definitions for the axes
left, width = 0.1, 0.65
bottom, height = 0.1, 0.8
bottom_h = left_h = left + width + 0.02
rect_scatter = [left, bottom, width, height]
rect_histx = [left, bottom_h, width, 0.2]
rect_histy = [left_h, bottom, 0.2, height]
# start with a rectangular Figure
plt.figure(1, figsize=(8, 8))
axScatter = plt.axes(rect_scatter)
plt.xlabel('Distance from pointing centre (deg)')
plt.ylabel('Integrated flux / Peak flux')
plt.xticks(rotation=270)
axScatter.plot([0,3.0],[1.0,1.0],'k-')
smids = []
svals = []
distancerange = np.arange(0,max(radialseps),0.3)
for i in range(0,len(distancerange)-1):
distancemin = distancerange[i]
distancemax = distancerange[i+1]
binvals = np.array([])
for j in range(0,len(radialseps)):
if distancemin < radialseps[j] < distancemax:
binvals = np.append(binvals,fluxratios[j])
midpoint = (distancemin+distancemax)/2.0
#bsmear = bandwidth_smearing2(6*arcsec2deg,150E6,midpoint,4*48.8E3)
#tsmear = time_smearing2(16.0,midpoint,6*arcsec2deg)
#smids.append(midpoint)
#svals.append(1.0/(bsmear*tsmear))
booterrl,booterrh = bootstrap(binvals, 100000, np.mean, 0.05)
booterrl,booterrh = bootstrap(binvals, 100000, np.median, 0.05)
axScatter.errorbar(midpoint,np.median(binvals),xerr=(distancemin-distancemax)/2.0,yerr=[[np.median(binvals)-booterrl],[booterrh-np.median(binvals)]],fmt='--o',ecolor='b',color='b',zorder=999999)
#axScatter.plot(smids,svals,'k--')
axHisty = plt.axes(rect_histy)
plt.xticks(rotation=270)
# no labels
axHisty.yaxis.set_major_formatter(nullfmt)
# the scatter plot:
axScatter.scatter(radialseps,fluxratios,color='blue',marker='+',s=10,alpha=0.5)
binwidth = 0.05
axScatter.set_xlim((0, 3.0))
axScatter.set_ylim((0.8, 2.5))
bins = np.arange(0.8, 3, binwidth)
axHisty.hist(fluxratios, bins=bins, orientation='horizontal',color='blue',alpha=0.5)
axHisty.set_ylim(axScatter.get_ylim())
plt.savefig(outname)
plt.close('all')
plt.cla()
plt.clf()
def separation(ra1, dec1, ra2, dec2):
# same as sepn but in degrees
return factor * sepn(ra1 / factor, dec1 / factor, ra2 / factor,
dec2 / factor)
def plot_flux_errors(catalog,fitsimage,outname,auxcatname,options=None):
if options is None:
options = o
if os.path.isfile(outname.replace('.png','_integrated.png')):
warn('Plot file %s exists, not making it' % outname)
else:
scat = Table.read(catalog)
fitsimage = fits.open(fitsimage)
fieldra = fitsimage[0].header['CRVAL1']
fielddec = fitsimage[0].header['CRVAL2']
fitsimage.close()
radialseps = sepn(scat['RA']*deg2rad,scat['DEC']*deg2rad,fieldra*deg2rad,fielddec*deg2rad)*rad2deg
plt.plot(np.sort(scat['Total_flux']),np.sort(scat['Total_flux'])*np.median(np.array(scat[auxcatname+'_Total_flux']/options[auxcatname+'_fluxfactor'])/np.array(scat['Total_flux'])),'r--')
plt.plot(scat['Total_flux'],scat[auxcatname+'_Total_flux']/options[auxcatname+'_fluxfactor'],'ko')
plt.xlabel('Integrated LOFAR flux (Jy)')
plt.ylabel('Integrated '+auxcatname+' flux (Jy)')
plt.xlim(xmin=0.01,xmax=0.5)
plt.ylim(ymin=0.01,ymax=0.5)
plt.semilogx()
plt.semilogy()
equality = np.arange(0,10,0.01)
plt.plot(equality,equality,'k-')
plt.savefig(outname.replace('.png','_integrated.png'))
plt.close('all')
plt.cla()
plt.clf()
plt.plot(np.sort(scat['Peak_flux']),np.sort(scat['Peak_flux'])*np.median(np.array(scat[auxcatname+'_Peak_flux']/options['%s_fluxfactor'%auxcatname])/np.array(scat['Peak_flux'])),'r--')
plt.plot(scat['Peak_flux'],scat[auxcatname+'_Peak_flux']/options[auxcatname+'_fluxfactor'],'ko')
plt.xlabel('Peak LOFAR flux (Jy)')
plt.ylabel('Peak '+auxcatname+' flux (Jy)')
plt.xlim(xmin=0.01,xmax=0.5)
plt.ylim(ymin=0.01,ymax=0.5)
plt.semilogx()
plt.semilogy()
equality = np.arange(0,10,0.01)
plt.plot(equality,equality,'k-')
plt.savefig(outname.replace('.png','_peak.png'))
plt.close('all')
plt.cla()
plt.clf()
plt.plot(radialseps,scat['Total_flux']/(scat[auxcatname+'_Total_flux']/options[auxcatname+'_fluxfactor']),'bo',alpha=0.4,markersize=3)
equality = np.arange(-0.01,10,0.01)
fractionrange = np.arange(0.99,1.01,0.002)
for i in fractionrange:
plt.plot(equality,1.0*i+0*equality,'k-')
plt.plot(equality,1.0+0*equality,'k-')
plt.xlabel('Distance from pointing centre (deg)')
plt.ylabel('Integrated LOFAR flux / Integrated '+auxcatname+' flux')
plt.xlim(xmin=0.0,xmax=2)
plt.ylim(ymin=0.5,ymax=2.0)
distancerange = np.arange(0,np.max(radialseps)+0.15,0.15)
for i in range(0,len(distancerange)-1):
distancemin = distancerange[i]
distancemax = distancerange[i+1]
binvals = np.array([])
for j in range(0,len(radialseps)):
if distancemin < radialseps[j] < distancemax:
binvals = np.append(binvals,scat['Total_flux'][j]/(scat[auxcatname+'_Total_flux'][j]/options[auxcatname+'_fluxfactor']))
midpoint = (distancemin+distancemax)/2.0
if len(binvals) > 0:
booterrl,booterrh = bootstrap(binvals, 100000, np.mean, 0.05)
booterrl,booterrh = bootstrap(binvals, 100000, np.median, 0.05)
# print booterrl,booterrh, binvals
plt.errorbar(midpoint,np.median(binvals),xerr=(distancemin-distancemax)/2.0,yerr=[[np.median(binvals)-booterrl],[booterrh-np.median(binvals)]],fmt='--o',ecolor='b',color='b',zorder=999999)
plt.savefig(outname.replace('.png','_total_radial.png'))
plt.close('all')
plt.cla()
plt.clf()
def plot_flux_ratios(catalog,fitsimage,outname,options=None):
if options is None:
options = o
if os.path.isfile(outname):
warn('Plot file %s exists, not making it' % outname)
else:
scat = Table.read(catalog)
fluxratios = scat['Total_flux']/scat['Peak_flux']
fitsimage = fits.open(fitsimage)
fieldra = fitsimage[0].header['CRVAL1']
fielddec = fitsimage[0].header['CRVAL2']
fitsimage.close()
radialseps = sepn(scat['RA']*deg2rad,scat['DEC']*deg2rad,fieldra*deg2rad,fielddec*deg2rad)*rad2deg
nullfmt = NullFormatter() # no labels
# definitions for the axes
left, width = 0.1, 0.65
bottom, height = 0.1, 0.8
bottom_h = left_h = left + width + 0.02
rect_scatter = [left, bottom, width, height]
rect_histx = [left, bottom_h, width, 0.2]
rect_histy = [left_h, bottom, 0.2, height]
# start with a rectangular Figure
plt.figure(1, figsize=(8, 8))
axScatter = plt.axes(rect_scatter)
plt.xlabel('Distance from pointing centre (deg)')
plt.ylabel('Integrated flux / Peak flux')
plt.xticks(rotation=270)
axScatter.plot([0,3.0],[1.0,1.0],'k-')
smids = []
svals = []
distancerange = np.arange(0,max(radialseps),0.3)
for i in range(0,len(distancerange)-1):
distancemin = distancerange[i]
distancemax = distancerange[i+1]
binvals = np.array([])
for j in range(0,len(radialseps)):
if distancemin < radialseps[j] < distancemax:
binvals = np.append(binvals,fluxratios[j])
midpoint = (distancemin+distancemax)/2.0
print i,binvals
#bsmear = bandwidth_smearing2(6*arcsec2deg,150E6,midpoint,4*48.8E3)
#tsmear = time_smearing2(16.0,midpoint,6*arcsec2deg)
#smids.append(midpoint)
#svals.append(1.0/(bsmear*tsmear))
#booterrl,booterrh = bootstrap(binvals, 100000, np.mean, 0.05)
if len(binvals)>0:
booterrl,booterrh = bootstrap(binvals, 100000, np.median, 0.05)
axScatter.errorbar(midpoint,np.median(binvals),xerr=(distancemin-distancemax)/2.0,yerr=[[np.median(binvals)-booterrl],[booterrh-np.median(binvals)]],fmt='--o',ecolor='b',color='b',zorder=999999)
#axScatter.plot(smids,svals,'k--')
axHisty = plt.axes(rect_histy)
plt.xticks(rotation=270)
# no labels
axHisty.yaxis.set_major_formatter(nullfmt)
# the scatter plot:
axScatter.scatter(radialseps,fluxratios,color='blue',marker='+',s=10,alpha=0.5)
binwidth = 0.05
axScatter.set_xlim((0, 3.0))
axScatter.set_ylim((0.8, 2.5))
bins = np.arange(0.8, 3, binwidth)
axHisty.hist(fluxratios, bins=bins, orientation='horizontal',color='blue',alpha=0.5)
axHisty.set_ylim(axScatter.get_ylim())
plt.savefig(outname)
plt.close('all')
plt.cla()
plt.clf()
id.append(r['id'])
'''
for l in open('/home/mjh/pipeline-master/ddf-pipeline/misc/DR2-pointings.txt').readlines():
bits=l.rstrip().split()
id.append(bits[0])
ra.append(float(bits[1]))
dec.append(float(bits[2]))
'''
ra = np.array(ra) * deg2rad
dec = np.array(dec) * deg2rad
r_ra = 270 * deg2rad
r_dec = 45 * deg2rad
sep = sepn(ra, dec, r_ra, r_dec)
i = np.argmin(sep)
print(i, id[i], sep[i] * rad2deg)
# mosaic distance is 2.2 deg
limit = 2.2 * deg2rad
area = 0
totarea = 0
ras = np.linspace(0, 360, 700) * deg2rad
decs = np.linspace(0, 90, 700) * deg2rad
for d in decs:
a = np.cos(d)
print('.', end=' ')
for r in ras:
sep = sepn(ra, dec, r, d)
def filter_catalogs(args,pointingras,pointingdecs,mosaiccat,outname,dessourcenums,cattype):
pointdirectories = args.pointdirectories
sourceids = np.array([])
sourceresolved = np.array([])
sourcera = np.array([])
e_sourcera = np.array([])
e_sourcera_tot = np.array([])
sourcedec = np.array([])
e_sourcedec = np.array([])
e_sourcedec_tot = np.array([])
sint = np.array([])
e_sint = np.array([])
e_sint_tot = np.array([])
speak = np.array([])
e_speak = np.array([])
e_speak_tot = np.array([])
maj = np.array([])
e_maj = np.array([])
smin = np.array([])
e_smin = np.array([])
dcmaj = np.array([])
e_dcmaj = np.array([])
dcsmin = np.array([])
e_dcsmin = np.array([])
pa = np.array([])
e_pa = np.array([])
dcpa = np.array([])
e_dcpa = np.array([])
rms_noise = np.array([])
stype = np.array([])
mosaic_identifier = np.array([])
gausid = np.array([])
islid = np.array([])
sourcenum = np.array([])
pointing = mosaiccat.replace('.cat.fits','-blanked.fits')
if cattype == 'gaus':
sourcecat = fits.open(mosaiccat)
mosaiccat = glob.glob(mosaiccat.replace('mosaic.cat.fits','mosaic-blanked_pybdsm/*/catalogues/mosaic-blanked.pybdsm.gaul.FITS'))[0]
cat = fits.open(mosaiccat)
f = fits.open(pointing)
rapointing = f[0].header['CRVAL1']*deg2rad
decpointing = f[0].header['CRVAL2']*deg2rad
f.close()
numsources = len(cat[1].data['RA'])
closepointingindex = np.where(sepn(pointingras,pointingdecs,rapointing,decpointing)*rad2deg < 5.0)
astromed = find_median_astrometry(pointdirectories,rapointing,decpointing)
keepindices = []
time1 = time.time()
for i in range(0,numsources):
if dessourcenums == None:
allsep = sepn(pointingras[closepointingindex],pointingdecs[closepointingindex],cat[1].data['RA'][i]*deg2rad,cat[1].data['DEC'][i]*deg2rad)
centsep = sepn(rapointing,decpointing,cat[1].data['RA'][i]*deg2rad,cat[1].data['DEC'][i]*deg2rad)
if min(allsep) != centsep:
continue
else:
keepindices.append(i)
else:
if cat[1].data['Source_id'][i] in dessourcenums:
keepindices.append(i)
else:
continue
if cattype == 'srl':
sc=SkyCoord(cat[1].data['RA'][i]*deg2rad*u.rad,cat[1].data['DEC'][i]*deg2rad*u.rad,frame='icrs')
if cattype == 'gaus':
sourceindex=cat[1].data['Source_id'][i]
sc=SkyCoord(sourcecat[1].data['RA'][sourceindex]*deg2rad*u.rad,sourcecat[1].data['DEC'][sourceindex]*deg2rad*u.rad,frame='icrs')
s=sc.to_string(style='hmsdms',sep='',precision=3)
s=sc.to_string(style='hmsdms',sep='',precision=2)
identity = str('ILTJ'+s).replace(' ','')[:-1]
sourceids = np.append(sourceids,identity)
if cattype == 'srl':
mosaic_identifier = np.append(mosaic_identifier,mosaiccat.split('/')[-2])
if cattype == 'gaus':
mosaic_identifier = np.append(mosaic_identifier,mosaiccat.split('/')[-5])
fluxratio = cat[1].data['Total_flux'][i]/cat[1].data['Peak_flux'][i]
snr = cat[1].data['Peak_flux'][i]/cat[1].data['Isl_rms'][i]
# Some equation to figure out if the source is resolved -- leave these dummy values for now.
if fluxratio > (1.483 + 1000.4/(snr**3.94)):
#if fluxratio > (1.50341355 + 1.78467767/(snr**0.78385826)):
sourceresolved = np.append(sourceresolved,'R')
else:
sourceresolved = np.append(sourceresolved,'U')
print 'Keeping %s sources for %s -- took %s'%(len(keepindices),pointing,time.time()-time1)
sourcera = np.append(sourcera,cat[1].data[keepindices]['RA'])
e_sourcera = np.append(e_sourcera,cat[1].data[keepindices]['E_RA'])
e_sourcera_tot = np.append(e_sourcera_tot,(cat[1].data[keepindices]['E_RA']**2.0 + (astromed*arcsec2deg)**2.0)**0.5) #$ ADD SOME ERROR TO THE SOURCE POSITIONS
sourcedec = np.append(sourcedec,cat[1].data[keepindices]['DEC'])
e_sourcedec = np.append(e_sourcedec,cat[1].data[keepindices]['E_DEC'])
e_sourcedec_tot = np.append(e_sourcedec_tot,(cat[1].data[keepindices]['E_DEC']**2.0 + (astromed*arcsec2deg)**2.0)**0.5) # ADD SOME ERROR TO THE SOURCE POSITIONS
sint = np.append(sint,cat[1].data[keepindices]['Total_flux'])
e_sint =np.append(e_sint,cat[1].data[keepindices]['E_Total_flux'])
e_sint_tot =np.append(e_sint_tot,(cat[1].data[keepindices]['E_Total_flux']**2.0 + (cat[1].data[keepindices]['Total_flux']*0.2)**2.0)**0.5)
speak = np.append(speak,cat[1].data[keepindices]['Peak_flux'])
e_speak =np.append(e_speak,cat[1].data[keepindices]['E_Peak_flux'])
e_speak_tot =np.append(e_speak_tot,(cat[1].data[keepindices]['E_Peak_flux']**2.0 + (cat[1].data[keepindices]['Peak_flux']*0.2)**2.0)**0.5)
maj = np.append(maj,cat[1].data[keepindices]['Maj'])
e_maj =np.append(e_maj,cat[1].data[keepindices]['E_Maj'])
smin = np.append(smin,cat[1].data[keepindices]['Min'])
e_smin = np.append(e_smin,cat[1].data[keepindices]['E_Min'])
dcmaj = np.append(dcmaj,cat[1].data[keepindices]['DC_Maj'])
e_dcmaj =np.append(e_dcmaj,cat[1].data[keepindices]['E_DC_Maj'])
dcsmin = np.append(dcsmin,cat[1].data[keepindices]['DC_Min'])
e_dcsmin = np.append(e_dcsmin,cat[1].data[keepindices]['E_DC_Min'])
pa = np.append(pa,cat[1].data[keepindices]['PA'])
e_pa = np.append(e_pa,cat[1].data[keepindices]['E_PA'])
dcpa = np.append(dcpa,cat[1].data[keepindices]['DC_PA'])
e_dcpa = np.append(e_dcpa,cat[1].data[keepindices]['E_DC_PA'])
rms_noise = np.append(rms_noise,cat[1].data[keepindices]['Isl_rms'])
stype = np.append(stype,cat[1].data[keepindices]['S_Code'])
islid = np.append(islid,cat[1].data[keepindices]['Isl_id'])
sourcenum = np.append(sourcenum,cat[1].data[keepindices]['Source_id'])
col1 = fits.Column(name='Source_Name',format='24A',unit='',array=sourceids)
col2 = fits.Column(name='RA',format='f8',unit='deg',array=sourcera)
col3 = fits.Column(name='E_RA',format='f8',unit='arcsec',array=e_sourcera*deg2arcsec)
col4 = fits.Column(name='E_RA_tot',format='f8',unit='arcsec',array=e_sourcera_tot*deg2arcsec)
col5 = fits.Column(name='DEC',format='f8',unit='deg',array=sourcedec)
col6 = fits.Column(name='E_DEC',format='f8',unit='arcsec',array=e_sourcedec*deg2arcsec)
col7 = fits.Column(name='E_DEC_tot',format='f8',unit='arcsec',array=e_sourcedec_tot*deg2arcsec)
col8 = fits.Column(name='Peak_flux',format='f8',unit='beam-1 mJy',array=speak*1000.0)
col9 = fits.Column(name='E_Peak_flux',format='f8',unit='beam-1 mJy',array=e_speak*1000.0)
col10 = fits.Column(name='E_Peak_flux_tot',format='f8',unit='beam-1 mJy',array=e_speak_tot*1000.0)
col11 = fits.Column(name='Total_flux',format='f8',unit='mJy',array=sint*1000.0)
col12 = fits.Column(name='E_Total_flux',format='f8',unit='mJy',array=e_sint*1000.0)
col13 = fits.Column(name='E_Total_flux_tot',format='f8',unit='mJy',array=e_sint_tot*1000.0)
#maj[np.where(sourceresolved=='U')] = np.nan
#e_maj[np.where(sourceresolved=='U')] = np.nan
#smin[np.where(sourceresolved=='U')] = np.nan
#e_smin[np.where(sourceresolved=='U')] = np.nan
#pa[np.where(sourceresolved=='U')] = np.nan
#e_pa[np.where(sourceresolved=='U')] = np.nan
col14 = fits.Column(name='Maj',format='f8',unit='arcsec',array=maj*deg2arcsec)
col15 = fits.Column(name='E_Maj',format='f8',unit='arcsec',array=e_maj*deg2arcsec)
col16 = fits.Column(name='Min',format='f8',unit='arcsec',array=smin*deg2arcsec)
col17 = fits.Column(name='E_Min',format='f8',unit='arcsec',array=e_smin*deg2arcsec)
col18 = fits.Column(name='DC_Maj',format='f8',unit='arcsec',array=dcmaj*deg2arcsec)
col19 = fits.Column(name='E_DC_Maj',format='f8',unit='arcsec',array=e_dcmaj*deg2arcsec)
col20 = fits.Column(name='DC_Min',format='f8',unit='arcsec',array=dcsmin*deg2arcsec)
col21 = fits.Column(name='E_DC_Min',format='f8',unit='arcsec',array=e_dcsmin*deg2arcsec)
col22 = fits.Column(name='PA',format='f8',unit='deg',array=pa)
col23 = fits.Column(name='E_PA',format='f8',unit='deg',array=e_pa)
col24 = fits.Column(name='DC_PA',format='f8',unit='deg',array=dcpa)
col25 = fits.Column(name='E_DC_PA',format='f8',unit='deg',array=e_dcpa)
#col20 = fits.Column(name='Resolved',format='1A',unit='',array=sourceresolved)
col26 = fits.Column(name='Isl_rms',format='f8',unit='beam-1 mJy',array=rms_noise*1000.0)
col27 = fits.Column(name='S_Code',format='1A',unit='',array=stype)
col28 = fits.Column(name='Mosaic_ID',format='9A',unit='',array=mosaic_identifier)
col29 = fits.Column(name='Isl_id',format='I8',unit='',array=islid)
# With unique source names that are matched with source and gaussian catalogs the source_id is not needed.
#col24 = fits.Column(name='Source_id',format='I8',unit='',array=sourcenum)
if cattype == 'gaus':
gausid = np.append(gausid,cat[1].data[keepindices]['Gaus_id'])
col30 = fits.Column(name='Gaus_id',format='I8',unit='',array=gausid)
if cattype == 'srl':
cols = fits.ColDefs([col1,col2,col3,col4,col5,col6,col7,col8,col9,col10,col11,col12,col13,col14,col15,col16,col17,col18,col19,col20,col21,col22,col23,col24,col25,col26,col27,col28,col29])
if cattype == 'gaus':
cols = fits.ColDefs([col1,col2,col3,col4,col5,col6,col7,col8,col9,col10,col11,col12,col13,col14,col15,col16,col17,col18,col19,col20,col21,col22,col23,col24,col25,col26,col27,col28,col29,col30])
tbhdu = fits.BinTableHDU.from_columns(cols)
if cattype == 'gaus':
regionfile = open('%s.gaus.reg'%outname,'w')
if cattype == 'srl':
regionfile = open('%s.srl.reg'%outname,'w')
regionfile.write('# Region file format: DS9 version 4.0\n')
regionfile.write('global color=green font="helvetica 10 normal" select=1 highlite=1 edit=1 move=1 delete=1 include=1 fixed=0 source\n')
regionfile.write('fk5\n')
for i in range(0,len(sourceids)):
if not np.isnan(maj[i]):
regionfile.write('ellipse(%s,%s,%s,%s,%s)\n'%(sourcera[i],sourcedec[i],maj[i],smin[i],pa[i]+90))
else:
regionfile.write('box(%s,%s,5.0",5.0",0.0)\n'%(sourcera[i],sourcedec[i]))
regionfile.close()
prihdr = fits.Header()
prihdr['NAME'] = outname
prihdu = fits.PrimaryHDU(header=prihdr)
tbhdulist = fits.HDUList([prihdu, tbhdu])
if cattype == 'srl':
outcat = outname +'.srl.fits'
if cattype == 'gaus':
outcat = outname +'.gaus.fits'
tbhdulist.writeto(outcat)
return sourcenum,outcat
