def do_archive(o, archivelist):
if o['mslist'] is None:
die('MS list must be specified')
with open(o['mslist'], 'r') as f:
msnames = [l.strip() for l in f.readlines()]
# find version number in case this has been run already
prefix = 'old'
count = 1
while os.path.isfile(msnames[0] + '/' + prefix + '.killms_p1.sols.npz'):
count += 1
prefix = 'old%i' % count
print 'prefix is', prefix
for ms in msnames:
for s in ['p1', 'ap1']:
if s in archivelist:
for file in ['npz', 'parset']:
rename(
ms + '/killMS.killms_' + s + '.sols.' + file,
ms + '/' + prefix + '.killms_' + s + '.sols.' + file)
if o['full_mslist'] is not None:
with open(o['full_mslist'], 'r') as f:
msnames = [l.strip() for l in f.readlines()]
for ms in msnames:
for s in ['f_ap1', 'f_ap2']:
if s in archivelist:
for file in ['npz', 'parset']:
rename(
ms + '/killMS.killms_' + s + '.sols.' + file, ms +
'/' + prefix + '.killms_' + s + '.sols.' + file)
def main(msin, config_path):
o = options(config_path, option_list)
if o['pbimage'] is None:
die('pbimage must be specified')
# fix up the new list-type options
for i, cat in enumerate(o['list']):
try:
o[cat] = o['filenames'][i]
except:
pass
try:
o[cat + '_matchrad'] = o['radii'][i]
except:
pass
try:
o[cat + '_fluxfactor'] = o['fluxfactor'][i]
except:
pass
if o['logging'] is not None and not os.path.isdir(o['logging']):
os.mkdir(o['logging'])
# pybdsm source finding
sfind_image(o['catprefix'], o['pbimage'], o['sfind_pixel_fraction'])
# matching with catalogs
for cat in o['list']:
print 'Doing catalogue', cat
crossmatch_image(o['catprefix'] + '.cat.fits', cat)
filter_catalog(o['catprefix'] + '.cat.fits',
o['catprefix'] + '.cat.fits_' + cat + '_match.fits',
o['pbimage'],
o['catprefix'] + '.cat.fits_' + cat +
'_match_filtered.fits',
cat,
options=o)
# Filter catalogs (only keep isolated compact sources within 3deg of pointing centre)
# Astrometric plots
if 'FIRST' in o['list']:
report('Plotting position offsets')
plot_position_offset(
'%s.cat.fits_FIRST_match_filtered.fits' % o['catprefix'],
o['pbimage'],
'%s.cat.fits_FIRST_match_filtered_positions.png' % o['catprefix'],
'FIRST',
options=o)
t = Table.read(o['catprefix'] + '.cat.fits_FIRST_match_filtered.fits')
bsra = np.percentile(bootstrap(t['FIRST_dRA'], np.mean, 10000),
(16, 84))
bsdec = np.percentile(bootstrap(t['FIRST_dDEC'], np.mean, 10000),
(16, 84))
mdra = np.mean(t['FIRST_dRA'])
mddec = np.mean(t['FIRST_dDEC'])
print 'Mean delta RA is %.3f arcsec (1-sigma %.3f -- %.3f arcsec)' % (
mdra, bsra[0], bsra[1])
print 'Mean delta DEC is %.3f arcsec (1-sigma %.3f -- %.3f arcsec)' % (
mddec, bsdec[0], bsdec[1])
report('Plotting flux ratios')
# Flux ratio plots (only compact sources)
plot_flux_ratios(
'%s.cat.fits_FIRST_match_filtered.fits' % o['catprefix'],
o['pbimage'],
'%s.cat.fits_FIRST_match_filtered_fluxerrors.png' % o['catprefix'],
options=o)
report('Plotting flux scale comparison')
# Flux scale comparison plots
if 'TGSS' in o['list']:
plot_flux_errors(
'%s.cat.fits_TGSS_match_filtered.fits' % o['catprefix'],
o['pbimage'],
'%s.cat.fits_TGSS_match_filtered_fluxratio.png' % o['catprefix'],
'TGSS',
options=o)
t = Table.read(o['catprefix'] + '.cat.fits_TGSS_match_filtered.fits')
ratios = t['Total_flux'] / (t['TGSS_Total_flux'] /
o['TGSS_fluxfactor'])
bsratio = np.percentile(bootstrap(ratios, np.median, 10000), (16, 84))
print 'Median LOFAR/TGSS ratio is %.3f (1-sigma %.3f -- %.3f)' % (
np.median(ratios), bsratio[0], bsratio[1])
if 'NVSS' in o['list']:
t = Table.read(o['catprefix'] + '.cat.fits_NVSS_match_filtered.fits')
t = t[t['Total_flux'] > 10e-3]
ratios = t['Total_flux'] / t['NVSS_Total_flux']
bsratio = np.percentile(bootstrap(ratios, np.median, 10000), (16, 84))
print 'Median LOFAR/NVSS ratio is %.3f (1-sigma %.3f -- %.3f)' % (
np.median(ratios), bsratio[0], bsratio[1])
# Noise estimate
hdu = fits.open(o['pbimage'])
imagenoise = get_rms(hdu)
print 'An estimate of the image noise is %.3f muJy/beam' % (imagenoise *
1E6)
return 0
if 'Facet' not in t.columns:
r.add_facet_labels(t)
plot_offsets(t,r.clist,'red')
if savefig is not None:
plt.savefig(savefig)
if __name__=='__main__':
# Main loop
if len(sys.argv)<2:
warn('quality_pipeline.py must be called with at least one parameter file\nor a command-line option list.\nE.g "pipeline.py example.cfg second_example.cfg --solutions-robust=0.1"\nSee below for a complete list of possible options with their default values.')
print_options(option_list)
sys.exit(1)
o=options(sys.argv[1:],option_list)
if o['pbimage'] is None:
die('pbimage must be specified')
if o['nonpbimage'] is None:
die('nonpbimage must be specified')
if o['list'] is not None:
# fix up the new list-type options
for i,cat in enumerate(o['list']):
try:
o[cat]=o['filenames'][i]
except:
pass
try:
o[cat+'_matchrad']=o['radii'][i]
except:
pass
try:
o[cat+'_fluxfactor']=o['fluxfactor'][i]
name = sys.argv[1]
try:
qsubfile = sys.argv[2]
except:
qsubfile = '/home/mjh/git/ddf-pipeline/pipeline.qsub'
try:
os.mkdir(name)
except OSError:
warn('Working directory already exists')
pass
os.chdir(name)
report('Downloading data')
if not download_dataset('https://lofar-webdav.grid.sara.nl',
'/SKSP/' + name + '/'):
die('Download failed to get the right number of files')
report('Unpacking data')
unpack()
report('Deleting tar files')
os.system('rm *.tar.gz')
report('Making ms lists')
if make_list():
report('Submit job')
os.system('qsub -N ddfp-' + name + ' -v WD=' + rootdir + '/' + name + ' ' +
qsubfile)
else:
die('make_list could not construct the MS list')
if savefig is not None:
plt.savefig(savefig)
if __name__ == '__main__':
# Main loop
if len(sys.argv) < 2:
warn(
'quality_pipeline.py must be called with at least one parameter file\nor a command-line option list.\nE.g "pipeline.py example.cfg second_example.cfg --solutions-robust=0.1"\nSee below for a complete list of possible options with their default values.'
)
print_options(option_list)
sys.exit(1)
o = options(sys.argv[1:], option_list)
if o['pbimage'] is None:
die('pbimage must be specified')
if o['nonpbimage'] is None:
die('nonpbimage must be specified')
if o['list'] is not None:
# fix up the new list-type options
for i, cat in enumerate(o['list']):
try:
o[cat] = o['filenames'][i]
except:
pass
try:
o[cat + '_matchrad'] = o['radii'][i]
except:
pass
try:
o[cat + '_fluxfactor'] = o['fluxfactor'][i]
def do_run_pipeline(name, basedir):
if name[0] != 'P' and name[0] != 'L':
die('This code should be used only with field or observation names',
database=False)
do_field = (name[0] == 'P')
try:
qsubfile = sys.argv[2]
except:
qsubfile = '/home/mjh/pipeline-master/ddf-pipeline/torque/pipeline.qsub'
workdir = basedir + '/' + name
try:
os.mkdir(workdir)
except OSError:
warn('Working directory already exists')
report('Downloading data')
if do_field:
success = download_field(name, basedir=basedir)
else:
success = download_dataset('https://lofar-webdav.grid.sara.nl',
'/SKSP/' + name + '/',
basedir=basedir)
if not success:
die('Download failed, see earlier errors', database=False)
report('Unpacking data')
try:
unpack(workdir=workdir)
except RuntimeError:
if do_field:
update_status(name, 'List failed', workdir=workdir)
raise
if do_field:
update_status(name, 'Unpacked', workdir=workdir)
report('Deleting tar files')
os.system('rm ' + workdir + '/*.tar.gz')
os.system('rm ' + workdir + '/*.tar')
averaged = False
report('Checking structure')
g = glob.glob(workdir + '/*.ms')
msl = MSList(None, mss=g)
uobsids = set(msl.obsids)
for thisobs in uobsids:
# check one MS with each ID
for m, ch, o in zip(msl.mss, msl.channels, msl.obsids):
if o == thisobs:
channels = len(ch)
print 'MS', m, 'has', channels, 'channels'
if channels > 20:
update_status(name, 'Averaging', workdir=workdir)
print 'Averaging needed for', thisobs, '!'
averaged = True
average(wildcard=workdir + '/*' + thisobs + '*')
os.system('rm -r ' + workdir + '/*' + thisobs +
'*pre-cal.ms')
break
report('Making ms lists')
success = make_list(workdir=workdir)
if do_field:
list_db_update(success, workdir=workdir)
if not success:
die('make_list could not construct the MS list', database=False)
report('Creating custom config file from template')
make_custom_config(name, workdir, do_field, averaged)
# now run the job
do_run_job(name, basedir=basedir, qsubfile=None, do_field=do_field)
def make_mosaic(args):
if args.scale is not None:
if len(args.scale) != len(args.directories):
die('Scales provided must match directories')
if args.noise is not None:
if len(args.noise) != len(args.directories):
die('Noises provided must match directories')
if args.rootname:
rootname=args.rootname+'-'
else:
rootname=''
if args.exact:
reproj=reproject_exact
else:
reproj=reproject_interp_chunk_2d
if args.do_lowres:
intname='image_full_low_m.int.restored.fits'
appname='image_full_low_m.app.restored.fits'
elif args.use_shifted:
intname='image_full_ampphase_di_m.NS_shift.int.facetRestored.fits'
appname='image_full_ampphase_di_m.NS_shift.app.facetRestored.fits'
else:
intname='image_full_ampphase_di_m.NS.int.restored.fits'
appname='image_full_ampphase_di_m.NS.app.restored.fits'
# astromap blanking if required
bth=None
try:
bth=float(args.astromap_blank)
except:
pass
threshold=float(args.beamcut)
hdus=[]
app=[]
astromaps=[]
wcs=[]
print 'Reading files...'
noise=[]
name=[]
for d in args.directories:
name.append(d.split('/')[-1])
hdu=fits.open(d+'/'+intname)
if args.find_noise:
print 'Estimating noise for', d+'/' + intname
if args.do_lowres:
noise.append(get_rms(hdu,boxsize=1500))
else:
noise.append(get_rms(hdu))
hdus.append(flatten(hdu))
app.append(flatten(fits.open(d+'/'+appname)))
if bth:
astromaps.append(flatten(fits.open(d+'/astromap.fits')))
if args.find_noise:
args.noise=noise
print 'Noise values are:'
for t,n in zip(name,noise):
print t,n
print 'Computing noise/beam factors...'
for i in range(len(app)):
np.seterr(divide='ignore')
app[i].data=np.divide(app[i].data,hdus[i].data)
app[i].data[app[i].data<threshold]=0
# at this point this is the beam factor: we want 1/sigma**2.0, so divide by central noise and square
if args.noise is not None:
if args.scale is not None:
app[i].data/=args.noise[i]*args.scale[i]
else:
app[i].data/=args.noise[i]
app[i].data=app[i].data**2.0
if args.scale is not None:
hdus[i].data*=args.scale[i]
if args.shift:
print 'Finding shifts (NOTE THIS CODE IS OBSOLETE)...'
# shift according to the FIRST delta ra/dec from quality pipeline
dras=[]
ddecs=[]
for d in args.directories:
t=Table.read(d+'/image_full_ampphase1m.cat.fits_FIRST_match_filtered.fits')
dras.append(np.mean(t['FIRST_dRA']))
ddecs.append(np.mean(t['FIRST_dDEC']))
print 'Applying shifts:',dras,ddecs
for i in range(len(app)):
for hdu in [hdus[i],app[i]]:
ra=hdu.header['CRVAL1']
dec=hdu.header['CRVAL2']
hdu.header['CRVAL1']-=dras[i]/(3600.0*np.cos(np.pi*dec/180.0))
hdu.header['CRVAL2']-=ddecs[i]/3600.0
for i in range(len(app)):
wcs.append(WCS(hdus[i].header))
# astromap blanking
if bth:
print 'Blanking using astrometry quality maps with threshold',bth,'arcsec'
for i in range(len(app)):
outname=rootname+'astroblank-'+name[i]+'.fits'
if args.load and os.path.isfile(outname):
print 'Loading previously blanked image'
hdu=fits.open(outname)
hdus[i].data=hdu[0].data
else:
print 'Blanking image',i
dmaxy,dmaxx=hdus[i].data.shape
count=0
am=astromaps[i]
awcs=WCS(am.header)
maxy,maxx=am.data.shape
for y in range(maxy):
for x in range(maxx):
value=am.data[y,x]
if np.isnan(value):
if y<maxy-1:
value=am.data[y+1,x]
if value>bth:
ra,dec=[float(f) for f in awcs.wcs_pix2world(x,y,0)]
rx,ry=[int(p) for p in wcs[i].wcs_world2pix(ra,dec,0)]
rxp=rx+21 # astromap pix size, with margin
ryp=ry+21
if rx<0: rx=0
if ry<0: ry=0
if rxp>dmaxx: rxp=dmaxx
if ryp>dmaxy: ryp=dmaxy
hdus[i].data[ry:ryp,rx:rxp]=np.nan
count+=1
print '... blanked',count*900.0/3600,'square arcmin'
outname=rootname+'astroblank-'+name[i]+'.fits'
if args.save: hdus[i].writeto(outname,clobber=True)
app[i].data[np.isnan(hdus[i].data)]=np.nan
# If the header is directly passed in, use it
try:
header=args.header
xsize=header['NAXIS1']
ysize=header['NAXIS2']
print 'Mosaic using header passed from calling program'
except:
header=None
if header is None:
if args.load_layout:
with open(rootname+'mosaic-header.pickle') as f:
header=pickle.load(f)
xsize=header['NAXIS1']
ysize=header['NAXIS2']
print 'Mosaic using loaded header'
else:
print 'Creating the mosaic header'
ras=np.array([w.wcs.crval[0] for w in wcs])
decs=np.array([w.wcs.crval[1] for w in wcs])
mra=np.mean(ras)
mdec=np.mean(decs)
print 'Will make mosaic at',mra,mdec
# we make a reference WCS and use it to find the extent in pixels
# needed for the combined image
rwcs=WCS(naxis=2)
rwcs.wcs.ctype=wcs[0].wcs.ctype
rwcs.wcs.cdelt=wcs[0].wcs.cdelt
rwcs.wcs.crval=[mra,mdec]
rwcs.wcs.crpix=[1,1]
xmin=0
xmax=0
ymin=0
ymax=0
for a,w in zip(app,wcs):
ys,xs=np.where(a.data)
axmin=xs.min()
aymin=ys.min()
axmax=xs.max()
aymax=ys.max()
del(xs)
del(ys)
print 'non-zero',axmin,aymin,axmax,aymax
for x,y in ((axmin,aymin),(axmax,aymin),(axmin,aymax),(axmax,aymax)):
ra,dec=[float(f) for f in w.wcs_pix2world(x,y,0)]
#print ra,dec
nx,ny=[float (f) for f in rwcs.wcs_world2pix(ra,dec,0)]
print nx,ny
if nx<xmin: xmin=nx
if nx>xmax: xmax=nx
if ny<ymin: ymin=ny
if ny>ymax: ymax=ny
print 'co-ord range:', xmin, xmax, ymin, ymax
xsize=int(xmax-xmin)
ysize=int(ymax-ymin)
rwcs.wcs.crpix=[-int(xmin)+1,-int(ymin)+1]
print 'checking:', rwcs.wcs_world2pix(mra,mdec,0)
print rwcs
header=rwcs.to_header()
header['NAXIS']=2
header['NAXIS1']=xsize
header['NAXIS2']=ysize
with open(rootname+'mosaic-header.pickle','w') as f:
pickle.dump(header,f)
isum=np.zeros([ysize,xsize])
wsum=np.zeros_like(isum)
mask=np.zeros_like(isum,dtype=np.bool)
print 'now making the mosaic'
for i in range(len(hdus)):
print 'image',i,'(',name[i],')'
outname=rootname+'reproject-'+name[i]+'.fits'
if args.load and os.path.exists(outname):
print 'loading...'
hdu=fits.open(outname)
r=hdu[0].data
else:
print 'reprojecting...'
r, footprint = reproj(hdus[i], header, hdu_in=0, parallel=False)
r[np.isnan(r)]=0
hdu = fits.PrimaryHDU(header=header,data=r)
if args.save: hdu.writeto(outname,clobber=True)
print 'weights',i,'(',name[i],')'
outname=rootname+'weight-'+name[i]+'.fits'
if args.load and os.path.exists(outname):
print 'loading...'
hdu=fits.open(outname)
w=hdu[0].data
mask|=(w>0)
else:
print 'reprojecting...'
w, footprint = reproj(app[i], header, hdu_in=0, parallel=False)
mask|=~np.isnan(w)
w[np.isnan(w)]=0
hdu = fits.PrimaryHDU(header=header,data=w)
if args.save: hdu.writeto(outname,clobber=True)
print 'add to mosaic...'
isum+=r*w
wsum+=w
if not(args.no_write):
isum/=wsum
# mask now contains True where a non-nan region was present in either map
isum[~mask]=np.nan
for ch in ('BMAJ', 'BMIN', 'BPA'):
header[ch]=hdus[0].header[ch]
header['ORIGIN']='ddf-pipeline '+version()
hdu = fits.PrimaryHDU(header=header,data=isum)
hdu.writeto(rootname+'mosaic.fits',clobber=True)
hdu = fits.PrimaryHDU(header=header,data=wsum)
hdu.writeto(rootname+'mosaic-weights.fits',clobber=True)
def make_mosaic(args):
if args.scale is not None:
if len(args.scale) != len(args.directories):
die('Scales provided must match directories')
if args.noise is not None:
if len(args.noise) != len(args.directories):
die('Noises provided must match directories')
if args.rootname:
rootname = args.rootname + '-'
else:
rootname = ''
if args.exact:
reproj = reproject_exact
else:
reproj = reproject_interp_chunk_2d
if args.do_lowres:
intname = 'image_full_low_m.int.restored.fits'
appname = 'image_full_low_m.app.restored.fits'
elif args.use_shifted:
intname = 'image_full_ampphase_di_m.NS_shift.int.facetRestored.fits'
appname = 'image_full_ampphase_di_m.NS_shift.app.facetRestored.fits'
else:
intname = 'image_full_ampphase_di_m.NS.int.restored.fits'
appname = 'image_full_ampphase_di_m.NS.app.restored.fits'
# astromap blanking if required
bth = None
try:
bth = float(args.astromap_blank)
except:
pass
threshold = float(args.beamcut)
hdus = []
app = []
astromaps = []
wcs = []
print 'Reading files...'
noise = []
name = []
for d in args.directories:
name.append(d.split('/')[-1])
hdu = fits.open(d + '/' + intname)
if args.find_noise:
print 'Estimating noise for', d + '/' + intname
if args.do_lowres:
noise.append(get_rms(hdu, boxsize=1500))
else:
noise.append(get_rms(hdu))
hdus.append(flatten(hdu))
app.append(flatten(fits.open(d + '/' + appname)))
if bth:
astromaps.append(flatten(fits.open(d + '/astromap.fits')))
if args.find_noise:
args.noise = noise
print 'Noise values are:'
for t, n in zip(name, noise):
print t, n
print 'Computing noise/beam factors...'
for i in range(len(app)):
np.seterr(divide='ignore')
app[i].data = np.divide(app[i].data, hdus[i].data)
app[i].data[app[i].data < threshold] = 0
# at this point this is the beam factor: we want 1/sigma**2.0, so divide by central noise and square
if args.noise is not None:
if args.scale is not None:
app[i].data /= args.noise[i] * args.scale[i]
else:
app[i].data /= args.noise[i]
app[i].data = app[i].data**2.0
if args.scale is not None:
hdus[i].data *= args.scale[i]
if args.shift:
print 'Finding shifts (NOTE THIS CODE IS OBSOLETE)...'
# shift according to the FIRST delta ra/dec from quality pipeline
dras = []
ddecs = []
for d in args.directories:
t = Table.read(
d +
'/image_full_ampphase1m.cat.fits_FIRST_match_filtered.fits')
dras.append(np.mean(t['FIRST_dRA']))
ddecs.append(np.mean(t['FIRST_dDEC']))
print 'Applying shifts:', dras, ddecs
for i in range(len(app)):
for hdu in [hdus[i], app[i]]:
ra = hdu.header['CRVAL1']
dec = hdu.header['CRVAL2']
hdu.header['CRVAL1'] -= dras[i] / (3600.0 *
np.cos(np.pi * dec / 180.0))
hdu.header['CRVAL2'] -= ddecs[i] / 3600.0
for i in range(len(app)):
wcs.append(WCS(hdus[i].header))
# astromap blanking
if bth:
print 'Blanking using astrometry quality maps with threshold', bth, 'arcsec'
for i in range(len(app)):
outname = rootname + 'astroblank-' + name[i] + '.fits'
if args.load and os.path.isfile(outname):
print 'Loading previously blanked image'
hdu = fits.open(outname)
hdus[i].data = hdu[0].data
else:
print 'Blanking image', i
dmaxy, dmaxx = hdus[i].data.shape
count = 0
am = astromaps[i]
awcs = WCS(am.header)
maxy, maxx = am.data.shape
for y in range(maxy):
for x in range(maxx):
value = am.data[y, x]
if np.isnan(value):
if y < maxy - 1:
value = am.data[y + 1, x]
if value > bth:
ra, dec = [
float(f) for f in awcs.wcs_pix2world(x, y, 0)
]
rx, ry = [
int(p)
for p in wcs[i].wcs_world2pix(ra, dec, 0)
]
rxp = rx + 21 # astromap pix size, with margin
ryp = ry + 21
if rx < 0: rx = 0
if ry < 0: ry = 0
if rxp > dmaxx: rxp = dmaxx
if ryp > dmaxy: ryp = dmaxy
hdus[i].data[ry:ryp, rx:rxp] = np.nan
count += 1
print '... blanked', count * 900.0 / 3600, 'square arcmin'
outname = rootname + 'astroblank-' + name[i] + '.fits'
if args.save: hdus[i].writeto(outname, clobber=True)
app[i].data[np.isnan(hdus[i].data)] = np.nan
# If the header is directly passed in, use it
try:
header = args.header
xsize = header['NAXIS1']
ysize = header['NAXIS2']
print 'Mosaic using header passed from calling program'
except:
header = None
if header is None:
if args.load_layout:
with open(rootname + 'mosaic-header.pickle') as f:
header = pickle.load(f)
xsize = header['NAXIS1']
ysize = header['NAXIS2']
print 'Mosaic using loaded header'
else:
print 'Creating the mosaic header'
ras = np.array([w.wcs.crval[0] for w in wcs])
decs = np.array([w.wcs.crval[1] for w in wcs])
mra = np.mean(ras)
mdec = np.mean(decs)
print 'Will make mosaic at', mra, mdec
# we make a reference WCS and use it to find the extent in pixels
# needed for the combined image
rwcs = WCS(naxis=2)
rwcs.wcs.ctype = wcs[0].wcs.ctype
rwcs.wcs.cdelt = wcs[0].wcs.cdelt
rwcs.wcs.crval = [mra, mdec]
rwcs.wcs.crpix = [1, 1]
xmin = 0
xmax = 0
ymin = 0
ymax = 0
for a, w in zip(app, wcs):
ys, xs = np.where(a.data)
axmin = xs.min()
aymin = ys.min()
axmax = xs.max()
aymax = ys.max()
del (xs)
del (ys)
print 'non-zero', axmin, aymin, axmax, aymax
for x, y in ((axmin, aymin), (axmax, aymin), (axmin, aymax),
(axmax, aymax)):
ra, dec = [float(f) for f in w.wcs_pix2world(x, y, 0)]
#print ra,dec
nx, ny = [float(f) for f in rwcs.wcs_world2pix(ra, dec, 0)]
print nx, ny
if nx < xmin: xmin = nx
if nx > xmax: xmax = nx
if ny < ymin: ymin = ny
if ny > ymax: ymax = ny
print 'co-ord range:', xmin, xmax, ymin, ymax
xsize = int(xmax - xmin)
ysize = int(ymax - ymin)
rwcs.wcs.crpix = [-int(xmin) + 1, -int(ymin) + 1]
print 'checking:', rwcs.wcs_world2pix(mra, mdec, 0)
print rwcs
header = rwcs.to_header()
header['NAXIS'] = 2
header['NAXIS1'] = xsize
header['NAXIS2'] = ysize
with open(rootname + 'mosaic-header.pickle', 'w') as f:
pickle.dump(header, f)
isum = np.zeros([ysize, xsize])
wsum = np.zeros_like(isum)
mask = np.zeros_like(isum, dtype=np.bool)
print 'now making the mosaic'
for i in range(len(hdus)):
print 'image', i, '(', name[i], ')'
outname = rootname + 'reproject-' + name[i] + '.fits'
if args.load and os.path.exists(outname):
print 'loading...'
hdu = fits.open(outname)
r = hdu[0].data
else:
print 'reprojecting...'
r, footprint = reproj(hdus[i], header, hdu_in=0, parallel=False)
r[np.isnan(r)] = 0
hdu = fits.PrimaryHDU(header=header, data=r)
if args.save: hdu.writeto(outname, clobber=True)
print 'weights', i, '(', name[i], ')'
outname = rootname + 'weight-' + name[i] + '.fits'
if args.load and os.path.exists(outname):
print 'loading...'
hdu = fits.open(outname)
w = hdu[0].data
mask |= (w > 0)
else:
print 'reprojecting...'
w, footprint = reproj(app[i], header, hdu_in=0, parallel=False)
mask |= ~np.isnan(w)
w[np.isnan(w)] = 0
hdu = fits.PrimaryHDU(header=header, data=w)
if args.save: hdu.writeto(outname, clobber=True)
print 'add to mosaic...'
isum += r * w
wsum += w
if not (args.no_write):
isum /= wsum
# mask now contains True where a non-nan region was present in either map
isum[~mask] = np.nan
for ch in ('BMAJ', 'BMIN', 'BPA'):
header[ch] = hdus[0].header[ch]
header['ORIGIN'] = 'ddf-pipeline ' + version()
hdu = fits.PrimaryHDU(header=header, data=isum)
hdu.writeto(rootname + 'mosaic.fits', clobber=True)
hdu = fits.PrimaryHDU(header=header, data=wsum)
hdu.writeto(rootname + 'mosaic-weights.fits', clobber=True)
print_options(option_list)
sys.exit(1)
o = options(sys.argv[1:], option_list)
if o['catch_signal']:
catcher = Catcher()
else:
catcher = None
uvrange = [o['image_uvmin'], o['uvmax']]
killms_uvrange = [0, 1000]
if o['solutions_uvmin'] is not None:
killms_uvrange[0] = o['solutions_uvmin']
if o['mslist'] is None:
die('MS list must be specified')
if o['logging'] is not None and not os.path.isdir(o['logging']):
os.mkdir(o['logging'])
# Set column name for first steps
colname = o['colname']
# Clear the shared memory
run('CleanSHM.py', dryrun=o['dryrun'])
# Check imaging weights -- needed before DDF
new = check_imaging_weight(o['mslist'])
if o['clearcache'] or new or o['redofrom']:
# Clear the cache, we don't know where it's been. If this is a
def run_bootstrap(o):
colname = 'DATA_DI_CORRECTED'
if o['mslist'] is None:
die('MS list must be specified')
if o['logging'] is not None and not os.path.isdir(o['logging']):
os.mkdir(o['logging'])
# check the data supplied
if o['frequencies'] is None or o['catalogues'] is None:
die('Frequencies and catalogues options must be specified')
if "DDF_PIPELINE_CATALOGS" not in os.environ.keys():
warn(
"You need to define the environment variable DDF_PIPELINE_CATALOGS where your catalogs are located"
)
sys.exit(2)
o["tgss"] = o["tgss"].replace("$$", os.environ["DDF_PIPELINE_CATALOGS"])
o["catalogues"] = [
l.replace("$$", os.environ["DDF_PIPELINE_CATALOGS"])
for l in o["catalogues"]
]
lCat = o["catalogues"] + [o["tgss"]]
for fCat in lCat:
if not os.path.isfile(fCat):
warn("Catalog %s does not exist" % fCat)
sys.exit(2)
cl = len(o['catalogues'])
if o['names'] is None:
o['names'] = [
os.path.basename(x).replace('.fits', '') for x in o['catalogues']
]
if o['radii'] is None:
o['radii'] = [10] * cl
if o['groups'] is None:
o['groups'] = range(cl)
if (len(o['frequencies']) != cl or len(o['radii']) != cl
or len(o['names']) != cl or len(o['groups']) != cl):
die('Names, groups, radii and frequencies entries must be the same length as the catalogue list'
)
low_uvrange = [o['image_uvmin'], 2.5 * 206.0 / o['low_psf_arcsec']]
if o['low_imsize'] is not None:
low_imsize = o['low_imsize'] # allow over-ride
else:
low_imsize = o['imsize'] * o['cellsize'] / o['low_cell']
low_robust = o['low_robust']
# Clear the shared memory
run('CleanSHM.py', dryrun=o['dryrun'])
# We use the individual ms in mslist.
m = MSList(o['mslist'])
Uobsid = set(m.obsids)
for obsid in Uobsid:
warn('Running bootstrap for obsid %s' % obsid)
freqs = []
omslist = []
for ms, ob, f in zip(m.mss, m.obsids, m.freqs):
if ob == obsid:
omslist.append(ms)
freqs.append(f)
if len(freqs) < 4:
die('Not enough frequencies to bootstrap. Check your mslist or MS naming scheme'
)
# sort to work in frequency order
freqs, omslist = (list(x) for x in zip(
*sorted(zip(freqs, omslist), key=lambda pair: pair[0])))
for f, ms in zip(freqs, omslist):
print ms, f
# generate the sorted input mslist
with open('temp_mslist.txt', 'w') as f:
for line in omslist:
f.write(line + '\n')
# Clean in cube mode
# As for the main pipeline, first make a dirty map
ddf_image('image_bootstrap_' + obsid + '_init',
'temp_mslist.txt',
cleanmask=None,
cleanmode='SSD',
ddsols='DDS0',
applysols='P',
majorcycles=0,
robust=low_robust,
uvrange=low_uvrange,
beamsize=o['low_psf_arcsec'],
imsize=low_imsize,
cellsize=o['low_cell'],
options=o,
colname=colname,
automask=True,
automask_threshold=15,
smooth=True,
cubemode=True,
conditional_clearcache=True)
external_mask = 'bootstrap_external_mask.fits'
make_external_mask(external_mask,
'image_bootstrap_' + obsid + '_init.dirty.fits',
use_tgss=True,
clobber=False,
cellsize='low_cell',
options=o)
# Deep SSD clean with this external mask and automasking
ddf_image('image_bootstrap_' + obsid,
'temp_mslist.txt',
cleanmask=external_mask,
reuse_psf=True,
reuse_dirty=True,
cleanmode='SSD',
ddsols='DDS0',
applysols='P',
majorcycles=5,
robust=low_robust,
uvrange=low_uvrange,
beamsize=o['low_psf_arcsec'],
imsize=low_imsize,
cellsize=o['low_cell'],
options=o,
colname=colname,
automask=True,
automask_threshold=15,
smooth=True,
cubemode=True,
conditional_clearcache=False)
if os.path.isfile('image_bootstrap_' + obsid +
'.cube.int.restored.pybdsm.srl'):
warn('Source list exists, skipping source extraction')
else:
warn('Running PyBDSM, please wait...')
img = bdsm.process_image('image_bootstrap_' + obsid +
'.cube.int.restored.fits',
thresh_pix=5,
rms_map=True,
atrous_do=True,
atrous_jmax=2,
group_by_isl=True,
rms_box=(80, 20),
adaptive_rms_box=True,
adaptive_thresh=80,
rms_box_bright=(35, 7),
mean_map='zero',
spectralindex_do=True,
specind_maxchan=1,
debug=True,
kappa_clip=3,
flagchan_rms=False,
flagchan_snr=False,
incl_chan=True,
spline_rank=1)
# Write out in ASCII to work round bug in pybdsm
img.write_catalog(catalog_type='srl',
format='ascii',
incl_chan='true')
img.export_image(img_type='rms', img_format='fits')
from make_fitting_product import make_catalogue
import fitting_factors
import find_outliers
# generate the fitting product
if os.path.isfile(obsid + 'crossmatch-1.fits'):
warn('Crossmatch table exists, skipping crossmatch')
else:
t = pt.table(omslist[0] + '/FIELD', readonly=True, ack=False)
direction = t[0]['PHASE_DIR']
ra, dec = direction[0]
if (ra < 0):
ra += 2 * np.pi
ra *= 180.0 / np.pi
dec *= 180.0 / np.pi
cats = zip(o['catalogues'], o['names'], o['groups'], o['radii'])
make_catalogue('image_bootstrap_' + obsid +
'.cube.int.restored.pybdsm.srl',
ra,
dec,
2.5,
cats,
outnameprefix=obsid)
freqlist = open(obsid + 'frequencies.txt', 'w')
for n, f in zip(o['names'], o['frequencies']):
freqlist.write('%f %s_Total_flux %s_E_Total_flux False\n' %
(f, n, n))
for i, f in enumerate(freqs):
freqlist.write('%f Total_flux_ch%i E_Total_flux_ch%i True\n' %
(f, i + 1, i + 1))
freqlist.close()
# Now call the fitting code
if os.path.isfile(obsid + 'crossmatch-results-1.npy'):
warn('Results 1 exists, skipping first fit')
else:
fitting_factors.run_all(1, name=obsid)
nreject = -1 # avoid error if we fail somewhere
if os.path.isfile(obsid + 'crossmatch-2.fits'):
warn('Second crossmatch exists, skipping outlier rejection')
else:
nreject = find_outliers.run_all(1, name=obsid)
if os.path.isfile(obsid + 'crossmatch-results-2.npy'):
warn('Results 2 exists, skipping second fit')
else:
if nreject == 0:
shutil.copyfile(obsid + 'crossmatch-results-1.npy',
obsid + 'crossmatch-results-2.npy')
if os.path.isfile(obsid + 'crossmatch-results-2.npy'):
warn('Results 2 exists, skipping first fit')
else:
fitting_factors.run_all(2, name=obsid)
# Now apply corrections
if o['full_mslist'] is None:
die('Need big mslist to apply corrections')
if not (o['dryrun']):
warn('Applying corrections to MS list')
scale = np.load(obsid + 'crossmatch-results-2.npy')[:, 0]
# InterpolatedUS gives us linear interpolation between points
# and extrapolation outside it
spl = InterpolatedUnivariateSpline(freqs, scale, k=1)
bigmslist = [s.strip() for s in open(o['full_mslist']).readlines()]
obigmslist = [ms for ms in bigmslist if obsid in ms]
for ms in obigmslist:
t = pt.table(ms)
try:
dummy = t.getcoldesc('SCALED_DATA')
except RuntimeError:
dummy = None
t.close()
if dummy is not None:
warn('Table ' + ms +
' has already been corrected, skipping')
else:
# in this version we need to scale both the original data and the data in colname
t = pt.table(ms + '/SPECTRAL_WINDOW',
readonly=True,
ack=False)
frq = t[0]['REF_FREQUENCY']
factor = spl(frq)
print frq, factor
t = pt.table(ms, readonly=False)
desc = t.getcoldesc(o['colname'])
desc['name'] = 'SCALED_DATA'
t.addcols(desc)
d = t.getcol(o['colname'])
d *= factor
t.putcol('SCALED_DATA', d)
try:
dummy = t.getcoldesc(colname)
except RuntimeError:
dummy = None
if dummy is not None:
desc = t.getcoldesc(colname)
newname = colname + '_SCALED'
desc['name'] = newname
t.addcols(desc)
d = t.getcol(colname)
d *= factor
t.putcol(newname, d)
t.close()
if os.path.isfile('image_bootstrap.app.mean.fits'):
warn('Mean bootstrap image exists, not creating it')
else:
warn('Creating mean bootstrap image')
hdus = []
for obsid in Uobsid:
hdus.append(
fits.open('image_bootstrap_' + obsid + '.app.restored.fits'))
for i in range(1, len(Uobsid)):
hdus[0][0].data += hdus[i][0].data
hdus[0][0].data /= len(Uobsid)
hdus[0].writeto('image_bootstrap.app.mean.fits')
def run_bootstrap(o):
colname='DATA_DI_CORRECTED'
if o['mslist'] is None:
die('MS list must be specified')
if o['logging'] is not None and not os.path.isdir(o['logging']):
os.mkdir(o['logging'])
# check the data supplied
if o['frequencies'] is None or o['catalogues'] is None:
die('Frequencies and catalogues options must be specified')
if "DDF_PIPELINE_CATALOGS" not in os.environ.keys():
warn("You need to define the environment variable DDF_PIPELINE_CATALOGS where your catalogs are located")
sys.exit(2)
o["tgss"]=o["tgss"].replace("$$",os.environ["DDF_PIPELINE_CATALOGS"])
o["catalogues"]=[l.replace("$$",os.environ["DDF_PIPELINE_CATALOGS"]) for l in o["catalogues"]]
lCat=o["catalogues"]+[o["tgss"]]
for fCat in lCat:
if not os.path.isfile(fCat):
warn("Catalog %s does not exist"%fCat)
sys.exit(2)
cl=len(o['catalogues'])
if o['names'] is None:
o['names']=[os.path.basename(x).replace('.fits','') for x in o['catalogues']]
if o['radii'] is None:
o['radii']=[10]*cl
if o['groups'] is None:
o['groups']=range(cl)
if (len(o['frequencies'])!=cl or len(o['radii'])!=cl or
len(o['names'])!=cl or len(o['groups'])!=cl):
die('Names, groups, radii and frequencies entries must be the same length as the catalogue list')
low_uvrange=[o['image_uvmin'],2.5*206.0/o['low_psf_arcsec']]
if o['low_imsize'] is not None:
low_imsize=o['low_imsize'] # allow over-ride
else:
low_imsize=o['imsize']*o['cellsize']/o['low_cell']
low_robust=o['low_robust']
# Clear the shared memory
run('CleanSHM.py',dryrun=o['dryrun'])
# We use the individual ms in mslist.
m=MSList(o['mslist'])
Uobsid = set(m.obsids)
for obsid in Uobsid:
warn('Running bootstrap for obsid %s' % obsid)
freqs=[]
omslist=[]
for ms,ob,f in zip(m.mss,m.obsids,m.freqs):
if ob==obsid:
omslist.append(ms)
freqs.append(f)
if len(freqs)<4:
die('Not enough frequencies to bootstrap. Check your mslist or MS naming scheme')
# sort to work in frequency order
freqs,omslist = (list(x) for x in zip(*sorted(zip(freqs, omslist), key=lambda pair: pair[0])))
for f,ms in zip(freqs,omslist):
print ms,f
# generate the sorted input mslist
with open('temp_mslist.txt','w') as f:
for line in omslist:
f.write(line+'\n')
# Clean in cube mode
# As for the main pipeline, first make a dirty map
ddf_image('image_bootstrap_'+obsid+'_init','temp_mslist.txt',
cleanmask=None,cleanmode='SSD',ddsols='DDS0',
applysols='P',majorcycles=0,robust=low_robust,
uvrange=low_uvrange,beamsize=o['low_psf_arcsec'],
imsize=low_imsize,cellsize=o['low_cell'],
options=o,colname=colname,automask=True,
automask_threshold=15,smooth=True,cubemode=True,
conditional_clearcache=True)
external_mask='bootstrap_external_mask.fits'
make_external_mask(external_mask,'image_bootstrap_'+obsid+'_init.dirty.fits',use_tgss=True,clobber=False,cellsize='low_cell',options=o)
# Deep SSD clean with this external mask and automasking
ddf_image('image_bootstrap_'+obsid,'temp_mslist.txt',
cleanmask=external_mask,reuse_psf=True,reuse_dirty=True,
cleanmode='SSD',ddsols='DDS0',applysols='P',
majorcycles=5,robust=low_robust,uvrange=low_uvrange,
beamsize=o['low_psf_arcsec'],imsize=low_imsize,
cellsize=o['low_cell'],options=o,
colname=colname,automask=True,
automask_threshold=15,smooth=True,cubemode=True,
conditional_clearcache=False)
if os.path.isfile('image_bootstrap_'+obsid+'.cube.int.restored.pybdsm.srl'):
warn('Source list exists, skipping source extraction')
else:
warn('Running PyBDSM, please wait...')
img=bdsm.process_image('image_bootstrap_'+obsid+'.cube.int.restored.fits',thresh_pix=5,rms_map=True,atrous_do=True,atrous_jmax=2,group_by_isl=True,rms_box=(80,20), adaptive_rms_box=True, adaptive_thresh=80, rms_box_bright=(35,7),mean_map='zero',spectralindex_do=True,specind_maxchan=1,debug=True,kappa_clip=3,flagchan_rms=False,flagchan_snr=False,incl_chan=True,spline_rank=1)
# Write out in ASCII to work round bug in pybdsm
img.write_catalog(catalog_type='srl',format='ascii',incl_chan='true')
img.export_image(img_type='rms',img_format='fits')
from make_fitting_product import make_catalogue
import fitting_factors
import find_outliers
# generate the fitting product
if os.path.isfile(obsid+'crossmatch-1.fits'):
warn('Crossmatch table exists, skipping crossmatch')
else:
t = pt.table(omslist[0]+ '/FIELD', readonly=True, ack=False)
direction = t[0]['PHASE_DIR']
ra, dec = direction[0]
if (ra<0):
ra+=2*np.pi
ra*=180.0/np.pi
dec*=180.0/np.pi
cats=zip(o['catalogues'],o['names'],o['groups'],o['radii'])
make_catalogue('image_bootstrap_'+obsid+'.cube.int.restored.pybdsm.srl',ra,dec,2.5,cats,outnameprefix=obsid)
freqlist=open(obsid+'frequencies.txt','w')
for n,f in zip(o['names'],o['frequencies']):
freqlist.write('%f %s_Total_flux %s_E_Total_flux False\n' % (f,n,n))
for i,f in enumerate(freqs):
freqlist.write('%f Total_flux_ch%i E_Total_flux_ch%i True\n' % (f,i+1,i+1))
freqlist.close()
# Now call the fitting code
if os.path.isfile(obsid+'crossmatch-results-1.npy'):
warn('Results 1 exists, skipping first fit')
else:
fitting_factors.run_all(1, name=obsid)
nreject=-1 # avoid error if we fail somewhere
if os.path.isfile(obsid+'crossmatch-2.fits'):
warn('Second crossmatch exists, skipping outlier rejection')
else:
nreject=find_outliers.run_all(1, name=obsid)
if os.path.isfile(obsid+'crossmatch-results-2.npy'):
warn('Results 2 exists, skipping second fit')
else:
if nreject==0:
shutil.copyfile(obsid+'crossmatch-results-1.npy',obsid+'crossmatch-results-2.npy')
if os.path.isfile(obsid+'crossmatch-results-2.npy'):
warn('Results 2 exists, skipping first fit')
else:
fitting_factors.run_all(2, name=obsid)
# Now apply corrections
if o['full_mslist'] is None:
die('Need big mslist to apply corrections')
if not(o['dryrun']):
warn('Applying corrections to MS list')
scale=np.load(obsid+'crossmatch-results-2.npy')[:,0]
# InterpolatedUS gives us linear interpolation between points
# and extrapolation outside it
spl = InterpolatedUnivariateSpline(freqs, scale, k=1)
bigmslist=[s.strip() for s in open(o['full_mslist']).readlines()]
obigmslist = [ms for ms in bigmslist if obsid in ms]
for ms in obigmslist:
t = pt.table(ms)
try:
dummy=t.getcoldesc('SCALED_DATA')
except RuntimeError:
dummy=None
t.close()
if dummy is not None:
warn('Table '+ms+' has already been corrected, skipping')
else:
# in this version we need to scale both the original data and the data in colname
t = pt.table(ms+'/SPECTRAL_WINDOW', readonly=True, ack=False)
frq=t[0]['REF_FREQUENCY']
factor=spl(frq)
print frq,factor
t=pt.table(ms,readonly=False)
desc=t.getcoldesc(o['colname'])
desc['name']='SCALED_DATA'
t.addcols(desc)
d=t.getcol(o['colname'])
d*=factor
t.putcol('SCALED_DATA',d)
try:
dummy=t.getcoldesc(colname)
except RuntimeError:
dummy=None
if dummy is not None:
desc=t.getcoldesc(colname)
newname=colname+'_SCALED'
desc['name']=newname
t.addcols(desc)
d=t.getcol(colname)
d*=factor
t.putcol(newname,d)
t.close()
if os.path.isfile('image_bootstrap.app.mean.fits'):
warn('Mean bootstrap image exists, not creating it')
else:
warn('Creating mean bootstrap image')
hdus=[]
for obsid in Uobsid:
hdus.append(fits.open('image_bootstrap_'+obsid+'.app.restored.fits'))
for i in range(1,len(Uobsid)):
hdus[0][0].data+=hdus[i][0].data
hdus[0][0].data/=len(Uobsid)
hdus[0].writeto('image_bootstrap.app.mean.fits')
from auxcodes import report, warn, die
from surveys_db import use_database, update_status
from download import download_dataset
from download_field import download_field
from unpack import unpack, unpack_db_update
from make_mslists import make_list, list_db_update
import sys
import os
rootdir = '/disks/paradata/shimwell/LoTSS-DR2/ongoing-leiden-runs'
os.chdir(rootdir)
name = sys.argv[1]
if name[0] != 'P' and name[0] != 'L':
die('This code should be used only with field or observation names',
database=False)
do_field = (name[0] == 'P')
try:
os.mkdir(name)
except OSError:
warn('Working directory already exists')
pass
os.chdir(name)
report('Downloading data')
if do_field:
success = download_field(name)
else:
success = download_dataset('https://lofar-webdav.grid.sara.nl',
'/SKSP/' + name + '/')
prefix = 'old%i' % count
print 'prefix is', prefix
for ms in msnames:
for s in ['p1', 'ap1']:
if s in archivelist:
for file in ['npz', 'parset']:
rename(
ms + '/killMS.killms_' + s + '.sols.' + file,
ms + '/' + prefix + '.killms_' + s + '.sols.' + file)
if o['full_mslist'] is not None:
with open(o['full_mslist'], 'r') as f:
msnames = [l.strip() for l in f.readlines()]
for ms in msnames:
for s in ['f_ap1', 'f_ap2']:
if s in archivelist:
for file in ['npz', 'parset']:
rename(
ms + '/killMS.killms_' + s + '.sols.' + file, ms +
'/' + prefix + '.killms_' + s + '.sols.' + file)
if __name__ == '__main__':
if len(sys.argv) < 2:
die('This script takes one argument, the name of the config file')
o = options(sys.argv[1])
do_archive(o, ['p1', 'ap1', 'f_ap1', 'f_ap2'])
from auxcodes import report,warn,die
from surveys_db import use_database,update_status
from download import download_dataset
from download_field import download_field
from unpack import unpack,unpack_db_update
from make_mslists import make_list,list_db_update
import sys
import os
rootdir='/disks/paradata/shimwell/LoTSS-DR2/ongoing-leiden-runs'
os.chdir(rootdir)
name=sys.argv[1]
if name[0]!='P' and name[0]!='L':
die('This code should be used only with field or observation names',database=False)
do_field=(name[0]=='P')
try:
os.mkdir(name)
except OSError:
warn('Working directory already exists')
pass
os.chdir(name)
report('Downloading data')
if do_field:
success=download_field(name)
else:
success=download_dataset('https://lofar-webdav.grid.sara.nl','/SKSP/'+name+'/')
def run_bootstrap(o):
if o['mslist'] is None:
die('MS list must be specified')
if o['logging'] is not None and not os.path.isdir(o['logging']):
os.mkdir(o['logging'])
# check the data supplied
if o['frequencies'] is None or o['catalogues'] is None:
die('Frequencies and catalogues options must be specified')
cl=len(o['catalogues'])
if o['names'] is None:
o['names']=[os.path.basename(x).replace('.fits','') for x in o['catalogues']]
if o['radii'] is None:
o['radii']=[10]*cl
if o['groups'] is None:
o['groups']=range(cl)
if (len(o['frequencies'])!=cl or len(o['radii'])!=cl or
len(o['names'])!=cl or len(o['groups'])!=cl):
die('Names, groups, radii and frequencies entries must be the same length as the catalogue list')
low_robust=-0.25
low_uvrange=[0.1,25.0]
# Clear the shared memory
run('CleanSHM.py',dryrun=o['dryrun'])
# We use the individual ms in mslist.
mslist=[s.strip() for s in open(o['mslist']).readlines()]
obsids = [os.path.basename(ms).split('_')[0] for ms in mslist]
Uobsid = set(obsids)
for obsid in Uobsid:
warn('Running bootstrap for obsid %s' % obsid)
# Get the frequencies -- need to take this from the MSs
omslist = [ms for ms in mslist if obsid in ms]
freqs=[]
for ms in omslist:
t = pt.table(ms+'/SPECTRAL_WINDOW', readonly=True, ack=False)
freqs.append(t[0]['REF_FREQUENCY'])
# sort to work in frequency order
freqs,omslist = (list(x) for x in zip(*sorted(zip(freqs, omslist), key=lambda pair: pair[0])))
for f,m in zip(freqs,omslist):
print m,f
# Clean in cube mode
with open('temp_mslist.txt','w') as f:
for line in omslist:
f.write(line+'\n')
ddf_image('image_bootstrap_'+obsid,'temp_mslist.txt',cleanmode='SSD',ddsols='killms_p1',applysols='P',majorcycles=4,robust=low_robust,uvrange=low_uvrange,beamsize=20,imsize=o['bsimsize'],cellsize=o['bscell'],options=o,colname=o['colname'],automask=True,automask_threshold=15,smooth=True,cubemode=True)
if os.path.isfile('image_bootstrap_'+obsid+'.cube.int.restored.pybdsm.srl'):
warn('Source list exists, skipping source extraction')
else:
warn('Running PyBDSM, please wait...')
img=bdsm.process_image('image_bootstrap_'+obsid+'.cube.int.restored.fits',thresh_pix=5,rms_map=True,atrous_do=True,atrous_jmax=2,group_by_isl=True,rms_box=(80,20), adaptive_rms_box=True, adaptive_thresh=80, rms_box_bright=(35,7),mean_map='zero',spectralindex_do=True,specind_maxchan=1,debug=True,kappa_clip=3,flagchan_rms=False,flagchan_snr=False,incl_chan=True,spline_rank=1)
# Write out in ASCII to work round bug in pybdsm
img.write_catalog(catalog_type='srl',format='ascii',incl_chan='true')
img.export_image(img_type='rms',img_format='fits')
from make_fitting_product import make_catalogue
import fitting_factors
import find_outliers
# generate the fitting product
if os.path.isfile(obsid+'crossmatch-1.fits'):
warn('Crossmatch table exists, skipping crossmatch')
else:
t = pt.table(omslist[0]+ '/FIELD', readonly=True, ack=False)
direction = t[0]['PHASE_DIR']
ra, dec = direction[0]
if (ra<0):
ra+=2*np.pi
ra*=180.0/np.pi
dec*=180.0/np.pi
cats=zip(o['catalogues'],o['names'],o['groups'],o['radii'])
make_catalogue('image_bootstrap_'+obsid+'.cube.int.restored.pybdsm.srl',ra,dec,2.5,cats,outnameprefix=obsid)
freqlist=open(obsid+'frequencies.txt','w')
for n,f in zip(o['names'],o['frequencies']):
freqlist.write('%f %s_Total_flux %s_E_Total_flux False\n' % (f,n,n))
for i,f in enumerate(freqs):
freqlist.write('%f Total_flux_ch%i E_Total_flux_ch%i True\n' % (f,i+1,i+1))
freqlist.close()
# Now call the fitting code
if os.path.isfile(obsid+'crossmatch-results-1.npy'):
warn('Results 1 exists, skipping first fit')
else:
fitting_factors.run_all(1, name=obsid)
nreject=-1 # avoid error if we fail somewhere
if os.path.isfile(obsid+'crossmatch-2.fits'):
warn('Second crossmatch exists, skipping outlier rejection')
else:
nreject=find_outliers.run_all(1, name=obsid)
if os.path.isfile(obsid+'crossmatch-results-2.npy'):
warn('Results 2 exists, skipping second fit')
else:
if nreject==0:
shutil.copyfile(obsid+'crossmatch-results-1.npy',obsid+'crossmatch-results-2.npy')
if os.path.isfile(obsid+'crossmatch-results-2.npy'):
warn('Results 2 exists, skipping first fit')
else:
fitting_factors.run_all(2, name=obsid)
# Now apply corrections
if o['full_mslist'] is None:
die('Need big mslist to apply corrections')
if not(o['dryrun']):
warn('Applying corrections to MS list')
scale=np.load(obsid+'crossmatch-results-2.npy')[:,0]
# InterpolatedUS gives us linear interpolation between points
# and extrapolation outside it
spl = InterpolatedUnivariateSpline(freqs, scale, k=1)
bigmslist=[s.strip() for s in open(o['full_mslist']).readlines()]
obigmslist = [ms for ms in bigmslist if obsid in ms]
for ms in obigmslist:
t = pt.table(ms)
try:
dummy=t.getcoldesc('SCALED_DATA')
except RuntimeError:
dummy=None
t.close()
if dummy is not None:
warn('Table '+ms+' has already been corrected, skipping')
else:
t = pt.table(ms+'/SPECTRAL_WINDOW', readonly=True, ack=False)
frq=t[0]['REF_FREQUENCY']
factor=spl(frq)
print frq,factor
t=pt.table(ms,readonly=False)
desc=t.getcoldesc(o['colname'])
desc['name']='SCALED_DATA'
t.addcols(desc)
d=t.getcol(o['colname'])
d*=factor
t.putcol('SCALED_DATA',d)
t.close()
if os.path.isfile('image_bootstrap.app.mean.fits'):
warn('Mean bootstrap image exists, not creating it')
else:
warn('Creating mean bootstrap image')
hdus=[]
for obsid in Uobsid:
hdus.append(fits.open('image_bootstrap_'+obsid+'.app.restored.fits'))
for i in range(1,len(Uobsid)):
hdus[0][0].data+=hdus[i][0].data
hdus[0][0].data/=len(Uobsid)
hdus[0].writeto('image_bootstrap.app.mean.fits')
def do_run_pipeline(name,basedir,qsubfile=None,do_field=True):
'''
set do_field False for the now obsolete behaviour of downloading
and imaging a particular observation
'''
if qsubfile is None:
qsubfile='/home/mjh/pipeline-master/ddf-pipeline/torque/pipeline.qsub'
workdir=basedir+'/'+name
try:
os.mkdir(workdir)
except OSError:
warn('Working directory already exists')
report('Downloading data')
if do_field:
success=download_field(name,basedir=basedir)
else:
success=download_dataset('https://lofar-webdav.grid.sara.nl','/SKSP/'+name+'/',basedir=basedir)
if not success:
die('Download failed, see earlier errors',database=False)
report('Unpacking data')
try:
unpack(workdir=workdir)
except RuntimeError:
if do_field:
update_status(name,'Unpack failed',workdir=workdir)
raise
if do_field:
update_status(name,'Unpacked',workdir=workdir)
report('Deleting tar files')
os.system('rm '+workdir+'/*.tar.gz')
os.system('rm '+workdir+'/*.tar')
averaged=False
report('Checking structure')
g=glob.glob(workdir+'/*.ms')
msl=MSList(None,mss=g)
dysco=np.any(msl.dysco)
uobsids=set(msl.obsids)
for thisobs in uobsids:
# check one MS with each ID
for m,ch,o,hc in zip(msl.mss,msl.channels,msl.obsids,msl.hascorrected):
if o==thisobs:
if not(hc):
print('MS',m,'has no corrected_data column, force use of DATA')
averaged=True
channels=len(ch)
print('MS',m,'has',channels,'channels')
if channels>20:
update_status(name,'Averaging',workdir=workdir)
print('Averaging needed for',thisobs,'!')
averaged=True
average(wildcard=workdir+'/*'+thisobs+'*')
os.system('rm -r '+workdir+'/*'+thisobs+'*pre-cal.ms')
break
report('Making ms lists')
success=make_list(workdir=workdir)
if do_field:
list_db_update(success,workdir=workdir)
if not success:
die('make_list could not construct the MS list',database=False)
report('Creating custom config file from template')
make_custom_config(name,workdir,do_field,averaged)
# now run the job
do_run_job(name,basedir=basedir,qsubfile=None,do_field=do_field,dysco=dysco)
