def do_polcubes(colname,
CurrentDDkMSSolName,
uvrange,imageoutname,
ddf_kw,
beamsize,imsize,cellsize,robust,
options,catcher):
o=options
m=MSList(o['full_mslist'])
ufreqs=sorted(set(m.freqs))
for i,freq in enumerate(ufreqs):
print('Image %i: channel map for frequency %.3f MHz' % (i,old_div(freq,1e6)))
# iterate over frequencies, finding all MS with the same values
fmslist=[]
for ms,f,chan in zip(m.mss,m.freqs,m.channels):
if f==freq:
fmslist.append(ms)
mslistname='stokes-mslist-%i.txt' % i
with open(mslistname,'w') as file:
for ms in fmslist:
file.write(ms+'\n')
channels=len(chan)
ThisImageName = '%s_QU_Cube%s'%(imageoutname,i)
ddf_image(ThisImageName,mslistname,
cleanmode='SSD',ddsols=CurrentDDkMSSolName,
applysols='AP',
polcubemode=True,
AllowNegativeInitHMP=True,
majorcycles=0,robust=robust,
colname=colname,use_dicomodel=False,
uvrange=uvrange,beamsize=beamsize,
imsize=imsize,cellsize=cellsize,peakfactor=0.001,
smooth=True,automask=True,automask_threshold=5,normalization=o['normalize'][2],channels=channels,
startchan=0,endchan=channels,options=o,
catcher=catcher)
outfile='%s_QU.cube.dirty.fits'%imageoutname
if os.path.isfile(outfile) or os.path.isfile(outfile+'.fz'):
warn('Uncorrected cube file already exists, not making it')
else:
report('Making uncorrected cube')
freqs,hdus=get_freqs_hdus('%s_QU_Cube*.cube.dirty.fits'%imageoutname)
make_cube(freqs,hdus,outfile)
outfile='%s_QU.cube.dirty.corr.fits'%imageoutname
if os.path.isfile(outfile) or os.path.isfile(outfile+'.fz'):
warn('Corrected cube file already exists, not making it')
else:
freqs,hdus=get_freqs_hdus('%s_QU_Cube*.cube.dirty.corr.fits'%imageoutname)
report('Making corrected cube')
make_cube(freqs,hdus,outfile)
def run_bootstrap(o):
colname = 'DATA_DI_CORRECTED'
if o['mslist'] is None:
die('MS list must be specified')
if o['logging'] is not None and not os.path.isdir(o['logging']):
os.mkdir(o['logging'])
# check the data supplied
if o['frequencies'] is None or o['catalogues'] is None:
die('Frequencies and catalogues options must be specified')
if "DDF_PIPELINE_CATALOGS" not in os.environ.keys():
warn(
"You need to define the environment variable DDF_PIPELINE_CATALOGS where your catalogs are located"
)
sys.exit(2)
o["tgss"] = o["tgss"].replace("$$", os.environ["DDF_PIPELINE_CATALOGS"])
o["catalogues"] = [
l.replace("$$", os.environ["DDF_PIPELINE_CATALOGS"])
for l in o["catalogues"]
]
lCat = o["catalogues"] + [o["tgss"]]
for fCat in lCat:
if not os.path.isfile(fCat):
warn("Catalog %s does not exist" % fCat)
sys.exit(2)
cl = len(o['catalogues'])
if o['names'] is None:
o['names'] = [
os.path.basename(x).replace('.fits', '') for x in o['catalogues']
]
if o['radii'] is None:
o['radii'] = [10] * cl
if o['groups'] is None:
o['groups'] = range(cl)
if (len(o['frequencies']) != cl or len(o['radii']) != cl
or len(o['names']) != cl or len(o['groups']) != cl):
die('Names, groups, radii and frequencies entries must be the same length as the catalogue list'
)
low_uvrange = [o['image_uvmin'], 2.5 * 206.0 / o['low_psf_arcsec']]
if o['low_imsize'] is not None:
low_imsize = o['low_imsize'] # allow over-ride
else:
low_imsize = o['imsize'] * o['cellsize'] / o['low_cell']
low_robust = o['low_robust']
# Clear the shared memory
run('CleanSHM.py', dryrun=o['dryrun'])
# We use the individual ms in mslist.
m = MSList(o['mslist'])
Uobsid = set(m.obsids)
for obsid in Uobsid:
warn('Running bootstrap for obsid %s' % obsid)
freqs = []
omslist = []
for ms, ob, f in zip(m.mss, m.obsids, m.freqs):
if ob == obsid:
omslist.append(ms)
freqs.append(f)
if len(freqs) < 4:
die('Not enough frequencies to bootstrap. Check your mslist or MS naming scheme'
)
# sort to work in frequency order
freqs, omslist = (list(x) for x in zip(
*sorted(zip(freqs, omslist), key=lambda pair: pair[0])))
for f, ms in zip(freqs, omslist):
print ms, f
# generate the sorted input mslist
with open('temp_mslist.txt', 'w') as f:
for line in omslist:
f.write(line + '\n')
# Clean in cube mode
# As for the main pipeline, first make a dirty map
ddf_image('image_bootstrap_' + obsid + '_init',
'temp_mslist.txt',
cleanmask=None,
cleanmode='SSD',
ddsols='DDS0',
applysols='P',
majorcycles=0,
robust=low_robust,
uvrange=low_uvrange,
beamsize=o['low_psf_arcsec'],
imsize=low_imsize,
cellsize=o['low_cell'],
options=o,
colname=colname,
automask=True,
automask_threshold=15,
smooth=True,
cubemode=True,
conditional_clearcache=True)
external_mask = 'bootstrap_external_mask.fits'
make_external_mask(external_mask,
'image_bootstrap_' + obsid + '_init.dirty.fits',
use_tgss=True,
clobber=False,
cellsize='low_cell',
options=o)
# Deep SSD clean with this external mask and automasking
ddf_image('image_bootstrap_' + obsid,
'temp_mslist.txt',
cleanmask=external_mask,
reuse_psf=True,
reuse_dirty=True,
cleanmode='SSD',
ddsols='DDS0',
applysols='P',
majorcycles=5,
robust=low_robust,
uvrange=low_uvrange,
beamsize=o['low_psf_arcsec'],
imsize=low_imsize,
cellsize=o['low_cell'],
options=o,
colname=colname,
automask=True,
automask_threshold=15,
smooth=True,
cubemode=True,
conditional_clearcache=False)
if os.path.isfile('image_bootstrap_' + obsid +
'.cube.int.restored.pybdsm.srl'):
warn('Source list exists, skipping source extraction')
else:
warn('Running PyBDSM, please wait...')
img = bdsm.process_image('image_bootstrap_' + obsid +
'.cube.int.restored.fits',
thresh_pix=5,
rms_map=True,
atrous_do=True,
atrous_jmax=2,
group_by_isl=True,
rms_box=(80, 20),
adaptive_rms_box=True,
adaptive_thresh=80,
rms_box_bright=(35, 7),
mean_map='zero',
spectralindex_do=True,
specind_maxchan=1,
debug=True,
kappa_clip=3,
flagchan_rms=False,
flagchan_snr=False,
incl_chan=True,
spline_rank=1)
# Write out in ASCII to work round bug in pybdsm
img.write_catalog(catalog_type='srl',
format='ascii',
incl_chan='true')
img.export_image(img_type='rms', img_format='fits')
from make_fitting_product import make_catalogue
import fitting_factors
import find_outliers
# generate the fitting product
if os.path.isfile(obsid + 'crossmatch-1.fits'):
warn('Crossmatch table exists, skipping crossmatch')
else:
t = pt.table(omslist[0] + '/FIELD', readonly=True, ack=False)
direction = t[0]['PHASE_DIR']
ra, dec = direction[0]
if (ra < 0):
ra += 2 * np.pi
ra *= 180.0 / np.pi
dec *= 180.0 / np.pi
cats = zip(o['catalogues'], o['names'], o['groups'], o['radii'])
make_catalogue('image_bootstrap_' + obsid +
'.cube.int.restored.pybdsm.srl',
ra,
dec,
2.5,
cats,
outnameprefix=obsid)
freqlist = open(obsid + 'frequencies.txt', 'w')
for n, f in zip(o['names'], o['frequencies']):
freqlist.write('%f %s_Total_flux %s_E_Total_flux False\n' %
(f, n, n))
for i, f in enumerate(freqs):
freqlist.write('%f Total_flux_ch%i E_Total_flux_ch%i True\n' %
(f, i + 1, i + 1))
freqlist.close()
# Now call the fitting code
if os.path.isfile(obsid + 'crossmatch-results-1.npy'):
warn('Results 1 exists, skipping first fit')
else:
fitting_factors.run_all(1, name=obsid)
nreject = -1 # avoid error if we fail somewhere
if os.path.isfile(obsid + 'crossmatch-2.fits'):
warn('Second crossmatch exists, skipping outlier rejection')
else:
nreject = find_outliers.run_all(1, name=obsid)
if os.path.isfile(obsid + 'crossmatch-results-2.npy'):
warn('Results 2 exists, skipping second fit')
else:
if nreject == 0:
shutil.copyfile(obsid + 'crossmatch-results-1.npy',
obsid + 'crossmatch-results-2.npy')
if os.path.isfile(obsid + 'crossmatch-results-2.npy'):
warn('Results 2 exists, skipping first fit')
else:
fitting_factors.run_all(2, name=obsid)
# Now apply corrections
if o['full_mslist'] is None:
die('Need big mslist to apply corrections')
if not (o['dryrun']):
warn('Applying corrections to MS list')
scale = np.load(obsid + 'crossmatch-results-2.npy')[:, 0]
# InterpolatedUS gives us linear interpolation between points
# and extrapolation outside it
spl = InterpolatedUnivariateSpline(freqs, scale, k=1)
bigmslist = [s.strip() for s in open(o['full_mslist']).readlines()]
obigmslist = [ms for ms in bigmslist if obsid in ms]
for ms in obigmslist:
t = pt.table(ms)
try:
dummy = t.getcoldesc('SCALED_DATA')
except RuntimeError:
dummy = None
t.close()
if dummy is not None:
warn('Table ' + ms +
' has already been corrected, skipping')
else:
# in this version we need to scale both the original data and the data in colname
t = pt.table(ms + '/SPECTRAL_WINDOW',
readonly=True,
ack=False)
frq = t[0]['REF_FREQUENCY']
factor = spl(frq)
print frq, factor
t = pt.table(ms, readonly=False)
desc = t.getcoldesc(o['colname'])
desc['name'] = 'SCALED_DATA'
t.addcols(desc)
d = t.getcol(o['colname'])
d *= factor
t.putcol('SCALED_DATA', d)
try:
dummy = t.getcoldesc(colname)
except RuntimeError:
dummy = None
if dummy is not None:
desc = t.getcoldesc(colname)
newname = colname + '_SCALED'
desc['name'] = newname
t.addcols(desc)
d = t.getcol(colname)
d *= factor
t.putcol(newname, d)
t.close()
if os.path.isfile('image_bootstrap.app.mean.fits'):
warn('Mean bootstrap image exists, not creating it')
else:
warn('Creating mean bootstrap image')
hdus = []
for obsid in Uobsid:
hdus.append(
fits.open('image_bootstrap_' + obsid + '.app.restored.fits'))
for i in range(1, len(Uobsid)):
hdus[0][0].data += hdus[i][0].data
hdus[0][0].data /= len(Uobsid)
hdus[0].writeto('image_bootstrap.app.mean.fits')
def do_run_pipeline(name, basedir):
if name[0] != 'P' and name[0] != 'L':
die('This code should be used only with field or observation names',
database=False)
do_field = (name[0] == 'P')
try:
qsubfile = sys.argv[2]
except:
qsubfile = '/home/mjh/pipeline-master/ddf-pipeline/torque/pipeline.qsub'
workdir = basedir + '/' + name
try:
os.mkdir(workdir)
except OSError:
warn('Working directory already exists')
report('Downloading data')
if do_field:
success = download_field(name, basedir=basedir)
else:
success = download_dataset('https://lofar-webdav.grid.sara.nl',
'/SKSP/' + name + '/',
basedir=basedir)
if not success:
die('Download failed, see earlier errors', database=False)
report('Unpacking data')
try:
unpack(workdir=workdir)
except RuntimeError:
if do_field:
update_status(name, 'List failed', workdir=workdir)
raise
if do_field:
update_status(name, 'Unpacked', workdir=workdir)
report('Deleting tar files')
os.system('rm ' + workdir + '/*.tar.gz')
os.system('rm ' + workdir + '/*.tar')
averaged = False
report('Checking structure')
g = glob.glob(workdir + '/*.ms')
msl = MSList(None, mss=g)
uobsids = set(msl.obsids)
for thisobs in uobsids:
# check one MS with each ID
for m, ch, o in zip(msl.mss, msl.channels, msl.obsids):
if o == thisobs:
channels = len(ch)
print 'MS', m, 'has', channels, 'channels'
if channels > 20:
update_status(name, 'Averaging', workdir=workdir)
print 'Averaging needed for', thisobs, '!'
averaged = True
average(wildcard=workdir + '/*' + thisobs + '*')
os.system('rm -r ' + workdir + '/*' + thisobs +
'*pre-cal.ms')
break
report('Making ms lists')
success = make_list(workdir=workdir)
if do_field:
list_db_update(success, workdir=workdir)
if not success:
die('make_list could not construct the MS list', database=False)
report('Creating custom config file from template')
make_custom_config(name, workdir, do_field, averaged)
# now run the job
do_run_job(name, basedir=basedir, qsubfile=None, do_field=do_field)
def do_run_pipeline(name,basedir,qsubfile=None,do_field=True):
'''
set do_field False for the now obsolete behaviour of downloading
and imaging a particular observation
'''
if qsubfile is None:
qsubfile='/home/mjh/pipeline-master/ddf-pipeline/torque/pipeline.qsub'
workdir=basedir+'/'+name
try:
os.mkdir(workdir)
except OSError:
warn('Working directory already exists')
report('Downloading data')
if do_field:
success=download_field(name,basedir=basedir)
else:
success=download_dataset('https://lofar-webdav.grid.sara.nl','/SKSP/'+name+'/',basedir=basedir)
if not success:
die('Download failed, see earlier errors',database=False)
report('Unpacking data')
try:
unpack(workdir=workdir)
except RuntimeError:
if do_field:
update_status(name,'Unpack failed',workdir=workdir)
raise
if do_field:
update_status(name,'Unpacked',workdir=workdir)
report('Deleting tar files')
os.system('rm '+workdir+'/*.tar.gz')
os.system('rm '+workdir+'/*.tar')
averaged=False
report('Checking structure')
g=glob.glob(workdir+'/*.ms')
msl=MSList(None,mss=g)
dysco=np.any(msl.dysco)
uobsids=set(msl.obsids)
for thisobs in uobsids:
# check one MS with each ID
for m,ch,o,hc in zip(msl.mss,msl.channels,msl.obsids,msl.hascorrected):
if o==thisobs:
if not(hc):
print('MS',m,'has no corrected_data column, force use of DATA')
averaged=True
channels=len(ch)
print('MS',m,'has',channels,'channels')
if channels>20:
update_status(name,'Averaging',workdir=workdir)
print('Averaging needed for',thisobs,'!')
averaged=True
average(wildcard=workdir+'/*'+thisobs+'*')
os.system('rm -r '+workdir+'/*'+thisobs+'*pre-cal.ms')
break
report('Making ms lists')
success=make_list(workdir=workdir)
if do_field:
list_db_update(success,workdir=workdir)
if not success:
die('make_list could not construct the MS list',database=False)
report('Creating custom config file from template')
make_custom_config(name,workdir,do_field,averaged)
# now run the job
do_run_job(name,basedir=basedir,qsubfile=None,do_field=do_field,dysco=dysco)