def do_polcubes(colname, CurrentDDkMSSolName, uvrange,imageoutname, ddf_kw, beamsize,imsize,cellsize,robust, options,catcher): o=options m=MSList(o['full_mslist']) ufreqs=sorted(set(m.freqs)) for i,freq in enumerate(ufreqs): print('Image %i: channel map for frequency %.3f MHz' % (i,old_div(freq,1e6))) # iterate over frequencies, finding all MS with the same values fmslist=[] for ms,f,chan in zip(m.mss,m.freqs,m.channels): if f==freq: fmslist.append(ms) mslistname='stokes-mslist-%i.txt' % i with open(mslistname,'w') as file: for ms in fmslist: file.write(ms+'\n') channels=len(chan) ThisImageName = '%s_QU_Cube%s'%(imageoutname,i) ddf_image(ThisImageName,mslistname, cleanmode='SSD',ddsols=CurrentDDkMSSolName, applysols='AP', polcubemode=True, AllowNegativeInitHMP=True, majorcycles=0,robust=robust, colname=colname,use_dicomodel=False, uvrange=uvrange,beamsize=beamsize, imsize=imsize,cellsize=cellsize,peakfactor=0.001, smooth=True,automask=True,automask_threshold=5,normalization=o['normalize'][2],channels=channels, startchan=0,endchan=channels,options=o, catcher=catcher) outfile='%s_QU.cube.dirty.fits'%imageoutname if os.path.isfile(outfile) or os.path.isfile(outfile+'.fz'): warn('Uncorrected cube file already exists, not making it') else: report('Making uncorrected cube') freqs,hdus=get_freqs_hdus('%s_QU_Cube*.cube.dirty.fits'%imageoutname) make_cube(freqs,hdus,outfile) outfile='%s_QU.cube.dirty.corr.fits'%imageoutname if os.path.isfile(outfile) or os.path.isfile(outfile+'.fz'): warn('Corrected cube file already exists, not making it') else: freqs,hdus=get_freqs_hdus('%s_QU_Cube*.cube.dirty.corr.fits'%imageoutname) report('Making corrected cube') make_cube(freqs,hdus,outfile)
def run_bootstrap(o): colname = 'DATA_DI_CORRECTED' if o['mslist'] is None: die('MS list must be specified') if o['logging'] is not None and not os.path.isdir(o['logging']): os.mkdir(o['logging']) # check the data supplied if o['frequencies'] is None or o['catalogues'] is None: die('Frequencies and catalogues options must be specified') if "DDF_PIPELINE_CATALOGS" not in os.environ.keys(): warn( "You need to define the environment variable DDF_PIPELINE_CATALOGS where your catalogs are located" ) sys.exit(2) o["tgss"] = o["tgss"].replace("$$", os.environ["DDF_PIPELINE_CATALOGS"]) o["catalogues"] = [ l.replace("$$", os.environ["DDF_PIPELINE_CATALOGS"]) for l in o["catalogues"] ] lCat = o["catalogues"] + [o["tgss"]] for fCat in lCat: if not os.path.isfile(fCat): warn("Catalog %s does not exist" % fCat) sys.exit(2) cl = len(o['catalogues']) if o['names'] is None: o['names'] = [ os.path.basename(x).replace('.fits', '') for x in o['catalogues'] ] if o['radii'] is None: o['radii'] = [10] * cl if o['groups'] is None: o['groups'] = range(cl) if (len(o['frequencies']) != cl or len(o['radii']) != cl or len(o['names']) != cl or len(o['groups']) != cl): die('Names, groups, radii and frequencies entries must be the same length as the catalogue list' ) low_uvrange = [o['image_uvmin'], 2.5 * 206.0 / o['low_psf_arcsec']] if o['low_imsize'] is not None: low_imsize = o['low_imsize'] # allow over-ride else: low_imsize = o['imsize'] * o['cellsize'] / o['low_cell'] low_robust = o['low_robust'] # Clear the shared memory run('CleanSHM.py', dryrun=o['dryrun']) # We use the individual ms in mslist. m = MSList(o['mslist']) Uobsid = set(m.obsids) for obsid in Uobsid: warn('Running bootstrap for obsid %s' % obsid) freqs = [] omslist = [] for ms, ob, f in zip(m.mss, m.obsids, m.freqs): if ob == obsid: omslist.append(ms) freqs.append(f) if len(freqs) < 4: die('Not enough frequencies to bootstrap. Check your mslist or MS naming scheme' ) # sort to work in frequency order freqs, omslist = (list(x) for x in zip( *sorted(zip(freqs, omslist), key=lambda pair: pair[0]))) for f, ms in zip(freqs, omslist): print ms, f # generate the sorted input mslist with open('temp_mslist.txt', 'w') as f: for line in omslist: f.write(line + '\n') # Clean in cube mode # As for the main pipeline, first make a dirty map ddf_image('image_bootstrap_' + obsid + '_init', 'temp_mslist.txt', cleanmask=None, cleanmode='SSD', ddsols='DDS0', applysols='P', majorcycles=0, robust=low_robust, uvrange=low_uvrange, beamsize=o['low_psf_arcsec'], imsize=low_imsize, cellsize=o['low_cell'], options=o, colname=colname, automask=True, automask_threshold=15, smooth=True, cubemode=True, conditional_clearcache=True) external_mask = 'bootstrap_external_mask.fits' make_external_mask(external_mask, 'image_bootstrap_' + obsid + '_init.dirty.fits', use_tgss=True, clobber=False, cellsize='low_cell', options=o) # Deep SSD clean with this external mask and automasking ddf_image('image_bootstrap_' + obsid, 'temp_mslist.txt', cleanmask=external_mask, reuse_psf=True, reuse_dirty=True, cleanmode='SSD', ddsols='DDS0', applysols='P', majorcycles=5, robust=low_robust, uvrange=low_uvrange, beamsize=o['low_psf_arcsec'], imsize=low_imsize, cellsize=o['low_cell'], options=o, colname=colname, automask=True, automask_threshold=15, smooth=True, cubemode=True, conditional_clearcache=False) if os.path.isfile('image_bootstrap_' + obsid + '.cube.int.restored.pybdsm.srl'): warn('Source list exists, skipping source extraction') else: warn('Running PyBDSM, please wait...') img = bdsm.process_image('image_bootstrap_' + obsid + '.cube.int.restored.fits', thresh_pix=5, rms_map=True, atrous_do=True, atrous_jmax=2, group_by_isl=True, rms_box=(80, 20), adaptive_rms_box=True, adaptive_thresh=80, rms_box_bright=(35, 7), mean_map='zero', spectralindex_do=True, specind_maxchan=1, debug=True, kappa_clip=3, flagchan_rms=False, flagchan_snr=False, incl_chan=True, spline_rank=1) # Write out in ASCII to work round bug in pybdsm img.write_catalog(catalog_type='srl', format='ascii', incl_chan='true') img.export_image(img_type='rms', img_format='fits') from make_fitting_product import make_catalogue import fitting_factors import find_outliers # generate the fitting product if os.path.isfile(obsid + 'crossmatch-1.fits'): warn('Crossmatch table exists, skipping crossmatch') else: t = pt.table(omslist[0] + '/FIELD', readonly=True, ack=False) direction = t[0]['PHASE_DIR'] ra, dec = direction[0] if (ra < 0): ra += 2 * np.pi ra *= 180.0 / np.pi dec *= 180.0 / np.pi cats = zip(o['catalogues'], o['names'], o['groups'], o['radii']) make_catalogue('image_bootstrap_' + obsid + '.cube.int.restored.pybdsm.srl', ra, dec, 2.5, cats, outnameprefix=obsid) freqlist = open(obsid + 'frequencies.txt', 'w') for n, f in zip(o['names'], o['frequencies']): freqlist.write('%f %s_Total_flux %s_E_Total_flux False\n' % (f, n, n)) for i, f in enumerate(freqs): freqlist.write('%f Total_flux_ch%i E_Total_flux_ch%i True\n' % (f, i + 1, i + 1)) freqlist.close() # Now call the fitting code if os.path.isfile(obsid + 'crossmatch-results-1.npy'): warn('Results 1 exists, skipping first fit') else: fitting_factors.run_all(1, name=obsid) nreject = -1 # avoid error if we fail somewhere if os.path.isfile(obsid + 'crossmatch-2.fits'): warn('Second crossmatch exists, skipping outlier rejection') else: nreject = find_outliers.run_all(1, name=obsid) if os.path.isfile(obsid + 'crossmatch-results-2.npy'): warn('Results 2 exists, skipping second fit') else: if nreject == 0: shutil.copyfile(obsid + 'crossmatch-results-1.npy', obsid + 'crossmatch-results-2.npy') if os.path.isfile(obsid + 'crossmatch-results-2.npy'): warn('Results 2 exists, skipping first fit') else: fitting_factors.run_all(2, name=obsid) # Now apply corrections if o['full_mslist'] is None: die('Need big mslist to apply corrections') if not (o['dryrun']): warn('Applying corrections to MS list') scale = np.load(obsid + 'crossmatch-results-2.npy')[:, 0] # InterpolatedUS gives us linear interpolation between points # and extrapolation outside it spl = InterpolatedUnivariateSpline(freqs, scale, k=1) bigmslist = [s.strip() for s in open(o['full_mslist']).readlines()] obigmslist = [ms for ms in bigmslist if obsid in ms] for ms in obigmslist: t = pt.table(ms) try: dummy = t.getcoldesc('SCALED_DATA') except RuntimeError: dummy = None t.close() if dummy is not None: warn('Table ' + ms + ' has already been corrected, skipping') else: # in this version we need to scale both the original data and the data in colname t = pt.table(ms + '/SPECTRAL_WINDOW', readonly=True, ack=False) frq = t[0]['REF_FREQUENCY'] factor = spl(frq) print frq, factor t = pt.table(ms, readonly=False) desc = t.getcoldesc(o['colname']) desc['name'] = 'SCALED_DATA' t.addcols(desc) d = t.getcol(o['colname']) d *= factor t.putcol('SCALED_DATA', d) try: dummy = t.getcoldesc(colname) except RuntimeError: dummy = None if dummy is not None: desc = t.getcoldesc(colname) newname = colname + '_SCALED' desc['name'] = newname t.addcols(desc) d = t.getcol(colname) d *= factor t.putcol(newname, d) t.close() if os.path.isfile('image_bootstrap.app.mean.fits'): warn('Mean bootstrap image exists, not creating it') else: warn('Creating mean bootstrap image') hdus = [] for obsid in Uobsid: hdus.append( fits.open('image_bootstrap_' + obsid + '.app.restored.fits')) for i in range(1, len(Uobsid)): hdus[0][0].data += hdus[i][0].data hdus[0][0].data /= len(Uobsid) hdus[0].writeto('image_bootstrap.app.mean.fits')
def do_run_pipeline(name, basedir): if name[0] != 'P' and name[0] != 'L': die('This code should be used only with field or observation names', database=False) do_field = (name[0] == 'P') try: qsubfile = sys.argv[2] except: qsubfile = '/home/mjh/pipeline-master/ddf-pipeline/torque/pipeline.qsub' workdir = basedir + '/' + name try: os.mkdir(workdir) except OSError: warn('Working directory already exists') report('Downloading data') if do_field: success = download_field(name, basedir=basedir) else: success = download_dataset('https://lofar-webdav.grid.sara.nl', '/SKSP/' + name + '/', basedir=basedir) if not success: die('Download failed, see earlier errors', database=False) report('Unpacking data') try: unpack(workdir=workdir) except RuntimeError: if do_field: update_status(name, 'List failed', workdir=workdir) raise if do_field: update_status(name, 'Unpacked', workdir=workdir) report('Deleting tar files') os.system('rm ' + workdir + '/*.tar.gz') os.system('rm ' + workdir + '/*.tar') averaged = False report('Checking structure') g = glob.glob(workdir + '/*.ms') msl = MSList(None, mss=g) uobsids = set(msl.obsids) for thisobs in uobsids: # check one MS with each ID for m, ch, o in zip(msl.mss, msl.channels, msl.obsids): if o == thisobs: channels = len(ch) print 'MS', m, 'has', channels, 'channels' if channels > 20: update_status(name, 'Averaging', workdir=workdir) print 'Averaging needed for', thisobs, '!' averaged = True average(wildcard=workdir + '/*' + thisobs + '*') os.system('rm -r ' + workdir + '/*' + thisobs + '*pre-cal.ms') break report('Making ms lists') success = make_list(workdir=workdir) if do_field: list_db_update(success, workdir=workdir) if not success: die('make_list could not construct the MS list', database=False) report('Creating custom config file from template') make_custom_config(name, workdir, do_field, averaged) # now run the job do_run_job(name, basedir=basedir, qsubfile=None, do_field=do_field)
def do_run_pipeline(name,basedir,qsubfile=None,do_field=True): ''' set do_field False for the now obsolete behaviour of downloading and imaging a particular observation ''' if qsubfile is None: qsubfile='/home/mjh/pipeline-master/ddf-pipeline/torque/pipeline.qsub' workdir=basedir+'/'+name try: os.mkdir(workdir) except OSError: warn('Working directory already exists') report('Downloading data') if do_field: success=download_field(name,basedir=basedir) else: success=download_dataset('https://lofar-webdav.grid.sara.nl','/SKSP/'+name+'/',basedir=basedir) if not success: die('Download failed, see earlier errors',database=False) report('Unpacking data') try: unpack(workdir=workdir) except RuntimeError: if do_field: update_status(name,'Unpack failed',workdir=workdir) raise if do_field: update_status(name,'Unpacked',workdir=workdir) report('Deleting tar files') os.system('rm '+workdir+'/*.tar.gz') os.system('rm '+workdir+'/*.tar') averaged=False report('Checking structure') g=glob.glob(workdir+'/*.ms') msl=MSList(None,mss=g) dysco=np.any(msl.dysco) uobsids=set(msl.obsids) for thisobs in uobsids: # check one MS with each ID for m,ch,o,hc in zip(msl.mss,msl.channels,msl.obsids,msl.hascorrected): if o==thisobs: if not(hc): print('MS',m,'has no corrected_data column, force use of DATA') averaged=True channels=len(ch) print('MS',m,'has',channels,'channels') if channels>20: update_status(name,'Averaging',workdir=workdir) print('Averaging needed for',thisobs,'!') averaged=True average(wildcard=workdir+'/*'+thisobs+'*') os.system('rm -r '+workdir+'/*'+thisobs+'*pre-cal.ms') break report('Making ms lists') success=make_list(workdir=workdir) if do_field: list_db_update(success,workdir=workdir) if not success: die('make_list could not construct the MS list',database=False) report('Creating custom config file from template') make_custom_config(name,workdir,do_field,averaged) # now run the job do_run_job(name,basedir=basedir,qsubfile=None,do_field=do_field,dysco=dysco)