def make_mask(imagename, thresh): fname = imagename + '.mask.fits' runcommand = "MakeMask.py --RestoredIm=%s --Th=%s --Box=50,2" % (imagename, thresh) run(runcommand) return fname
def make_mask(imagename, thresh, verbose=False, options=None, external_mask=None, catcher=None): if catcher: catcher.check() # mask_use specifies a mask file to use if options is None: options = o # attempt to get global fname = imagename + '.mask.fits' runcommand = "MakeMask.py --RestoredIm=%s --Th=%s --Box=50,2" % (imagename, thresh) if options['restart'] and os.path.isfile(fname): warn('File ' + fname + ' already exists, skipping MakeMask step') if verbose: print 'Would have run', runcommand else: run(runcommand, dryrun=options['dryrun'], log=logfilename('MM-' + imagename + '.log', options=options), quiet=options['quiet']) if external_mask is not None: merge_mask(fname, external_mask, fname)
def killms_data(imagename, mslist, outsols, clusterfile=None, colname='CORRECTED_DATA', niterkf=6, dicomodel=None, uvrange=None, wtuv=None, robust=None, catcher=None, options=None): if options is None: options = o # attempt to get global if it exists cache_dir = find_cache_dir(options) # run killms individually on each MS -- allows restart if it failed in the middle filenames = [l.strip() for l in open(mslist, 'r').readlines()] for f in filenames: if catcher: catcher.check() checkname = f + '/killMS.' + outsols + '.sols.npz' if options['restart'] and os.path.isfile(checkname): warn('Solutions file ' + checkname + ' already exists, not running killMS step') else: runcommand = "killMS.py --MSName %s --SolverType KAFCA --PolMode Scalar --BaseImageName %s --dt %f --BeamMode LOFAR --LOFARBeamMode=A --NIterKF %i --CovQ 0.1 --LambdaKF=%f --NCPU %i --OutSolsName %s --NChanSols %i --PowerSmooth=%f --InCol %s --DDFCacheDir=%s" % ( f, imagename, options['dt'], niterkf, options['LambdaKF'], options['NCPU_killms'], outsols, options['NChanSols'], options['PowerSmooth'], colname, cache_dir) if robust is None: runcommand += ' --Weighting Natural' else: runcommand += ' --Weighting Briggs --Robust=%f' % robust if uvrange is not None: if wtuv is not None: runcommand += ' --WTUV=%f --WeightUVMinMax=%f,%f' % ( wtuv, uvrange[0], uvrange[1]) else: runcommand += ' --UVMinMax=%f,%f' % (uvrange[0], uvrange[1]) if clusterfile is not None: runcommand += ' --NodesFile ' + clusterfile if dicomodel is not None: runcommand += ' --DicoModel ' + dicomodel if options['nobar']: runcommand += ' --DoBar=0' rootfilename = outsols.split('/')[-1] f = f.replace("/", "_") run(runcommand, dryrun=options['dryrun'], log=logfilename('KillMS-' + f + '_' + rootfilename + '.log', options=options), quiet=options['quiet'])
def mask_dicomodel(indico, maskname, outdico, catcher=None): if catcher: catcher.check() if o['restart'] and os.path.isfile(outdico): warn('File ' + outdico + ' already exists, skipping MaskDicoModel step') return False else: runcommand = "MaskDicoModel.py --MaskName=%s --InDicoModel=%s --OutDicoModel=%s" % ( maskname, indico, outdico) run(runcommand, dryrun=o['dryrun'], log=logfilename('MaskDicoModel-' + maskname + '.log'), quiet=o['quiet']) return True
def make_model(maskname, imagename, catcher=None): # returns True if the step was run, False if skipped if catcher: catcher.check() fname = imagename + '.npy' if o['restart'] and os.path.isfile(fname): warn('File ' + fname + ' already exists, skipping MakeModel step') return False else: runcommand = "MakeModel.py --MaskName=%s --BaseImageName=%s --NCluster=%i --DoPlot=0" % ( maskname, imagename, o['ndir']) run(runcommand, dryrun=o['dryrun'], log=logfilename('MakeModel-' + maskname + '.log'), quiet=o['quiet']) return True
def ddf_shift(imagename, shiftfile, catcher=None, options=None, verbose=False): if catcher: catcher.check() if options is None: options = o # attempt to get global if it exists cache_dir = find_cache_dir(options) runcommand = 'DDF.py ' + imagename + '.parset --Output-Name=' + imagename + '_shift --Image-Mode=RestoreAndShift --Output-ShiftFacetsFile=' + shiftfile + ' --Predict-InitDicoModel ' + imagename + '.DicoModel --Cache-SmoothBeam=force --Cache-Dir=' + cache_dir fname = imagename + '_shift.app.facetRestored.fits' if options['restart'] and os.path.isfile(fname): warn('File ' + fname + ' already exists, skipping DDF-shift step') if verbose: print 'would have run', runcommand else: run(runcommand, dryrun=options['dryrun'], log=logfilename('DDF-' + imagename + '_shift.log', options=options), quiet=options['quiet'])
def image_vlow(wd=None): if wd is not None: os.chdir(wd) update_status(None, 'Running') run('CleanSHM.py') run('DDF.py --Output-Name=image_full_vlow_nocut --Data-MS=big-mslist.txt --Deconv-PeakFactor 0.001000 --Data-ColName DATA --Parallel-NCPU=%i --Beam-CenterNorm=1 --Deconv-CycleFactor=0 --Deconv-MaxMinorIter=1000000 --Deconv-MaxMajorIter=2 --Deconv-Mode SSD --Beam-Model=LOFAR --Beam-LOFARBeamMode=A --Weight-Robust -0.20000 --Image-NPix=2000 --CF-wmax 50000 --CF-Nw 100 --Output-Also onNeds --Image-Cell 15.00000 --Facets-NFacets=11 --SSDClean-NEnlargeData 0 --Freq-NDegridBand 1 --Beam-NBand 1 --Facets-DiamMax 1.5 --Facets-DiamMin 0.1 --Deconv-RMSFactor=3.000000 --SSDClean-ConvFFTSwitch 10000 --Data-Sort 1 --Cache-Dir=. --Log-Memory 1 --GAClean-RMSFactorInitHMP 1.000000 --GAClean-MaxMinorIterInitHMP 10000.000000 --GAClean-AllowNegativeInitHMP True --DDESolutions-SolsDir=SOLSDIR --Cache-Weight=reset --Output-Mode=Clean --Output-RestoringBeam 60.000000 --Weight-ColName="IMAGING_WEIGHT" --Freq-NBand=2 --RIME-DecorrMode=FT --SSDClean-SSDSolvePars [S,Alpha] --SSDClean-BICFactor 0 --Mask-Auto=1 --Mask-SigTh=4.00 --DDESolutions-GlobalNorm=None --DDESolutions-DDModeGrid=AP --DDESolutions-DDModeDeGrid=AP --DDESolutions-DDSols=[DDS3_full_smoothed,DDS3_full_slow] --Selection-UVRangeKm=[0.000000,7.0] --GAClean-MinSizeInit=10 --Beam-Smooth=1 --Debug-Pdb=never' % getcpus()) vlowmask = make_mask('image_full_vlow_nocut.app.restored.fits', 3.0) run('DDF.py --Output-Name=image_full_vlow_nocut_m --Data-MS=big-mslist.txt --Deconv-PeakFactor 0.001000 --Data-ColName DATA --Parallel-NCPU=%i --Beam-CenterNorm=1 --Deconv-CycleFactor=0 --Deconv-MaxMinorIter=1000000 --Deconv-MaxMajorIter=2 --Deconv-Mode SSD --Beam-Model=LOFAR --Beam-LOFARBeamMode=A --Weight-Robust -0.20000 --Image-NPix=2000 --CF-wmax 50000 --CF-Nw 100 --Output-Also onNeds --Image-Cell 15.00000 --Facets-NFacets=11 --SSDClean-NEnlargeData 0 --Freq-NDegridBand 1 --Beam-NBand 1 --Facets-DiamMax 1.5 --Facets-DiamMin 0.1 --Deconv-RMSFactor=3.000000 --SSDClean-ConvFFTSwitch 10000 --Data-Sort 1 --Cache-Dir=. --Log-Memory 1 --GAClean-RMSFactorInitHMP 1.000000 --GAClean-MaxMinorIterInitHMP 10000.000000 --GAClean-AllowNegativeInitHMP True --DDESolutions-SolsDir=SOLSDIR --Cache-Weight=reset --Output-Mode=Clean --Output-RestoringBeam 60.000000 --Weight-ColName="IMAGING_WEIGHT" --Freq-NBand=2 --RIME-DecorrMode=FT --SSDClean-SSDSolvePars [S,Alpha] --SSDClean-BICFactor 0 --Mask-Auto=1 --Mask-SigTh=3.00 --DDESolutions-GlobalNorm=None --DDESolutions-DDModeGrid=AP --DDESolutions-DDModeDeGrid=AP --DDESolutions-DDSols=[DDS3_full_smoothed,DDS3_full_slow] --Selection-UVRangeKm=[0.000000,7.0] --GAClean-MinSizeInit=10 --Beam-Smooth=1 --Debug-Pdb=never --Predict-InitDicoModel=image_full_vlow_nocut.DicoModel --Mask-External=%s' % (getcpus(), vlowmask)) update_status(None, 'Complete')
def run_wsclean(useIDG=False, stringMultiscaleScales="0,4,8,16,32,64,128,256", stringMultiscaleScalesIDG="0,4,8,16,32,64", sizePixels=2000, taperGaussian=60.0, beamsize=60.0, scale=15.0, numberOfSubbands=6, uvDistanceMinLambda=0, IDGMode="cpu", name='WSCLEAN_low'): update_status(None, 'WSCLEAN') g = glob.glob('*.archive0') stringMSs = ' '.join(g) print("Imaging with multiscale CLEAN (at low-resolution)...") if (useIDG): # Generate IDG configuration file. print("Generating IDG configuration file...") with open("aconfig.txt", "w") as configurationFile: configurationFile.write( "aterms=[beam]\nbeam.differential = true\nbeam.update_interval = 600\nbeam.usechannelfreq = true" ) command = "wsclean -no-update-model-required -size " + str( sizePixels ) + " " + str( sizePixels ) + " -reorder -weight briggs -0.5 -weighting-rank-filter 3 -clean-border 1 -mgain 0.8 -no-fit-beam -data-column DATA -join-channels -channels-out " + str( numberOfSubbands ) + " -padding 1.2 -multiscale -multiscale-scales " + stringMultiscaleScalesIDG + " -auto-mask 3.0 -auto-threshold 2.5 -taper-gaussian " + str( taperGaussian ) + "arcsec -circular-beam -beam-size " + str( beamsize ) + "arcsec -pol i -name " + name + " -scale " + str( scale ) + "arcsec -niter 100000 -minuv-l " + str( uvDistanceMinLambda ) + " -use-idg -idg-mode " + IDGMode + " -aterm-kernel-size 16 -aterm-config aconfig.txt " + stringMSs else: command = "wsclean -no-update-model-required -apply-primary-beam -use-differential-lofar-beam -size " + str( sizePixels ) + " " + str( sizePixels ) + " -reorder -weight briggs -0.5 -weighting-rank-filter 3 -clean-border 1 -mgain 0.8 -no-fit-beam -data-column DATA -join-channels -channels-out " + str( numberOfSubbands ) + " -padding 1.2 -multiscale -multiscale-scales " + stringMultiscaleScales + " -auto-mask 3.0 -auto-threshold 2.5 -taper-gaussian " + str( taperGaussian) + "arcsec -circular-beam -beam-size " + str( beamsize) + "arcsec -pol i -name " + name + " -scale " + str( scale) + "arcsec -niter 100000 -minuv-l " + str( uvDistanceMinLambda ) + " -baseline-averaging 10.0 " + stringMSs print(command) run(command) print('Now correcting for beam...') f = 'image_full_vlow_nocut_m.app.restored.fits' dapp = fits.open(f) dint = fits.open(f.replace('app', 'int')) beam = dint[0].data / dapp[0].data wsuc = fits.open('WSCLEAN_low-MFS-image.fits') wsuc[0].data *= beam[0, 0, 12:-13, 12:-13] wsuc.writeto('WSCLEAN_low-MFS-image-int.fits') update_status(None, 'WSCLEAN complete')
def subtract_sources(): update_status(None, 'Subtracting') run('sub-sources-outside-region.py -b fullfield -p SUB -c DATA --uselowres -t 8 -f 5' ) update_status(None, 'Subtracted')
# os.system(cmd) # Apparently it can be dangerous to remove a column for tiled storagemanagers, comment out! #for ms in msfiles: # removecolumn(ms, 'PREDICT_SUB') # removecolumn(ms, 'DATA_SUB') os.system('rm -f ' + outdico) # clean up os.system('rm -f ' + outmask) if boxfile != 'fullfield': mask_region(fullmask, boxfile, outmask) else: outmask = fullmask run("MaskDicoModel.py --MaskName=%s --InDicoModel=%s --OutDicoModel=%s" % (outmask, indico, outdico)) #if uselowres == False: ##imagenpix = 20000 #robust=-0.5 #imagecell = 1.5 #else: ##imagenpix = 6000 #robust = -0.25 #imagecell = 4.5 if holesfixed: print('Starting DDF for prediction') if args['h5sols'] != None: run("DDF.py --Output-Name=image_dd_SUB --Data-ChunkHours=" + str(args['chunkhours']) + " --Data-MS=" + args['mslist'] + " --Deconv-PeakFactor 0.001000 --Data-ColName " +
def ddf_image(imagename, mslist, cleanmask=None, cleanmode='HMP', ddsols=None, applysols=None, threshold=None, majorcycles=3, use_dicomodel=False, robust=0, beamsize=None, beamsize_minor=None, beamsize_pa=None, reuse_psf=False, reuse_dirty=False, verbose=False, saveimages=None, imsize=None, cellsize=None, uvrange=None, colname='CORRECTED_DATA', peakfactor=0.1, dicomodel_base=None, options=None, do_decorr=None, normalization=None, dirty_from_resid=False, clusterfile=None, HMPsize=None, automask=True, automask_threshold=10.0, smooth=False, noweights=False, cubemode=False, apply_weights=True, catcher=None, rms_factor=3.0): if catcher: catcher.check() # saveimages lists _additional_ images to save if saveimages is None: saveimages = '' saveimages += 'onNeds' if options is None: options = o # attempt to get global if it exists if HMPsize is None: HMPsize = options['HMPsize'] if do_decorr is None: do_decorr = options['do_decorr'] if beamsize is None: beamsize = options['psf_arcsec'] if imsize is None: imsize = options['imsize'] if cellsize is None: cellsize = options['cellsize'] cache_dir = find_cache_dir(options) if majorcycles > 0: fname = imagename + '.app.restored.fits' else: fname = imagename + '.dirty.fits' runcommand = "DDF.py --Output-Name=%s --Data-MS=%s --Deconv-PeakFactor %f --Data-ColName %s --Parallel-NCPU=%i --Image-Mode=Clean --Beam-CenterNorm=1 --Deconv-CycleFactor=0 --Deconv-MaxMinorIter=1000000 --Deconv-MaxMajorIter=%s --Deconv-Mode %s --Beam-Model=LOFAR --Beam-LOFARBeamMode=A --Weight-Robust %f --Image-NPix=%i --CF-wmax 50000 --CF-Nw 100 --Output-Also %s --Image-Cell %f --Facets-NFacets=11 --SSDClean-NEnlargeData 0 --Freq-NDegridBand 1 --Beam-NBand 1 --Facets-DiamMax 1.5 --Facets-DiamMin 0.1 --Deconv-RMSFactor=%f --Data-Sort 1 --Cache-Dir=%s" % ( imagename, mslist, peakfactor, colname, options['NCPU_DDF'], majorcycles, cleanmode, robust, imsize, saveimages, float(cellsize), rms_factor, cache_dir) if beamsize_minor is not None: runcommand += ' --Output-RestoringBeam %f,%f,%f' % ( beamsize, beamsize_minor, beamsize_pa) elif beamsize is not None: runcommand += ' --Output-RestoringBeam %f' % (beamsize) if apply_weights: runcommand += ' --Weight-ColName="IMAGING_WEIGHT"' else: runcommand += ' --Weight-ColName="None"' if cubemode: channels = len(open(mslist).readlines()) runcommand += ' --Output-Cubes I --Freq-NBand=%i' % channels else: runcommand += ' --Freq-NBand=2' if do_decorr: runcommand += ' --RIME-DecorrMode=FT' if cleanmode == 'SSD': runcommand += ' --SSDClean-SSDSolvePars [S,Alpha] --SSDClean-BICFactor 0' if clusterfile is not None: runcommand += ' --Facets-CatNodes=%s' % clusterfile if automask: runcommand += ' --Mask-Auto=1 --Mask-SigTh=%.2f' % automask_threshold if cleanmask is not None: runcommand += ' --Mask-External=%s' % cleanmask if applysols is not None: if normalization is not None: if normalization[:3] == 'Abs': normalization = 'Mean' + normalization # backward compat. hack runcommand += ' --DDESolutions-GlobalNorm=' + normalization runcommand += ' --DDESolutions-DDModeGrid=%s --DDESolutions-DDModeDeGrid=%s --DDESolutions-DDSols=%s' % ( applysols, applysols, ddsols) if use_dicomodel: if dicomodel_base is not None: runcommand += ' --Predict-InitDicoModel=%s.DicoModel' % dicomodel_base else: raise RuntimeError( 'use_dicomodel is set but no dicomodel supplied') if threshold is not None: runcommand += ' --Deconv-FluxThreshold=%f' % threshold if uvrange is not None: runcommand += ' --Selection-UVRangeKm=[%f,%f]' % (uvrange[0], uvrange[1]) if dirty_from_resid and reuse_dirty: raise RuntimeError('Cannot combine reuse_dirty and dirty_from_resid') if dirty_from_resid: # possible that crashes could destroy the cache, so need to check if os.path.exists(cache_dir + '/' + mslist + '.ddfcache/LastResidual'): runcommand += ' --Cache-Dirty forceresidual' if reuse_dirty: if os.path.exists(cache_dir + '/' + mslist + '.ddfcache/Dirty'): runcommand += ' --Cache-Dirty forcedirty' if reuse_psf: if os.path.exists(cache_dir + '/' + mslist + '.ddfcache/PSF'): runcommand += ' --Cache-PSF force' if HMPsize is not None: runcommand += ' --SSDClean-MinSizeInitHMP=%i' % HMPsize if options['nobar']: runcommand += ' --Log-Boring=1' if smooth: runcommand += ' --Beam-Smooth=1' if options['restart'] and os.path.isfile(fname): warn('File ' + fname + ' already exists, skipping DDF step') if verbose: print 'would have run', runcommand else: run(runcommand, dryrun=options['dryrun'], log=logfilename('DDF-' + imagename + '.log', options=options), quiet=options['quiet'])
uvrange = [o['image_uvmin'], o['uvmax']] killms_uvrange = [0, 1000] if o['solutions_uvmin'] is not None: killms_uvrange[0] = o['solutions_uvmin'] if o['mslist'] is None: die('MS list must be specified') if o['logging'] is not None and not os.path.isdir(o['logging']): os.mkdir(o['logging']) # Set column name for first steps colname = o['colname'] # Clear the shared memory run('CleanSHM.py', dryrun=o['dryrun']) # Check imaging weights -- needed before DDF new = check_imaging_weight(o['mslist']) if o['clearcache'] or new or o['redofrom']: # Clear the cache, we don't know where it's been. If this is a # completely new dataset it is always safe (and required) to # clear the cache -- solves problems where the cache is not # stored per dataset. If we are redoing, cache needs to be removed full_clearcache(o) if o['redofrom']: if not os.path.isdir(o['archive_dir']): os.mkdir(o['archive_dir'])
def run_bootstrap(o): colname = 'DATA_DI_CORRECTED' if o['mslist'] is None: die('MS list must be specified') if o['logging'] is not None and not os.path.isdir(o['logging']): os.mkdir(o['logging']) # check the data supplied if o['frequencies'] is None or o['catalogues'] is None: die('Frequencies and catalogues options must be specified') if "DDF_PIPELINE_CATALOGS" not in os.environ.keys(): warn( "You need to define the environment variable DDF_PIPELINE_CATALOGS where your catalogs are located" ) sys.exit(2) o["tgss"] = o["tgss"].replace("$$", os.environ["DDF_PIPELINE_CATALOGS"]) o["catalogues"] = [ l.replace("$$", os.environ["DDF_PIPELINE_CATALOGS"]) for l in o["catalogues"] ] lCat = o["catalogues"] + [o["tgss"]] for fCat in lCat: if not os.path.isfile(fCat): warn("Catalog %s does not exist" % fCat) sys.exit(2) cl = len(o['catalogues']) if o['names'] is None: o['names'] = [ os.path.basename(x).replace('.fits', '') for x in o['catalogues'] ] if o['radii'] is None: o['radii'] = [10] * cl if o['groups'] is None: o['groups'] = range(cl) if (len(o['frequencies']) != cl or len(o['radii']) != cl or len(o['names']) != cl or len(o['groups']) != cl): die('Names, groups, radii and frequencies entries must be the same length as the catalogue list' ) low_uvrange = [o['image_uvmin'], 2.5 * 206.0 / o['low_psf_arcsec']] if o['low_imsize'] is not None: low_imsize = o['low_imsize'] # allow over-ride else: low_imsize = o['imsize'] * o['cellsize'] / o['low_cell'] low_robust = o['low_robust'] # Clear the shared memory run('CleanSHM.py', dryrun=o['dryrun']) # We use the individual ms in mslist. m = MSList(o['mslist']) Uobsid = set(m.obsids) for obsid in Uobsid: warn('Running bootstrap for obsid %s' % obsid) freqs = [] omslist = [] for ms, ob, f in zip(m.mss, m.obsids, m.freqs): if ob == obsid: omslist.append(ms) freqs.append(f) if len(freqs) < 4: die('Not enough frequencies to bootstrap. Check your mslist or MS naming scheme' ) # sort to work in frequency order freqs, omslist = (list(x) for x in zip( *sorted(zip(freqs, omslist), key=lambda pair: pair[0]))) for f, ms in zip(freqs, omslist): print ms, f # generate the sorted input mslist with open('temp_mslist.txt', 'w') as f: for line in omslist: f.write(line + '\n') # Clean in cube mode # As for the main pipeline, first make a dirty map ddf_image('image_bootstrap_' + obsid + '_init', 'temp_mslist.txt', cleanmask=None, cleanmode='SSD', ddsols='DDS0', applysols='P', majorcycles=0, robust=low_robust, uvrange=low_uvrange, beamsize=o['low_psf_arcsec'], imsize=low_imsize, cellsize=o['low_cell'], options=o, colname=colname, automask=True, automask_threshold=15, smooth=True, cubemode=True, conditional_clearcache=True) external_mask = 'bootstrap_external_mask.fits' make_external_mask(external_mask, 'image_bootstrap_' + obsid + '_init.dirty.fits', use_tgss=True, clobber=False, cellsize='low_cell', options=o) # Deep SSD clean with this external mask and automasking ddf_image('image_bootstrap_' + obsid, 'temp_mslist.txt', cleanmask=external_mask, reuse_psf=True, reuse_dirty=True, cleanmode='SSD', ddsols='DDS0', applysols='P', majorcycles=5, robust=low_robust, uvrange=low_uvrange, beamsize=o['low_psf_arcsec'], imsize=low_imsize, cellsize=o['low_cell'], options=o, colname=colname, automask=True, automask_threshold=15, smooth=True, cubemode=True, conditional_clearcache=False) if os.path.isfile('image_bootstrap_' + obsid + '.cube.int.restored.pybdsm.srl'): warn('Source list exists, skipping source extraction') else: warn('Running PyBDSM, please wait...') img = bdsm.process_image('image_bootstrap_' + obsid + '.cube.int.restored.fits', thresh_pix=5, rms_map=True, atrous_do=True, atrous_jmax=2, group_by_isl=True, rms_box=(80, 20), adaptive_rms_box=True, adaptive_thresh=80, rms_box_bright=(35, 7), mean_map='zero', spectralindex_do=True, specind_maxchan=1, debug=True, kappa_clip=3, flagchan_rms=False, flagchan_snr=False, incl_chan=True, spline_rank=1) # Write out in ASCII to work round bug in pybdsm img.write_catalog(catalog_type='srl', format='ascii', incl_chan='true') img.export_image(img_type='rms', img_format='fits') from make_fitting_product import make_catalogue import fitting_factors import find_outliers # generate the fitting product if os.path.isfile(obsid + 'crossmatch-1.fits'): warn('Crossmatch table exists, skipping crossmatch') else: t = pt.table(omslist[0] + '/FIELD', readonly=True, ack=False) direction = t[0]['PHASE_DIR'] ra, dec = direction[0] if (ra < 0): ra += 2 * np.pi ra *= 180.0 / np.pi dec *= 180.0 / np.pi cats = zip(o['catalogues'], o['names'], o['groups'], o['radii']) make_catalogue('image_bootstrap_' + obsid + '.cube.int.restored.pybdsm.srl', ra, dec, 2.5, cats, outnameprefix=obsid) freqlist = open(obsid + 'frequencies.txt', 'w') for n, f in zip(o['names'], o['frequencies']): freqlist.write('%f %s_Total_flux %s_E_Total_flux False\n' % (f, n, n)) for i, f in enumerate(freqs): freqlist.write('%f Total_flux_ch%i E_Total_flux_ch%i True\n' % (f, i + 1, i + 1)) freqlist.close() # Now call the fitting code if os.path.isfile(obsid + 'crossmatch-results-1.npy'): warn('Results 1 exists, skipping first fit') else: fitting_factors.run_all(1, name=obsid) nreject = -1 # avoid error if we fail somewhere if os.path.isfile(obsid + 'crossmatch-2.fits'): warn('Second crossmatch exists, skipping outlier rejection') else: nreject = find_outliers.run_all(1, name=obsid) if os.path.isfile(obsid + 'crossmatch-results-2.npy'): warn('Results 2 exists, skipping second fit') else: if nreject == 0: shutil.copyfile(obsid + 'crossmatch-results-1.npy', obsid + 'crossmatch-results-2.npy') if os.path.isfile(obsid + 'crossmatch-results-2.npy'): warn('Results 2 exists, skipping first fit') else: fitting_factors.run_all(2, name=obsid) # Now apply corrections if o['full_mslist'] is None: die('Need big mslist to apply corrections') if not (o['dryrun']): warn('Applying corrections to MS list') scale = np.load(obsid + 'crossmatch-results-2.npy')[:, 0] # InterpolatedUS gives us linear interpolation between points # and extrapolation outside it spl = InterpolatedUnivariateSpline(freqs, scale, k=1) bigmslist = [s.strip() for s in open(o['full_mslist']).readlines()] obigmslist = [ms for ms in bigmslist if obsid in ms] for ms in obigmslist: t = pt.table(ms) try: dummy = t.getcoldesc('SCALED_DATA') except RuntimeError: dummy = None t.close() if dummy is not None: warn('Table ' + ms + ' has already been corrected, skipping') else: # in this version we need to scale both the original data and the data in colname t = pt.table(ms + '/SPECTRAL_WINDOW', readonly=True, ack=False) frq = t[0]['REF_FREQUENCY'] factor = spl(frq) print frq, factor t = pt.table(ms, readonly=False) desc = t.getcoldesc(o['colname']) desc['name'] = 'SCALED_DATA' t.addcols(desc) d = t.getcol(o['colname']) d *= factor t.putcol('SCALED_DATA', d) try: dummy = t.getcoldesc(colname) except RuntimeError: dummy = None if dummy is not None: desc = t.getcoldesc(colname) newname = colname + '_SCALED' desc['name'] = newname t.addcols(desc) d = t.getcol(colname) d *= factor t.putcol(newname, d) t.close() if os.path.isfile('image_bootstrap.app.mean.fits'): warn('Mean bootstrap image exists, not creating it') else: warn('Creating mean bootstrap image') hdus = [] for obsid in Uobsid: hdus.append( fits.open('image_bootstrap_' + obsid + '.app.restored.fits')) for i in range(1, len(Uobsid)): hdus[0][0].data += hdus[i][0].data hdus[0][0].data /= len(Uobsid) hdus[0].writeto('image_bootstrap.app.mean.fits')
def run_bootstrap(o): colname='DATA_DI_CORRECTED' if o['mslist'] is None: die('MS list must be specified') if o['logging'] is not None and not os.path.isdir(o['logging']): os.mkdir(o['logging']) # check the data supplied if o['frequencies'] is None or o['catalogues'] is None: die('Frequencies and catalogues options must be specified') if "DDF_PIPELINE_CATALOGS" not in os.environ.keys(): warn("You need to define the environment variable DDF_PIPELINE_CATALOGS where your catalogs are located") sys.exit(2) o["tgss"]=o["tgss"].replace("$$",os.environ["DDF_PIPELINE_CATALOGS"]) o["catalogues"]=[l.replace("$$",os.environ["DDF_PIPELINE_CATALOGS"]) for l in o["catalogues"]] lCat=o["catalogues"]+[o["tgss"]] for fCat in lCat: if not os.path.isfile(fCat): warn("Catalog %s does not exist"%fCat) sys.exit(2) cl=len(o['catalogues']) if o['names'] is None: o['names']=[os.path.basename(x).replace('.fits','') for x in o['catalogues']] if o['radii'] is None: o['radii']=[10]*cl if o['groups'] is None: o['groups']=range(cl) if (len(o['frequencies'])!=cl or len(o['radii'])!=cl or len(o['names'])!=cl or len(o['groups'])!=cl): die('Names, groups, radii and frequencies entries must be the same length as the catalogue list') low_uvrange=[o['image_uvmin'],2.5*206.0/o['low_psf_arcsec']] if o['low_imsize'] is not None: low_imsize=o['low_imsize'] # allow over-ride else: low_imsize=o['imsize']*o['cellsize']/o['low_cell'] low_robust=o['low_robust'] # Clear the shared memory run('CleanSHM.py',dryrun=o['dryrun']) # We use the individual ms in mslist. m=MSList(o['mslist']) Uobsid = set(m.obsids) for obsid in Uobsid: warn('Running bootstrap for obsid %s' % obsid) freqs=[] omslist=[] for ms,ob,f in zip(m.mss,m.obsids,m.freqs): if ob==obsid: omslist.append(ms) freqs.append(f) if len(freqs)<4: die('Not enough frequencies to bootstrap. Check your mslist or MS naming scheme') # sort to work in frequency order freqs,omslist = (list(x) for x in zip(*sorted(zip(freqs, omslist), key=lambda pair: pair[0]))) for f,ms in zip(freqs,omslist): print ms,f # generate the sorted input mslist with open('temp_mslist.txt','w') as f: for line in omslist: f.write(line+'\n') # Clean in cube mode # As for the main pipeline, first make a dirty map ddf_image('image_bootstrap_'+obsid+'_init','temp_mslist.txt', cleanmask=None,cleanmode='SSD',ddsols='DDS0', applysols='P',majorcycles=0,robust=low_robust, uvrange=low_uvrange,beamsize=o['low_psf_arcsec'], imsize=low_imsize,cellsize=o['low_cell'], options=o,colname=colname,automask=True, automask_threshold=15,smooth=True,cubemode=True, conditional_clearcache=True) external_mask='bootstrap_external_mask.fits' make_external_mask(external_mask,'image_bootstrap_'+obsid+'_init.dirty.fits',use_tgss=True,clobber=False,cellsize='low_cell',options=o) # Deep SSD clean with this external mask and automasking ddf_image('image_bootstrap_'+obsid,'temp_mslist.txt', cleanmask=external_mask,reuse_psf=True,reuse_dirty=True, cleanmode='SSD',ddsols='DDS0',applysols='P', majorcycles=5,robust=low_robust,uvrange=low_uvrange, beamsize=o['low_psf_arcsec'],imsize=low_imsize, cellsize=o['low_cell'],options=o, colname=colname,automask=True, automask_threshold=15,smooth=True,cubemode=True, conditional_clearcache=False) if os.path.isfile('image_bootstrap_'+obsid+'.cube.int.restored.pybdsm.srl'): warn('Source list exists, skipping source extraction') else: warn('Running PyBDSM, please wait...') img=bdsm.process_image('image_bootstrap_'+obsid+'.cube.int.restored.fits',thresh_pix=5,rms_map=True,atrous_do=True,atrous_jmax=2,group_by_isl=True,rms_box=(80,20), adaptive_rms_box=True, adaptive_thresh=80, rms_box_bright=(35,7),mean_map='zero',spectralindex_do=True,specind_maxchan=1,debug=True,kappa_clip=3,flagchan_rms=False,flagchan_snr=False,incl_chan=True,spline_rank=1) # Write out in ASCII to work round bug in pybdsm img.write_catalog(catalog_type='srl',format='ascii',incl_chan='true') img.export_image(img_type='rms',img_format='fits') from make_fitting_product import make_catalogue import fitting_factors import find_outliers # generate the fitting product if os.path.isfile(obsid+'crossmatch-1.fits'): warn('Crossmatch table exists, skipping crossmatch') else: t = pt.table(omslist[0]+ '/FIELD', readonly=True, ack=False) direction = t[0]['PHASE_DIR'] ra, dec = direction[0] if (ra<0): ra+=2*np.pi ra*=180.0/np.pi dec*=180.0/np.pi cats=zip(o['catalogues'],o['names'],o['groups'],o['radii']) make_catalogue('image_bootstrap_'+obsid+'.cube.int.restored.pybdsm.srl',ra,dec,2.5,cats,outnameprefix=obsid) freqlist=open(obsid+'frequencies.txt','w') for n,f in zip(o['names'],o['frequencies']): freqlist.write('%f %s_Total_flux %s_E_Total_flux False\n' % (f,n,n)) for i,f in enumerate(freqs): freqlist.write('%f Total_flux_ch%i E_Total_flux_ch%i True\n' % (f,i+1,i+1)) freqlist.close() # Now call the fitting code if os.path.isfile(obsid+'crossmatch-results-1.npy'): warn('Results 1 exists, skipping first fit') else: fitting_factors.run_all(1, name=obsid) nreject=-1 # avoid error if we fail somewhere if os.path.isfile(obsid+'crossmatch-2.fits'): warn('Second crossmatch exists, skipping outlier rejection') else: nreject=find_outliers.run_all(1, name=obsid) if os.path.isfile(obsid+'crossmatch-results-2.npy'): warn('Results 2 exists, skipping second fit') else: if nreject==0: shutil.copyfile(obsid+'crossmatch-results-1.npy',obsid+'crossmatch-results-2.npy') if os.path.isfile(obsid+'crossmatch-results-2.npy'): warn('Results 2 exists, skipping first fit') else: fitting_factors.run_all(2, name=obsid) # Now apply corrections if o['full_mslist'] is None: die('Need big mslist to apply corrections') if not(o['dryrun']): warn('Applying corrections to MS list') scale=np.load(obsid+'crossmatch-results-2.npy')[:,0] # InterpolatedUS gives us linear interpolation between points # and extrapolation outside it spl = InterpolatedUnivariateSpline(freqs, scale, k=1) bigmslist=[s.strip() for s in open(o['full_mslist']).readlines()] obigmslist = [ms for ms in bigmslist if obsid in ms] for ms in obigmslist: t = pt.table(ms) try: dummy=t.getcoldesc('SCALED_DATA') except RuntimeError: dummy=None t.close() if dummy is not None: warn('Table '+ms+' has already been corrected, skipping') else: # in this version we need to scale both the original data and the data in colname t = pt.table(ms+'/SPECTRAL_WINDOW', readonly=True, ack=False) frq=t[0]['REF_FREQUENCY'] factor=spl(frq) print frq,factor t=pt.table(ms,readonly=False) desc=t.getcoldesc(o['colname']) desc['name']='SCALED_DATA' t.addcols(desc) d=t.getcol(o['colname']) d*=factor t.putcol('SCALED_DATA',d) try: dummy=t.getcoldesc(colname) except RuntimeError: dummy=None if dummy is not None: desc=t.getcoldesc(colname) newname=colname+'_SCALED' desc['name']=newname t.addcols(desc) d=t.getcol(colname) d*=factor t.putcol(newname,d) t.close() if os.path.isfile('image_bootstrap.app.mean.fits'): warn('Mean bootstrap image exists, not creating it') else: warn('Creating mean bootstrap image') hdus=[] for obsid in Uobsid: hdus.append(fits.open('image_bootstrap_'+obsid+'.app.restored.fits')) for i in range(1,len(Uobsid)): hdus[0][0].data+=hdus[i][0].data hdus[0][0].data/=len(Uobsid) hdus[0].writeto('image_bootstrap.app.mean.fits')
def run_bootstrap(o): if o['mslist'] is None: die('MS list must be specified') if o['logging'] is not None and not os.path.isdir(o['logging']): os.mkdir(o['logging']) # check the data supplied if o['frequencies'] is None or o['catalogues'] is None: die('Frequencies and catalogues options must be specified') cl=len(o['catalogues']) if o['names'] is None: o['names']=[os.path.basename(x).replace('.fits','') for x in o['catalogues']] if o['radii'] is None: o['radii']=[10]*cl if o['groups'] is None: o['groups']=range(cl) if (len(o['frequencies'])!=cl or len(o['radii'])!=cl or len(o['names'])!=cl or len(o['groups'])!=cl): die('Names, groups, radii and frequencies entries must be the same length as the catalogue list') low_robust=-0.25 low_uvrange=[0.1,25.0] # Clear the shared memory run('CleanSHM.py',dryrun=o['dryrun']) # We use the individual ms in mslist. mslist=[s.strip() for s in open(o['mslist']).readlines()] obsids = [os.path.basename(ms).split('_')[0] for ms in mslist] Uobsid = set(obsids) for obsid in Uobsid: warn('Running bootstrap for obsid %s' % obsid) # Get the frequencies -- need to take this from the MSs omslist = [ms for ms in mslist if obsid in ms] freqs=[] for ms in omslist: t = pt.table(ms+'/SPECTRAL_WINDOW', readonly=True, ack=False) freqs.append(t[0]['REF_FREQUENCY']) # sort to work in frequency order freqs,omslist = (list(x) for x in zip(*sorted(zip(freqs, omslist), key=lambda pair: pair[0]))) for f,m in zip(freqs,omslist): print m,f # Clean in cube mode with open('temp_mslist.txt','w') as f: for line in omslist: f.write(line+'\n') ddf_image('image_bootstrap_'+obsid,'temp_mslist.txt',cleanmode='SSD',ddsols='killms_p1',applysols='P',majorcycles=4,robust=low_robust,uvrange=low_uvrange,beamsize=20,imsize=o['bsimsize'],cellsize=o['bscell'],options=o,colname=o['colname'],automask=True,automask_threshold=15,smooth=True,cubemode=True) if os.path.isfile('image_bootstrap_'+obsid+'.cube.int.restored.pybdsm.srl'): warn('Source list exists, skipping source extraction') else: warn('Running PyBDSM, please wait...') img=bdsm.process_image('image_bootstrap_'+obsid+'.cube.int.restored.fits',thresh_pix=5,rms_map=True,atrous_do=True,atrous_jmax=2,group_by_isl=True,rms_box=(80,20), adaptive_rms_box=True, adaptive_thresh=80, rms_box_bright=(35,7),mean_map='zero',spectralindex_do=True,specind_maxchan=1,debug=True,kappa_clip=3,flagchan_rms=False,flagchan_snr=False,incl_chan=True,spline_rank=1) # Write out in ASCII to work round bug in pybdsm img.write_catalog(catalog_type='srl',format='ascii',incl_chan='true') img.export_image(img_type='rms',img_format='fits') from make_fitting_product import make_catalogue import fitting_factors import find_outliers # generate the fitting product if os.path.isfile(obsid+'crossmatch-1.fits'): warn('Crossmatch table exists, skipping crossmatch') else: t = pt.table(omslist[0]+ '/FIELD', readonly=True, ack=False) direction = t[0]['PHASE_DIR'] ra, dec = direction[0] if (ra<0): ra+=2*np.pi ra*=180.0/np.pi dec*=180.0/np.pi cats=zip(o['catalogues'],o['names'],o['groups'],o['radii']) make_catalogue('image_bootstrap_'+obsid+'.cube.int.restored.pybdsm.srl',ra,dec,2.5,cats,outnameprefix=obsid) freqlist=open(obsid+'frequencies.txt','w') for n,f in zip(o['names'],o['frequencies']): freqlist.write('%f %s_Total_flux %s_E_Total_flux False\n' % (f,n,n)) for i,f in enumerate(freqs): freqlist.write('%f Total_flux_ch%i E_Total_flux_ch%i True\n' % (f,i+1,i+1)) freqlist.close() # Now call the fitting code if os.path.isfile(obsid+'crossmatch-results-1.npy'): warn('Results 1 exists, skipping first fit') else: fitting_factors.run_all(1, name=obsid) nreject=-1 # avoid error if we fail somewhere if os.path.isfile(obsid+'crossmatch-2.fits'): warn('Second crossmatch exists, skipping outlier rejection') else: nreject=find_outliers.run_all(1, name=obsid) if os.path.isfile(obsid+'crossmatch-results-2.npy'): warn('Results 2 exists, skipping second fit') else: if nreject==0: shutil.copyfile(obsid+'crossmatch-results-1.npy',obsid+'crossmatch-results-2.npy') if os.path.isfile(obsid+'crossmatch-results-2.npy'): warn('Results 2 exists, skipping first fit') else: fitting_factors.run_all(2, name=obsid) # Now apply corrections if o['full_mslist'] is None: die('Need big mslist to apply corrections') if not(o['dryrun']): warn('Applying corrections to MS list') scale=np.load(obsid+'crossmatch-results-2.npy')[:,0] # InterpolatedUS gives us linear interpolation between points # and extrapolation outside it spl = InterpolatedUnivariateSpline(freqs, scale, k=1) bigmslist=[s.strip() for s in open(o['full_mslist']).readlines()] obigmslist = [ms for ms in bigmslist if obsid in ms] for ms in obigmslist: t = pt.table(ms) try: dummy=t.getcoldesc('SCALED_DATA') except RuntimeError: dummy=None t.close() if dummy is not None: warn('Table '+ms+' has already been corrected, skipping') else: t = pt.table(ms+'/SPECTRAL_WINDOW', readonly=True, ack=False) frq=t[0]['REF_FREQUENCY'] factor=spl(frq) print frq,factor t=pt.table(ms,readonly=False) desc=t.getcoldesc(o['colname']) desc['name']='SCALED_DATA' t.addcols(desc) d=t.getcol(o['colname']) d*=factor t.putcol('SCALED_DATA',d) t.close() if os.path.isfile('image_bootstrap.app.mean.fits'): warn('Mean bootstrap image exists, not creating it') else: warn('Creating mean bootstrap image') hdus=[] for obsid in Uobsid: hdus.append(fits.open('image_bootstrap_'+obsid+'.app.restored.fits')) for i in range(1,len(Uobsid)): hdus[0][0].data+=hdus[i][0].data hdus[0][0].data/=len(Uobsid) hdus[0].writeto('image_bootstrap.app.mean.fits')
def compress_fits(filename,q): command='fpack -q %i %s' % (q,filename) run(command)