def selectCC(self, checkBeam=True, keepInBeam=True, maskname=None): """ remove cc from a skymodel according to masks checkBeam: remove according to beam (see keepInBeam) keepInBeam: if beamReg is present and is True: remove sources outside beam if beamReg is present and is False: remove source inside beam maskname: a possible mask, otherwise try to use standard """ if maskname is None: maskname = self.maskname if not os.path.exists(maskname): raise("Missing mask in selectCC: %s." % maskname) if checkBeam: if self.beamReg is None: raise('Missing beam in selectCC.') logger.info('Predict (apply beam reg %s)...' % self.beamReg) blank_image_reg(maskname, self.beamReg, inverse=keepInBeam, blankval=0) # if keep_in_beam set to 0 everything outside beam.reg # apply mask logger.info('%s: Apply mask (%s) on skymodel...' % (self.imagename,maskname)) lsm = lsmtool.load(self.skymodel) lsm.select('%s == True' % maskname) lsm.group('single') # group to 1 patch lsm.write(self.skymodel_cut, format = 'makesourcedb', clobber=True) del lsm # convert from txt to blob logger.info('%s: Make skydb...' % self.imagename) lib_util.check_rm(self.skydb) os.system('makesourcedb outtype="blob" format="<" in="'+self.skymodel_cut+'" out="'+self.skydb+'"')
def selectCC(self, keepInBeam=True): """ remove cc from a skymodel according to masks keepInBeam: if beamReg is present and is True: remove sources outside beam if beamReg is present and is False: remove source inside beam """ self.makeMask() if self.facetReg is not None: logger.info('Predict (apply facet reg %s)...' % self.facetReg) blank_image_reg(self.maskname, self.facetReg, inverse=True, blankval=0) # set to 0 pixels outside facet mask if self.beamReg is not None: logger.info('Predict (apply beam reg %s)...' % self.beamReg) blank_image_reg(self.maskname, self.beamReg, inverse=keepInBeam, blankval=0) # if keep_in_beam set to 0 everything outside beam.reg # apply mask logger.info('%s: Apply mask on skymodel...' % self.imagename) lsm = lsmtool.load(self.skymodel) lsm.select('%s == True' % self.maskname) lsm.group('single') # group to 1 patch lsm.write(self.skymodel_cut, format = 'makesourcedb', clobber=True) del lsm # convert from txt to blob logger.info('%s: Make skydb...' % self.imagename) lib_util.check_rm(self.skydb) os.system('makesourcedb outtype="blob" format="<" in="'+self.skymodel_cut+'" out="'+self.skydb+'"')
def makeBeamReg(self, outfile, pb_cut=None, to_null=False, freq='mid'): """ Create a ds9 region of the beam outfile : str output file pb_cut : float, optional diameter of the beam to_null : bool, optional arrive to the first null, not the FWHM freq: min,max,med which frequency to use to estimate the beam size """ logger.debug('Making PB region: '+outfile) ra, dec = self.getPhaseCentre() if pb_cut is None: radius = self.getFWHM(freq=freq)/2. else: radius = pb_cut/2. if to_null: radius *= 2 # rough estimation s = Shape('circle', None) s.coord_format = 'fk5' s.coord_list = [ ra, dec, radius ] # ra, dec, radius s.coord_format = 'fk5' s.attr = ([], {'width': '2', 'point': 'cross', 'font': '"helvetica 16 normal roman"'}) s.comment = 'color=red text="beam"' regions = pyregion.ShapeList([s]) lib_util.check_rm(outfile) regions.write(outfile)
def clean(p, MSs, size=2., apply_beam=False): """ p = patch name mss = list of mss to clean size = in deg of the image """ # set pixscale and imsize pixscale = MSs.getListObj()[0].getResolution() / 3. imsize = int(size / (pixscale / 3600.)) if imsize < 512: imsize = 512 if imsize % 2 == 1: imsize += 1 # make even logger.debug('Image size: ' + str(imsize) + ' - Pixel scale: ' + str(pixscale)) if apply_beam: idg_parms = '-use-idg -grid-with-beam -use-differential-lofar-beam -beam-aterm-update 400' else: idg_params = '-use-idg' # clean 1 logger.info('Cleaning (' + str(p) + ')...') imagename = 'img/ddcal-' + str(p) s.add('wsclean -reorder -temp-dir /dev/shm -name ' + imagename + ' -size '+str(imsize)+' '+str(imsize)+' -j '+str(s.max_processors)+' \ -scale ' +str(pixscale)+'arcsec -weight briggs -0.5 -niter 100000 -no-update-model-required -minuv-l 30 -mgain 0.85 -clean-border 1 \ -auto-threshold 20 ' +idg_parms+' \ -join-channels -fit-spectral-pol 2 -channels-out 10 -save-source-list ' +MSs.getStrWsclean(), \ log='wsclean-'+str(p)+'.log', commandType='wsclean', processors='max') s.run(check=True) # make mask im = lib_img.Image(imagename + '-MFS-image.fits', userReg=userReg) os.system('mv %s %s' % (im.skymodel, im.skymodel + '-first')) # copy the source list im.makeMask(threshisl=3) # clean 2 #-multiscale -multiscale-scale-bias 0.5 \ #-auto-mask 3 -rms-background-window 40 -rms-background-method rms-with-min \ logger.info('Cleaning w/ mask (' + str(p) + ')...') s.add('wsclean -continue -reorder -temp-dir /dev/shm -name ' + imagename + ' -size '+str(imsize)+' '+str(imsize)+' -j '+str(s.max_processors)+' \ -scale ' +str(pixscale)+'arcsec -weight briggs -0.5 -niter 1000000 -no-update-model-required -minuv-l 30 -mgain 0.85 -clean-border 1 \ -auto-threshold 0.1 -fits-mask ' +im.maskname+' '+idg_parms+' \ -join-channels -fit-spectral-pol 2 -channels-out 10 -save-source-list ' +MSs.getStrWsclean(), \ log='wscleanM-'+str(p)+'.log', commandType='wsclean', processors='max') s.run(check=True) os.system('cat logs/wscleanM-' + str(p) + '.log | grep "background noise"') os.system('grep -v \'^Format\' %s >> %s' % (im.skymodel + '-first', im.skymodel)) # merge the source lists lib_util.check_rm(im.skymodel + '-first') return imagename
def move(self, pathMSNew, overwrite=False, keepOrig=False): """ Move (or rename) the MS to another locus in the file system. """ logger.debug('Move: '+self.pathMS+' -> '+pathMSNew) if overwrite == True: lib_util.check_rm(pathMSNew) if not os.path.exists(pathMSNew): if keepOrig: shutil.copytree(self.pathMS, pathMSNew) else: shutil.move(self.pathMS, pathMSNew) self.setPathVariables(pathMSNew)
def cut_skymodel(skymodel_in, skymodel_out, d, do_skydb=True, do_regions=False): """ Load full skymodel and extract sources in the square around the calibrator of the given size """ lsm = lsmtool.load(skymodel_in) # select all sources within a sqare of patch size lsm.select('Ra > %f' % (d.position[0]-(d.size/2)/np.cos(d.position[1]* np.pi / 180.))) lsm.select('Ra < %f' % (d.position[0]+(d.size/2)/np.cos(d.position[1]* np.pi / 180.))) lsm.select('Dec > %f' % (d.position[1]-d.size/2)) lsm.select('Dec < %f' % (d.position[1]+d.size/2)) if do_regions: lsm.write('ddcal/masks/regions-c%02i/%s.reg' % (cmaj,d.name), format='ds9', clobber=True) lsm.write(dir_skymodel, format='makesourcedb', clobber=True) lib_util.check_rm(dir_skydb) s.add('makesourcedb outtype="blob" format="<" in="%s" out="%s"' % (dir_skymodel, dir_skydb), log='makesourcedb_cl.log', commandType='general' ) s.run(check=True)
def set_region(self, loc): """ Creates a ds9 regionfile that covers the DD-cal model """ assert self.size is not None and self.position is not None # we need this to be already set self.region_file = loc+'/'+self.name+'.reg' s = Shape('circle', None) s.coord_format = 'fk5' s.coord_list = [ self.position[0], self.position[1], self.size/2. ] # ra, dec, radius s.coord_format = 'fk5' s.attr = ([], {'width': '2', 'point': 'cross', 'font': '"helvetica 16 normal roman"'}) s.comment = 'color=red text="%s"' % self.name regions = pyregion.ShapeList([s]) lib_util.check_rm(self.region_file) regions.write(self.region_file)
sys.exit(1) if avg_factor_f < 1 or keep_IS: avg_factor_f = 1 avg_factor_t = int(np.round(4 / timeint)) # to 4 sec if avg_factor_t < 1 or keep_IS: avg_factor_t = 1 MSout = getName(MS.pathMS) flog.write(MS.nameMS + '.MS\n') if avg_factor_f != 1 or avg_factor_t != 1: logger.info( '%s->%s: Average in freq (factor of %i) and time (factor of %i)...' % (MS.nameMS, MSout, avg_factor_f, avg_factor_t)) if keep_IS: s.add('DPPP '+parset_dir+'/DPPP-avg.parset msin='+MS.pathMS+' msout='+MSout+' msin.datacolumn=DATA \ avg.timestep=' +str(avg_factor_t)+' avg.freqstep='+str(avg_factor_f), \ log=MS.nameMS+'_avg.log', commandType='DPPP') else: s.add('DPPP '+parset_dir+'/DPPP-avg.parset msin='+MS.pathMS+' msout='+MSout+' msin.datacolumn=DATA \ msin.baseline="[CR]S*&" \ avg.timestep=' +str(avg_factor_t)+' avg.freqstep='+str(avg_factor_f), \ log=MS.nameMS+'_avg.log', commandType='DPPP') s.run(check=True, maxThreads=1) # limit threads to prevent I/O isssues lib_util.check_rm(MS.pathMS) else: logger.info('%s->%s: Move data - no averaging...' % (MS.nameMS, MSout)) MS.move(MSout) logger.info("Done.")
""" for cal_dir in sorted(glob.glob('../../cals/*/*')): cal_timestamps = set( [ms.split('_')[1][1:] for ms in glob.glob(cal_dir + '/cals/*MS')]) if timestamp in cal_timestamps: logger.info('Calibrator found: %s (t=%s)' % (cal_dir, timestamp)) return cal_dir + '/cals' logger.error('Missing calibrator.') sys.exit() ########################################################## with w.if_todo('setup'): logger.info('Cleaning...') lib_util.check_rm('cal*h5') lib_util.check_rm('plots*') lib_util.check_rm('peel*') lib_util.check_rm('img') os.makedirs('img') MSs = lib_ms.AllMSs(glob.glob(data_dir + '/*MS'), s, check_flags=False) for timestamp in set( [os.path.basename(ms).split('_')[1][1:] for ms in MSs.getListStr()]): mss_toconcat = sorted( glob.glob(data_dir + '/' + target + '_t' + timestamp + '_SB*.MS')) MS_concat = target + '_t' + timestamp + '_concat.MS' MS_concat_bkp = target + '_t' + timestamp + '_concat.MS-bkp' if os.path.exists(MS_concat_bkp): logger.info('Restoring bkp data: %s...' % MS_concat_bkp)
cor.ph.correction=phaseOrig000 cor.amp.correction=amplitude000 cor.amp.updateweights=False', log='$nameMS_corIONO.log', commandType="DPPP") logger.info('Subtract model...') MSs.run( 'taql "update $pathMS set CORRECTED_DATA = CORRECTED_DATA - MODEL_DATA"', log='$nameMS_taql2.log', commandType='general') logger.info('Flag...') MSs.run('DPPP ' + parset_dir + '/DPPP-flag2.parset msin=$pathMS', log='$nameMS_flag2.log', commandType='DPPP') lib_util.check_rm('img') os.makedirs('img') imgsizepix = MSs.getListObj()[0].getFWHM() * 3600 / 5 logger.info('Cleaning low-res...') imagename = 'img/calLR' lib_util.run_wsclean(s, 'wscleanLR.log', MSs.getStrWsclean(), name=imagename, size=imgsizepix/5, scale='60arcsec', \ weight='briggs 0.', taper_gaussian='240arcsec', niter=10000, no_update_model_required='', minuv_l=30, maxuv_l=2000, mgain=0.85, \ use_idg='', grid_with_beam='', use_differential_lofar_beam='', beam_aterm_update=400, \ parallel_deconvolution=512, \ auto_mask=10, auto_threshold=1, pol='IQUV', join_polarizations='', join_channels='', fit_spectral_pol=3, channels_out=9, deconvolution_channels=3) logger.info('Cleaning normal...') imagename = 'img/cal' lib_util.run_wsclean(s, 'wscleanA.log', MSs.getStrWsclean(), name=imagename, size=imgsizepix, scale='5arcsec', \
ateams_todemix = [] for ateam in ateams: sep = MSs.getListObj()[0].distBrightSource(ateam) if sep < 4 or sep > 15: logger.debug('No demix of %s (sep: %.0f deg)' % (ateam, sep)) else: ateams_todemix.append(ateam) logger.warning('Demix of %s (sep: %.0f deg)' % (ateam, sep)) if len(ateams_todemix) < 0: logger.info('Notingh to demix.') sys.exit() if os.path.exists('mss-predemix'): logger.warning('Reset mss...') lib_util.check_rm('mss/*MS') else: logger.info('Move mss in mss-predemix...') os.system('mv mss mss-predemix') os.system('mkdir mss') MSs = lib_ms.AllMSs(glob.glob('mss-predemix/TC*[0-9].MS'), s) if include_target: if not os.path.exists('demix_combined.skymodel'): logger.info('Including target...') fwhm = MSs.getListObj()[0].getFWHM(freq='min') phasecentre = MSs.getListObj()[0].getPhaseCentre() radeg = phasecentre[0] decdeg = phasecentre[1] # get model the size of the image (radius=fwhm/2)
# parse parset parset = lib_util.getParset() parset_dir = parset.get('LOFAR_extract', 'parset_dir') maxniter = parset.getint('LOFAR_extract', 'maxniter') target_reg_file = parset.get('LOFAR_extract', 'extractRegion') # default 'target.reg' phSolMode = parset.get('LOFAR_extract', 'phSolMode') # default: tecandphase if phSolMode not in ['tecandphase', 'phase']: logger.error('phSolMode {} not supported. Choose tecandphase, phase.') sys.exit() userReg = parset.get('model', 'userReg') ############################ with w.if_todo('cleaning'): logger.info('Cleaning...') lib_util.check_rm('extract') lib_util.check_rm('img') os.makedirs('img') os.makedirs('extract/init') os.system('cp ddcal/c01/images/wideDD-c01.app.restored.fits extract/init/' ) # copy ddcal image os.system('cp ddcal/c01/images/wideDD-c01.DicoModel extract/init/' ) # copy dico model os.system( 'cp ddcal/c01/solutions/interp.h5 extract/init/') # copy fnal dde sol lib_util.check_rm('mss-extract') if not os.path.exists('mss-extract'): logger.info('Copy MS...') os.system('cp -r mss-avg mss-extract') # region must be a list of ds9 circles and polygons (other shapes can be implemented in lib_util.Rgion_helper()
import numpy as np ############################################### from LiLF import lib_ms, lib_util, lib_log logger_obj = lib_log.Logger('pipeline-demix.logger') logger = lib_log.logger s = lib_util.Scheduler(log_dir=logger_obj.log_dir, dry=False) # parse parset parset = lib_util.getParset() parset_dir = parset.get('LOFAR_demix', 'parset_dir') data_dir = parset.get('LOFAR_demix', 'data_dir') skydb = parset.get('LOFAR_demix', 'demix_model') ############################################## # Demix MSs = lib_ms.AllMSs(glob.glob(data_dir + '/*MS'), s) for MS in MSs.getListStr(): lib_util.check_rm(os.path.basename(MS) + '_' + os.path.basename(skydb)) print(('cp -r ' + skydb + ' ' + os.path.basename(MS) + '_' + os.path.basename(skydb))) os.system('cp -r ' + skydb + ' ' + os.path.basename(MS) + '_' + os.path.basename(skydb)) logger.info('Demixing...') MSs.run('DPPP '+parset_dir+'/DPPP_demix.parset msin=$pathMS msout=$nameMS demixer.skymodel=$nameMS.MS_'+os.path.basename(skydb)+' demixer.instrumentmodel=$nameMS/instrument_demix', \ log='$nameMS_demix.log', commandType='DPPP') logger.info("Done.")
""" for cal_dir in sorted(glob.glob('../../cals/*/*')): cal_timestamps = set( [ms.split('_')[1][1:] for ms in glob.glob(cal_dir + '/cals/*MS')]) if timestamp in cal_timestamps: logger.info('Calibrator found: %s (t=%s)' % (cal_dir, timestamp)) return cal_dir + '/cals' logger.error('Missing calibrator.') sys.exit() ########################################################## if w.todo('setup'): logger.info('Cleaning...') lib_util.check_rm('cal*h5') lib_util.check_rm('plots*') lib_util.check_rm('img') os.makedirs('img') MSs = lib_ms.AllMSs(glob.glob(data_dir + '/*MS'), s, check_flags=False) for timestamp in set( [os.path.basename(ms).split('_')[1][1:] for ms in MSs.getListStr()]): mss_toconcat = sorted( glob.glob(data_dir + '/' + target + '_t' + timestamp + '_SB*.MS')) MS_concat = target + '_t' + timestamp + '_concat.MS' MS_concat_bkp = target + '_t' + timestamp + '_concat.MS-bkp' if os.path.exists(MS_concat_bkp): logger.info('Restoring bkp data...') lib_util.check_rm(MS_concat)
# parse parset parset = lib_util.getParset() parset_dir = parset.get('LOFAR_timesplit', 'parset_dir') data_dir = parset.get('LOFAR_timesplit', 'data_dir') cal_dir = parset.get('LOFAR_timesplit', 'cal_dir') ngroups = parset.getint('LOFAR_timesplit', 'ngroups') initc = parset.getint( 'LOFAR_timesplit', 'initc') # initial tc num (useful for multiple observation of same target) bl2flag = parset.get('flag', 'stations') ################################################# # Clean with w.if_todo('clean'): logger.info('Cleaning...') lib_util.check_rm('mss*') ### DONE with w.if_todo('copy'): MSs = lib_ms.AllMSs(glob.glob(data_dir + '/*MS'), s) logger.info('Copy data...') for MS in MSs.getListObj(): #if min(MS.getFreqs()) > 30.e6: # overwrite=True to prevent updating the weights twice MS.move(MS.nameMS + '.MS', keepOrig=True, overwrite=True) ### DONE MSs = lib_ms.AllMSs(glob.glob('*MS'), s) ##################################################
logger_obj = lib_log.Logger('pipeline-self.logger') logger = lib_log.logger s = lib_util.Scheduler(log_dir=logger_obj.log_dir, dry=False) w = lib_util.Walker('pipeline-self.walker') parset = lib_util.getParset() parset_dir = parset.get('LOFAR_self', 'parset_dir') sourcedb = parset.get('model', 'sourcedb') apparent = parset.getboolean('model', 'apparent') userReg = parset.get('model', 'userReg') ############################################################################# # Clear if w.todo('cleaning'): logger.info('Cleaning...') lib_util.check_rm('img') os.makedirs('img') # here images, models, solutions for each group will be saved lib_util.check_rm('self') if not os.path.exists('self/images'): os.makedirs('self/images') if not os.path.exists('self/solutions'): os.makedirs('self/solutions') if not os.path.exists('self/plots'): os.makedirs('self/plots') w.done('cleaning') ### DONE MSs = lib_ms.AllMSs(glob.glob('mss/TC*[0-9].MS'), s) try: MSs.print_HAcov() except:
else: lib_util.run_wsclean(s, 'wscleanB-'+str(p)+'.log', MSs.getStrWsclean(), name=imagename, size=imsize, save_source_list='', scale=str(pixscale)+'arcsec', \ weight=weight, niter=100000, no_update_model_required='', minuv_l=30, maxuv_l=maxuv_l, mgain=0.85, \ multiscale='', multiscale_scales='0,10,20,40,80', baseline_averaging='', parallel_deconvolution=512, local_rms='', auto_threshold=0.75, auto_mask=1.5, fits_mask=im.maskname, \ join_channels='', fit_spectral_pol=3, channels_out=9, deconvolution_channels=3) os.system('cat logs/wscleanA-' + str(p) + '.log logs/wscleanB-' + str(p) + '.log | grep "background noise"') ############################################################# with w.if_todo('cleaning'): logger.info('Cleaning...') lib_util.check_rm('ddcal') os.makedirs('ddcal/masks') os.makedirs('ddcal/plots') os.makedirs('ddcal/images') os.makedirs('ddcal/solutions') os.makedirs('ddcal/skymodels') ### DONE ############################################################ # use SUBTRACTED_DATA (no pre-correction - subtraction would not work) or CORRECTED_DATA (DIE iono correction)? logger.info('Copy data...') if not os.path.exists('mss-dd'): os.makedirs('mss-dd') MSs_self.run('DPPP '+parset_dir+'/DPPP-avg.parset msin=$pathMS msout=mss-dd/$nameMS.MS msin.datacolumn=CORRECTED_DATA avg.freqstep=1 avg.timestep=1', \ log='$nameMS_avg.log', commandType='DPPP')
parset = lib_util.getParset() parset_dir = parset.get('LOFAR_dd-incr', 'parset_dir') maxniter = parset.getint('LOFAR_dd', 'maxniter') calFlux = parset.getfloat('LOFAR_dd', 'calFlux') userReg = parset.get('model', 'userReg') #################################################### MSs_self = lib_ms.AllMSs(glob.glob('mss/TC*[0-9].MS'), s) # make beam phasecentre = MSs_self.getListObj()[0].getPhaseCentre() fwhm = MSs_self.getListObj()[0].getFWHM(freq='mid') ############################ logger.info('Cleaning...') lib_util.check_rm('ddcal') os.makedirs('ddcal/masks') os.makedirs('ddcal/plots') os.makedirs('ddcal/images') os.makedirs('ddcal/solutions') os.makedirs('ddcal/skymodels') def clean(p, MSs, size, res='normal', apply_beam=False): """ p = patch name mss = list of mss to clean size = in deg of the image """ # set pixscale and imsize pixscale = MSs.getListObj()[0].getResolution()
dry=False) #, maxThreads = 4) w = lib_util.Walker('pipeline-facet-self.walker') # parse parset parset = lib_util.getParset() parset_dir = parset.get('LOFAR_facet_self', 'parset_dir') maxniter = parset.getint('LOFAR_facet_self', 'maxniter') userReg = parset.get('model', 'userReg') mosaic_image = lib_img.Image('ddcal/images/c%02i/mos-MFS-image.fits' % lastcycle) ############################ if w.todo('cleaning'): logger.info('Cleaning...') lib_util.check_rm('plot*') lib_util.check_rm('cal*h5') lib_util.check_rm('img') lib_util.check_rm('facet') os.makedirs('img') os.makedirs('facet') lib_util.check_rm('mss-facet') if not os.path.exists('mss-facet'): os.system('cp -r mss-dd mss-facet') w.done('cleaning') ### DONE MSs = lib_ms.AllMSs(glob.glob('mss-facet/*MS'), s) def clean(p, MSs, size, res='normal', apply_beam=False):
######################################################## from LiLF import lib_ms, lib_img, lib_util, lib_log logger_obj = lib_log.Logger('pipeline-self.logger') logger = lib_log.logger s = lib_util.Scheduler(log_dir=logger_obj.log_dir, dry=False) parset = lib_util.getParset() parset_dir = parset.get('LOFAR_self', 'parset_dir') sourcedb = parset.get('model', 'sourcedb') apparent = parset.getboolean('model', 'apparent') userReg = parset.get('model', 'userReg') ############################################################################# # Clear logger.info('Cleaning...') lib_util.check_rm('img') os.makedirs('img') # here images, models, solutions for each group will be saved lib_util.check_rm('self') if not os.path.exists('self/images'): os.makedirs('self/images') if not os.path.exists('self/solutions'): os.makedirs('self/solutions') if not os.path.exists('self/plots'): os.makedirs('self/plots') lib_util.check_rm('img') os.makedirs('img') MSs = lib_ms.AllMSs(glob.glob('mss/TC*[0-9].MS'), s) try: MSs.plot_HAcov('HAcov.png') except: logger.error('Problem with HAcov, continue anyway.')
imagename = 'img/ddcalM-' + str(p) lib_util.run_wsclean(s, 'wscleanB-'+str(p)+'.log', MSs.getStrWsclean(), name=imagename, do_predict=True, \ size=imsize, save_source_list='', scale=str(pixscale)+'arcsec', \ weight=weight, niter=100000, no_update_model_required='', minuv_l=30, maxuv_l=maxuv_l, mgain=0.85, \ multiscale='', multiscale_scale_bias=0.65, multiscale_scales='0,10,20,40,80', baseline_averaging='', parallel_deconvolution=512, local_rms='', auto_threshold=0.75, auto_mask=1.5, fits_mask=im.maskname, \ join_channels='', fit_spectral_pol=3, channels_out=ch_out) #, deconvolution_channels=3) os.system('cat logs/wscleanA-' + str(p) + '.log logs/wscleanB-' + str(p) + '.log | grep "background noise"') ############################################################# if w.todo('cleaning'): logger.info('Cleaning...') lib_util.check_rm('ddcal') os.makedirs('ddcal/init') lib_util.check_rm('img') os.makedirs('img') w.done('cleaning') ### DONE # goes down to 8 seconds/4 chans if not os.path.exists('mss-avg'): MSs = lib_ms.AllMSs(glob.glob('mss/TC*[0-9].MS'), s) timeint = MSs.getListObj()[0].getTimeInt() avgtimeint = int(round(8 / timeint)) nchan_init = MSs.getListObj()[0].getNchan() # avg (x8) sol (x6) - we need a multiple of 8x6=48, the largest that is <nchan # survey after avg (x8): 60, final number of sol 10
logger = lib_log.logger s = lib_util.Scheduler(log_dir=logger_obj.log_dir, dry=False) parset = lib_util.getParset() parset_dir = parset.get('uGMRT_self', 'parset_dir') sourcedb = parset.get('model', 'sourcedb') # relative to tgts dir "./tgts/xxx/model.txt userReg = parset.get('model', 'userReg') # relative to tgts dir "./tgts/xxx/region.ref" MSs = lib_ms.AllMSs(glob.glob('mss/*.MS'), s) ############################################################################ # Clear logger.info('Cleaning...') lib_util.check_rm('img') os.makedirs('img') lib_util.check_rm('self') os.makedirs('self/images') os.makedirs('self/solutions') os.makedirs('self/plots') is_wideband = len( MSs.getListObj()[0].getFreqs()) > 1000 # if > 1000 chans, it is wideband # make beam phasecentre = MSs.getListObj()[0].getPhaseCentre() MSs.getListObj()[0].makeBeamReg('self/beam.reg') beamReg = 'self/beam.reg' #################################################################
local_rms='', auto_threshold=0.75, auto_mask=1.5, fits_mask=im.maskname, join_channels='', fit_spectral_pol=3, channels_out=ch_out) #, deconvolution_channels=3) os.system('cat logs/wscleanB-' + str(p) + '.log | grep "background noise"') ############################################################# with w.if_todo('cleaning'): logger.info('Cleaning...') lib_util.check_rm('ddcal') os.makedirs('ddcal/init') lib_util.check_rm('img') os.makedirs('img') lib_util.check_rm('mss-avg') ### DONE # goes down to 8 seconds/4 chans if not os.path.exists('mss-avg'): MSs = lib_ms.AllMSs(glob.glob('mss/TC*[0-9].MS'), s) timeint = MSs.getListObj()[0].getTimeInt() avgtimeint = int(round(8 / timeint)) # to 8 seconds nchan_init = MSs.getListObj()[0].getNchan() # chan: avg (x8) sol (x6) - we need a multiple of 8x6=48, the largest that is <nchan # survey after avg (x8): 60, final number of sol 10 # pointed after avg (x8): 120, final number of sol 20
s = lib_util.Scheduler(log_dir = logger_obj.log_dir, dry = False) # parse parset parset = lib_util.getParset() parset_dir = parset.get('LOFAR_timesplit','parset_dir') data_dir = parset.get('LOFAR_timesplit','data_dir') cal_dir = parset.get('LOFAR_timesplit','cal_dir') ngroups = parset.getint('LOFAR_timesplit','ngroups') initc = parset.getint('LOFAR_timesplit','initc') # initial tc num (useful for multiple observation of same target) if 'LBAsurvey' in os.getcwd(): data_dir = '/home/fdg/lofar1/LBAsurvey/%s/%s' % (os.getcwd().split('/')[-2], os.getcwd().split('/')[-1]) ################################################# # Clean logger.info('Cleaning...') lib_util.check_rm('mss*') MSs = lib_ms.AllMSs( glob.glob(data_dir+'/*MS'), s ) logger.info('Copy data...') for MS in MSs.getListObj(): if min(MS.getFreqs()) > 30.e6: # overwrite=True to prevent updating the weights twice MS.move(MS.nameMS+'.MS', keepOrig=True, overwrite=True) MSs = lib_ms.AllMSs( glob.glob('*MS'), s ) ################################################## # Find solutions to apply if 'LBAsurvey' in os.getcwd(): #cal_dir = 'portal_lei:/disks/paradata/fdg/LBAsurvey/cal_'+os.getcwd().split('/')[-2]+'_3C*/' for cal_dir in glob.glob('../3c*'):
######################################################## from LiLF import lib_ms, lib_img, lib_util, lib_log logger_obj = lib_log.Logger('pipeline-ateam.logger') logger = lib_log.logger s = lib_util.Scheduler(log_dir=logger_obj.log_dir, dry=False) # parse parset parset = lib_util.getParset() parset_dir = parset.get('LOFAR_ateam', 'parset_dir') bl2flag = parset.get('flag', 'stations') data_dir = '../tgts-bkp/' ########################################################## logger.info('Cleaning...') #lib_util.check_rm('cal*h5') lib_util.check_rm('plots*') lib_util.check_rm('img') os.makedirs('img') MSs = lib_ms.AllMSs(sorted(glob.glob(data_dir + '/*MS')), s) # copy data (avg to 1ch/sb and 10 sec) nchan = int(MSs.getListObj()[0].getNchan()) # average 4 chans timeint = MSs.getListObj()[0].getTimeInt() avg_time = int(np.rint(30. / timeint)) # average to 30 s logger.info('Copy data...') for obs in set([os.path.basename(ms).split('_')[0] for ms in MSs.getListStr()]): mss_toconcat = sorted(glob.glob(data_dir + '/' + obs + '*MS'))
w = lib_util.Walker('script-benchmark.walker') parset = lib_util.getParset() parset_dir = parset.get('LOFAR_self', 'parset_dir') parser = argparse.ArgumentParser(description="Benchmark LOFAR software.") parser.add_argument("msfiles", type=str, help="Input ms files, glob-like string") parser.add_argument("--sourcedb", default='', help="If you want to provide a certain skymodel.") args = parser.parse_args() ############################################################################# # Clear with w.if_todo('cleaning'): lib_util.check_rm('bench') if not os.path.exists('bench/img'): os.makedirs('bench/img') logger.info('Cleaning...') lib_util.check_rm('bench/img') os.makedirs('bench/img') lib_util.check_rm('bench/plots') os.makedirs('bench/plots') ### DONE print(args.msfiles) MSs = lib_ms.AllMSs(glob.glob(args.msfiles), s) sourcedb = args.sourcedb ncpu = mp.cpu_count() nms = len(MSs.getListObj()) logger.info(f'Found {ncpu} cpus and {nms} MS files')
userReg = parset.get('model', 'userReg') uvrange = [0.03, 1e6] image_robust = -0.15 ncluster = 5 #################################################### MSs_self = lib_ms.AllMSs(glob.glob('mss/TC*[0-9].MS'), s) # make beam phasecentre = MSs_self.getListObj()[0].getPhaseCentre() fwhm = MSs_self.getListObj()[0].getFWHM(freq='mid') nchan = MSs_self.getListObj()[0].getNchan() ############################ logger.info('Cleaning...') lib_util.check_rm('ddfcal') os.makedirs('ddfcal/masks') os.makedirs('ddfcal/plots') os.makedirs('ddfcal/images') os.makedirs('ddfcal/solutions') os.makedirs('ddfcal/cache') lib_util.check_rm('img') os.makedirs('img') # Clear the shared memory s.add('ClearSHM.py', log='clearSHM.log', commandType='singularity', processors='max') s.run(check=True)