# parse parset parset = lib_util.getParset() parset_dir = parset.get('cal', 'parset_dir') data_dir = parset.get('cal', 'data_dir') skymodel = parset.get('cal', 'skymodel') imaging = parset.getboolean('cal', 'imaging') bl2flag = parset.get('flag', 'stations') if 'LBAsurvey' in os.getcwd(): obs = os.getcwd().split('/')[-2] # assumes .../c??-o??/3c196 calname = os.getcwd().split('/')[-1] # assumes .../c??-o??/3c196 data_dir = '../../download/%s/%s' % (obs, calname) ############################################################# MSs = lib_ms.AllMSs(glob.glob(data_dir + '/*MS'), s) # copy data logger.info('Copy data...') for MS in MSs.getListObj(): MS.move(MS.nameMS + '.MS', keepOrig=True) MSs = lib_ms.AllMSs(glob.glob('*MS'), s) calname = MSs.getListObj()[0].getNameField() obsmode = MSs.getListObj()[0].getObsMode() if min(MSs.getFreqs()) < 40.e6: iono3rd = True logger.debug('Include iono 3rd order.') else: iono3rd = False
os.makedirs('extract/init') os.system('cp ddcal/c01/images/wideDD-c01.app.restored.fits extract/init/' ) # copy ddcal image os.system('cp ddcal/c01/images/wideDD-c01.DicoModel extract/init/' ) # copy dico model os.system( 'cp ddcal/c01/solutions/interp.h5 extract/init/') # copy fnal dde sol lib_util.check_rm('mss-extract') if not os.path.exists('mss-extract'): logger.info('Copy MS...') os.system('cp -r mss-avg mss-extract') # region must be a list of ds9 circles and polygons (other shapes can be implemented in lib_util.Rgion_helper() target_reg = lib_util.Region_helper(target_reg_file) center = target_reg.get_center() # center of the extract region MSs = lib_ms.AllMSs(glob.glob('mss-extract/*MS'), s) ch_out = MSs.getChout(4e6) # chout from dd-serial fwhm = MSs.getListObj()[0].getFWHM(freq='mid') phase_center = MSs.getListObj()[0].getPhaseCentre() # read image, h5parm, make mask wideDD_image = lib_img.Image('extract/init/wideDD-c01.app.restored.fits') dde_h5parm = 'extract/init/interp.h5' # make mask for subtraction mask_ddcal = wideDD_image.imagename.replace( '.fits', '_mask-ddcal.fits') # this is used to find calibrators wideDD_image.makeMask(threshpix=5, atrous_do=True, maskname=mask_ddcal, write_srl=True, write_ds9=True)
logger = lib_log.logger s = lib_util.Scheduler(log_dir=logger_obj.log_dir, dry=False) #, maxThreads = 4) # parse parset parset = lib_util.getParset() parset_dir = parset.get('LOFAR_ddfacet', 'parset_dir') maxniter = parset.getint('LOFAR_ddfacet', 'maxniter') calFlux = parset.getfloat('LOFAR_ddfacet', 'calFlux') userReg = parset.get('model', 'userReg') uvrange = [0.03, 1e6] image_robust = -0.15 ncluster = 5 #################################################### MSs_self = lib_ms.AllMSs(glob.glob('mss/TC*[0-9].MS'), s) # make beam phasecentre = MSs_self.getListObj()[0].getPhaseCentre() fwhm = MSs_self.getListObj()[0].getFWHM(freq='mid') nchan = MSs_self.getListObj()[0].getNchan() ############################ logger.info('Cleaning...') lib_util.check_rm('ddfcal') os.makedirs('ddfcal/masks') os.makedirs('ddfcal/plots') os.makedirs('ddfcal/images') os.makedirs('ddfcal/solutions') os.makedirs('ddfcal/cache') lib_util.check_rm('img')
import numpy as np ############################################### from LiLF import lib_ms, lib_util, lib_log logger_obj = lib_log.Logger('pipeline-demix.logger') logger = lib_log.logger s = lib_util.Scheduler(log_dir=logger_obj.log_dir, dry=False) # parse parset parset = lib_util.getParset() parset_dir = parset.get('LOFAR_demix', 'parset_dir') data_dir = parset.get('LOFAR_demix', 'data_dir') skydb = parset.get('LOFAR_demix', 'demix_model') ############################################## # Demix MSs = lib_ms.AllMSs(glob.glob(data_dir + '/*MS'), s) for MS in MSs.getListStr(): lib_util.check_rm(os.path.basename(MS) + '_' + os.path.basename(skydb)) print(('cp -r ' + skydb + ' ' + os.path.basename(MS) + '_' + os.path.basename(skydb))) os.system('cp -r ' + skydb + ' ' + os.path.basename(MS) + '_' + os.path.basename(skydb)) logger.info('Demixing...') MSs.run('DPPP '+parset_dir+'/DPPP_demix.parset msin=$pathMS msout=$nameMS demixer.skymodel=$nameMS.MS_'+os.path.basename(skydb)+' demixer.instrumentmodel=$nameMS/instrument_demix', \ log='$nameMS_demix.log', commandType='DPPP') logger.info("Done.")
######################################################## from LiLF import lib_ms, lib_img, lib_util, lib_log, lib_dd logger_obj = lib_log.Logger('pipeline-self.logger') logger = lib_log.logger s = lib_util.Scheduler(log_dir=logger_obj.log_dir, dry=False) parset = lib_util.getParset() parset_dir = parset.get('uGMRT_self', 'parset_dir') sourcedb = parset.get('model', 'sourcedb') # relative to tgts dir "./tgts/xxx/model.txt userReg = parset.get('model', 'userReg') # relative to tgts dir "./tgts/xxx/region.ref" MSs = lib_ms.AllMSs(glob.glob('mss/*.MS'), s) ############################################################################ # Clear logger.info('Cleaning...') lib_util.check_rm('img') os.makedirs('img') lib_util.check_rm('self') os.makedirs('self/images') os.makedirs('self/solutions') os.makedirs('self/plots') is_wideband = len( MSs.getListObj()[0].getFreqs()) > 1000 # if > 1000 chans, it is wideband # make beam
cal_dir = parset.get('LOFAR_timesplit', 'cal_dir') ngroups = parset.getint('LOFAR_timesplit', 'ngroups') initc = parset.getint( 'LOFAR_timesplit', 'initc') # initial tc num (useful for multiple observation of same target) bl2flag = parset.get('flag', 'stations') ################################################# # Clean with w.if_todo('clean'): logger.info('Cleaning...') lib_util.check_rm('mss*') ### DONE with w.if_todo('copy'): MSs = lib_ms.AllMSs(glob.glob(data_dir + '/*MS'), s) logger.info('Copy data...') for MS in MSs.getListObj(): #if min(MS.getFreqs()) > 30.e6: # overwrite=True to prevent updating the weights twice MS.move(MS.nameMS + '.MS', keepOrig=True, overwrite=True) ### DONE MSs = lib_ms.AllMSs(glob.glob('*MS'), s) ################################################## # Find solutions to apply if cal_dir == '': obsid = MSs.getListObj()[0].getObsID() # try standard location
for i, line in enumerate(df): ms = re.findall(r'L[0-9]*.*_SB[0-9]*_uv', line)[0] if ms + '.MS' in downloaded or ms + '.dppp.MS' in downloaded: continue if ms + '.MS' in glob.glob('*MS') or ms + '.dppp.MS' in glob.glob( '*MS'): continue s.add('wget -nv "' + line[:-1] + '" -O - | tar -x', log=ms + '_download.log', commandType='general') # print 'wget -nv "'+line[:-1]+'" -O - | tar -x' logger.debug('Queue download of: ' + line[:-1]) s.run(check=True, maxThreads=4) MSs = lib_ms.AllMSs(glob.glob('*MS'), s) if len(MSs.getListStr()) == 0: logger.info('Done.') sys.exit(0) ####################################### if fix_table: logger.info('Fix MS table...') MSs.run('fixMS_TabRef.py $pathMS', log='$nameMS_fixms.log', commandType='python') # only ms created in range (2/2013->2/2014) with pt.table(MSs.getListStr()[0] + '/OBSERVATION', readonly=True, ack=False) as obs:
for i, line in enumerate(df): ms = re.findall(r'L[0-9]*.*_SB[0-9]*_uv', line)[0] if ms + '.MS' in downloaded or ms + '.dppp.MS' in downloaded: continue if ms + '.MS' in glob.glob('*MS') or ms + '.dppp.MS' in glob.glob( '*MS'): continue s.add('wget -nv "' + line[:-1] + '" -O - | tar -x', log=ms + '_download.log', commandType='general') # print 'wget -nv "'+line[:-1]+'" -O - | tar -x' logger.debug('Queue download of: ' + line[:-1]) s.run(check=True, maxThreads=4) MSs = lib_ms.AllMSs(glob.glob('*MS'), s, check_flags=False) if len(MSs.getListStr()) == 0: logger.info('Done.') sys.exit(0) ###################################### with pt.table(MSs.getListStr()[0] + '/OBSERVATION', readonly=True, ack=False) as obs: t = Time(obs.getcell('TIME_RANGE', 0)[0] / (24 * 3600.), format='mjd') time = np.int(t.iso.replace('-', '')[0:8]) if fix_table: #logger.info('Fix MS table...') #MSs.run('fixMS_TabRef.py $pathMS', log='$nameMS_fixms.log', commandType='python') # only ms created in range (2/2013->2/2014)
from LiLF import lib_ms, lib_img, lib_util, lib_log logger_obj = lib_log.Logger('pipeline-cal.logger') logger = lib_log.logger s = lib_util.Scheduler(log_dir=logger_obj.log_dir, dry=False) w = lib_util.Walker('pipeline-cal.walker') # parse parset parset = lib_util.getParset() parset_dir = parset.get('LOFAR_cal', 'parset_dir') data_dir = parset.get('LOFAR_cal', 'data_dir') skymodel = parset.get('LOFAR_cal', 'skymodel') imaging = parset.getboolean('LOFAR_cal', 'imaging') bl2flag = parset.get('flag', 'stations') ############################################################# MSs = lib_ms.AllMSs(glob.glob(data_dir + '/*MS'), s, check_flags=False) with w.if_todo('copy'): # copy data logger.info('Copy data...') for MS in MSs.getListObj(): MS.move(MS.nameMS + '.MS', keepOrig=True, overwrite=False) ### DONE MSs = lib_ms.AllMSs(glob.glob('*MS'), s, check_flags=False) calname = MSs.getListObj()[0].getNameField() for MS in MSs.getListObj(): os.system('cp -r %s %s' % (skymodel, MS.pathMS)) if min(MSs.getFreqs()) < 35.e6:
############################################### from LiLF import lib_ms, lib_util, lib_log logger_obj = lib_log.Logger('pipeline-demix.logger') logger = lib_log.logger s = lib_util.Scheduler(log_dir=logger_obj.log_dir, dry=False) # parse parset parset = lib_util.getParset() parset_dir = parset.get('LOFAR_demix', 'parset_dir') data_dir = parset.get('LOFAR_demix', 'data_dir') demix_skymodel = parset.get('LOFAR_demix', 'demix_model') # skymodel to be converted in skydb include_target = parset.getboolean('LOFAR_demix', 'include_target') ############################################## MSs = lib_ms.AllMSs(glob.glob('mss-predemix/TC*[0-9].MS'), s) ateams = ['VirA', 'TauA'] ateams_todemix = [] for ateam in ateams: sep = MSs.getListObj()[0].distBrightSource(ateam) if sep < 4 or sep > 15: logger.debug('No demix of %s (sep: %.0f deg)' % (ateam, sep)) else: ateams_todemix.append(ateam) logger.warning('Demix of %s (sep: %.0f deg)' % (ateam, sep)) if len(ateams_todemix) < 0: logger.info('Notingh to demix.') sys.exit() if os.path.exists('mss-predemix'):
s = lib_util.Scheduler(log_dir=logger_obj.log_dir, dry=False) # parse parset parset = lib_util.getParset() parset_dir = parset.get('uGMRT_cal', 'parset_dir') skymodel = parset.get('uGMRT_cal', 'skymodel') bl2flag = parset.get('flag', 'stations') ############################################################# # cicle on each msID assuming telescope properties are similar in all calibrator scans of the same msID msIDs = np.unique([int(name[5:7]) for name in glob.glob('cals/*')]) for msID in msIDs: logger.info('Working on MSid %02i' % msID) MSs_cals = lib_ms.AllMSs(glob.glob('cals/%02i*/*MS' % msID), s) calnames = [MS.getNameField() for MS in MSs_cals.getListObj()] for MS in MSs_cals.getListObj(): os.system('cp -r %s %s' % (skymodel, MS.pathMS)) logger.info('Add columns...') MSs_cals.run('addcol2ms.py -m $pathMS -c MODEL_DATA -i DATA', log="$nameMS_addcol.log", commandType="python") # predict to save time ms:MODEL_DATA for i, MS in enumerate(MSs_cals.getListObj()): logger.info('%s: add model to MODEL_DATA (%s)...' % (MS.pathMS, calnames[i])) s.add("DPPP " + parset_dir + "/DPPP-predict.parset msin="+MS.pathMS+" pre.sourcedb="+MS.pathMS+"/" + os.path.basename(skymodel) + " pre.sources=" + calnames[i], \ log=MS.nameMS+"_pre.log", commandType="DPPP")
if timestamp in cal_timestamps: logger.info('Calibrator found: %s (t=%s)' % (cal_dir, timestamp)) return cal_dir + '/cals' logger.error('Missing calibrator.') sys.exit() ########################################################## logger.info('Cleaning...') lib_util.check_rm('cal*h5') lib_util.check_rm('plots*') lib_util.check_rm('img') os.makedirs('img') MSs = lib_ms.AllMSs(glob.glob(data_dir + '/*MS'), s, check_flags=False) for timestamp in set( [os.path.basename(ms).split('_')[1][1:] for ms in MSs.getListStr()]): mss_toconcat = glob.glob(data_dir + '/' + target + '_t' + timestamp + '_SB*.MS') MS_concat = target + '_t' + timestamp + '_concat.MS' MS_concat_bkp = target + '_t' + timestamp + '_concat.MS-bkp' if os.path.exists(MS_concat_bkp): logger.info('Restoring bkp data...') os.system('rm -r %s' % MS_concat) os.system('cp -r %s %s' % (MS_concat_bkp, MS_concat)) else: logger.info('Making %s...' % MS_concat) s.add('DPPP '+parset_dir+'/DPPP-avg.parset msin=\"'+str(mss_toconcat)+'\" msout='+MS_concat,\ log=MS_concat+'_avg.log', commandType='DPPP')
# parse parset parset = lib_util.getParset('lilf.config') parset_dir = parset.get('uGMRT_init', 'parset_dir') data_dir = parset.get('uGMRT_init', 'data_dir') bl2flag = parset.get('flag', 'antennas') ############################# # 1. Download data from NCRA server ############################# # 2. Convert into MS ############################# # 3. Set-up directory structure ans split MSs = lib_ms.AllMSs(glob.glob(data_dir + '/*MS'), s) numberOfMSs = len(MSs.getListObj()) if not os.path.exists('cals'): os.mkdir('cals') if not os.path.exists('tgts'): os.mkdir('tgts') for msID, MS in enumerate(MSs.getListObj()): pathMS = MS.pathMS logger.info("- Split-up of original MS '%s' (msID: %02i) -" % (pathMS, msID)) scanIDs = tables.taql("select distinct SCAN_NUMBER from $pathMS").getcol( "SCAN_NUMBER") numberOfScans = len(scanIDs)
from LiLF import lib_ms, lib_img, lib_util, lib_log logger_obj = lib_log.Logger('pipeline-cal.logger') logger = lib_log.logger s = lib_util.Scheduler(log_dir=logger_obj.log_dir, dry=False) w = lib_util.Walker('pipeline-cal.walker') # parse parset parset = lib_util.getParset() parset_dir = parset.get('LOFAR_cal', 'parset_dir') data_dir = parset.get('LOFAR_cal', 'data_dir') skymodel = parset.get('LOFAR_cal', 'skymodel') imaging = parset.getboolean('LOFAR_cal', 'imaging') bl2flag = parset.get('flag', 'stations') ############################################################# MSs = lib_ms.AllMSs(glob.glob(data_dir + '/*MS'), s, check_flags=False) if w.todo('copy'): # copy data logger.info('Copy data...') for MS in MSs.getListObj(): MS.move(MS.nameMS + '.MS', keepOrig=True, overwrite=False) w.done('copy') ### DONE MSs = lib_ms.AllMSs(glob.glob('*MS'), s, check_flags=False) calname = MSs.getListObj()[0].getNameField() for MS in MSs.getListObj(): os.system('cp -r %s %s' % (skymodel, MS.pathMS))
import pyrap.tables as pt from astropy.time import Time import numpy as np ######################################################## from LiLF import lib_ms, lib_img, lib_util, lib_log logger_obj = lib_log.Logger('pipeline-test.logger') logger = lib_log.logger s = lib_util.Scheduler(log_dir = logger_obj.log_dir, dry = False) # parse parset #parset = lib_util.getParset() parset_dir = '/home/baq1889/scripts/LiLF/parsets/LOFAR_dd' ############################################################# MSs = lib_ms.AllMSs( glob.glob('*MS'), s, check_flags=False ) skymodel_voro_skydb = 'skymodel00_voro.skydb' c=0 # Calibration - ms:SMOOTHED_DATA logger.info('Calibrating...') MSs.run('DPPP '+parset_dir+'/DPPP-solDD.parset msin=$pathMS ddecal.h5parm=$pathMS/cal-core.h5 \ ddecal.sourcedb='+skymodel_cl_skydb+' ddecal.solint=15 ddecal.nchan=30', \ log='$nameMS_solDD-core.log', commandType='DPPP') lib_util.run_losoto(s, 'core', [MS+'/cal-core.h5' for MS in MSs.getListStr()], \ [parset_dir+'/losoto-core.parset']) logger.info('re-calibration...') # predict and corrupt each facet
return cal_dir + '/cals' logger.error('Missing calibrator.') sys.exit() ########################################################## with w.if_todo('setup'): logger.info('Cleaning...') lib_util.check_rm('cal*h5') lib_util.check_rm('plots*') lib_util.check_rm('peel*') lib_util.check_rm('img') os.makedirs('img') MSs = lib_ms.AllMSs(glob.glob(data_dir + '/*MS'), s, check_flags=False) for timestamp in set( [os.path.basename(ms).split('_')[1][1:] for ms in MSs.getListStr()]): mss_toconcat = sorted( glob.glob(data_dir + '/' + target + '_t' + timestamp + '_SB*.MS')) MS_concat = target + '_t' + timestamp + '_concat.MS' MS_concat_bkp = target + '_t' + timestamp + '_concat.MS-bkp' if os.path.exists(MS_concat_bkp): logger.info('Restoring bkp data: %s...' % MS_concat_bkp) lib_util.check_rm(MS_concat) os.system('cp -r %s %s' % (MS_concat_bkp, MS_concat)) else: logger.info('Making %s...' % MS_concat) s.add('DPPP '+parset_dir+'/DPPP-avg.parset msin=\"'+str(mss_toconcat)+'\" msout='+MS_concat,\
if w.todo('cleaning'): logger.info('Cleaning...') lib_util.check_rm('plot*') lib_util.check_rm('cal*h5') lib_util.check_rm('img') lib_util.check_rm('facet') os.makedirs('img') os.makedirs('facet') lib_util.check_rm('mss-facet') if not os.path.exists('mss-facet'): os.system('cp -r mss-dd mss-facet') w.done('cleaning') ### DONE MSs = lib_ms.AllMSs(glob.glob('mss-facet/*MS'), s) def clean(p, MSs, size, res='normal', apply_beam=False): """ p = patch name mss = list of mss to clean size = in deg of the image """ # set pixscale and imsize pixscale = MSs.getListObj()[0].getResolution() if res == 'normal': pixscale = float('%.1f' % (pixscale / 2.5)) elif res == 'high': pixscale = float('%.1f' % (pixscale / 3.5)) elif res == 'low':
from LiLF import lib_ms, lib_img, lib_util, lib_log, lib_dd_parallel logger_obj = lib_log.Logger('pipeline-dd.logger') logger = lib_log.logger s = lib_util.Scheduler(log_dir=logger_obj.log_dir, dry=False) w = lib_util.Walker('pipeline-dd.walker') # parse parset parset = lib_util.getParset() parset_dir = parset.get('LOFAR_dd-parallel', 'parset_dir') maxniter = parset.getint('LOFAR_dd-parallel', 'maxniter') calFlux = parset.getfloat('LOFAR_dd-parallel', 'calFlux') userReg = parset.get('model', 'userReg') aterm_imaging = False MSs_self = lib_ms.AllMSs(glob.glob('mss/TC*[0-9].MS'), s) # make beam phasecentre = MSs_self.getListObj()[0].getPhaseCentre() fwhm = MSs_self.getListObj()[0].getFWHM(freq='mid') def clean(p, MSs, size, res='normal', apply_beam=False): """ p = patch name mss = list of mss to clean size = in deg of the image """ # set pixscale and imsize pixscale = MSs.resolution if res == 'normal':
'.log | grep "background noise"') ############################################################# with w.if_todo('cleaning'): logger.info('Cleaning...') lib_util.check_rm('ddcal') os.makedirs('ddcal/init') lib_util.check_rm('img') os.makedirs('img') lib_util.check_rm('mss-avg') ### DONE # goes down to 8 seconds/4 chans if not os.path.exists('mss-avg'): MSs = lib_ms.AllMSs(glob.glob('mss/TC*[0-9].MS'), s) timeint = MSs.getListObj()[0].getTimeInt() avgtimeint = int(round(8 / timeint)) # to 8 seconds nchan_init = MSs.getListObj()[0].getNchan() # chan: avg (x8) sol (x6) - we need a multiple of 8x6=48, the largest that is <nchan # survey after avg (x8): 60, final number of sol 10 # pointed after avg (x8): 120, final number of sol 20 nchan = nchan_init - nchan_init % 48 os.makedirs('mss-avg') logger.info('Averaging in time (%is -> %is), channels: %ich -> %ich)' % (timeint, timeint * avgtimeint, nchan_init, nchan)) MSs.run( 'DPPP ' + parset_dir + '/DPPP-avg.parset msin=$pathMS msout=mss-avg/$nameMS.MS msin.datacolumn=CORRECTED_DATA msin.nchan=' + str(nchan) + ' \ avg.timestep=' + str(avgtimeint) + ' avg.freqstep=1',
# Clear if w.todo('cleaning'): logger.info('Cleaning...') lib_util.check_rm('img') os.makedirs('img') # here images, models, solutions for each group will be saved lib_util.check_rm('self') if not os.path.exists('self/images'): os.makedirs('self/images') if not os.path.exists('self/solutions'): os.makedirs('self/solutions') if not os.path.exists('self/plots'): os.makedirs('self/plots') w.done('cleaning') ### DONE MSs = lib_ms.AllMSs(glob.glob('mss/TC*[0-9].MS'), s) try: MSs.print_HAcov() except: logger.error('Problem with HAcov, continue anyway.') # make beam to the first mid null phasecentre = MSs.getListObj()[0].getPhaseCentre() MSs.getListObj()[0].makeBeamReg('self/beam.reg', freq='mid', to_null=True) beamReg = 'self/beam.reg' # set image size imgsizepix = int(2.1 * MSs.getListObj()[0].getFWHM(freq='mid') * 3600 / 10.) if imgsizepix % 2 != 0: imgsizepix += 1 # prevent odd img sizes #################################################################
logger = lib_log.logger s = lib_util.Scheduler(log_dir=logger_obj.log_dir, dry=False) # parse parset parset = lib_util.getParset() parset_dir = parset.get('LOFAR_ateam', 'parset_dir') bl2flag = parset.get('flag', 'stations') data_dir = '../tgts-bkp/' ########################################################## logger.info('Cleaning...') #lib_util.check_rm('cal*h5') lib_util.check_rm('plots*') lib_util.check_rm('img') os.makedirs('img') MSs = lib_ms.AllMSs(sorted(glob.glob(data_dir + '/*MS')), s) # copy data (avg to 1ch/sb and 10 sec) nchan = int(MSs.getListObj()[0].getNchan()) # average 4 chans timeint = MSs.getListObj()[0].getTimeInt() avg_time = int(np.rint(30. / timeint)) # average to 30 s logger.info('Copy data...') for obs in set([os.path.basename(ms).split('_')[0] for ms in MSs.getListStr()]): mss_toconcat = sorted(glob.glob(data_dir + '/' + obs + '*MS')) # add missing SB with a fake name not to leave frequency holes num_init = int(re.findall(r'\d+', mss_toconcat[0])[-1]) num_fin = int(re.findall(r'\d+', mss_toconcat[-1])[-1])
help="If you want to provide a certain skymodel.") args = parser.parse_args() ############################################################################# # Clear with w.if_todo('cleaning'): lib_util.check_rm('bench') if not os.path.exists('bench/img'): os.makedirs('bench/img') logger.info('Cleaning...') lib_util.check_rm('bench/img') os.makedirs('bench/img') lib_util.check_rm('bench/plots') os.makedirs('bench/plots') ### DONE print(args.msfiles) MSs = lib_ms.AllMSs(glob.glob(args.msfiles), s) sourcedb = args.sourcedb ncpu = mp.cpu_count() nms = len(MSs.getListObj()) logger.info(f'Found {ncpu} cpus and {nms} MS files') # set image size imgsizepix = int(2.1 * MSs.getListObj()[0].getFWHM(freq='mid') * 3600 / 10.) if imgsizepix % 2 != 0: imgsizepix += 1 # prevent odd img sizes # set clean componet fit order (use 5 for large BW) bandwidth = MSs.getBandwidth() # make beam to the first mid null phasecentre = MSs.getListObj()[0].getPhaseCentre() # MSs.getListObj()[0].makeBeamReg('bench/beam.reg', freq='mid', to_null=True)
from LiLF import lib_ms, lib_img, lib_util, lib_log lib_log.Logger('pipeline-ateam.logger') logger = lib_log.logger s = lib_util.Scheduler(dry=False) # parse parset parset = lib_util.getParset() parset_dir = parset.get('ateam', 'parset_dir') bl2flag = parset.get('flag', 'stations') data_dir = '../tgts-bkp/' ########################################################## lib_util.check_rm('img') os.makedirs('img') mss = sorted(glob.glob(data_dir + '/*MS')) MSs = lib_ms.AllMSs(mss, s) # HBA/LBA if min(MSs.getFreqs()) < 80.e6: hba = False else: hba = True # wsclean temp-dir temp_dir = '.' if s.get_cluster() == 'Hamburg_fat': temp_dir = '/localwork.ssd' # copy data (avg to 1ch/sb and 10 sec) nchan = MSs.getListObj()[0].getNchan() timeint = MSs.getListObj()[0].getTimeInt() avg_time = int(np.rint(10. / timeint)) logger.info('Copy data...')