#!/usr/bin/env python3 # -*- coding: utf-8 -*- # A simple benchmark script for DP3 and wsclean # they need to be in "./mss/" import sys, os, glob, re, argparse import numpy as np import time import lsmtool import multiprocessing as mp ######################################################## from LiLF import lib_ms, lib_img, lib_util, lib_log logger_obj = lib_log.Logger('script-benchmark.logger') logger = lib_log.logger s = lib_util.Scheduler(log_dir=logger_obj.log_dir, dry=False) w = lib_util.Walker('script-benchmark.walker') parset = lib_util.getParset() parset_dir = parset.get('LOFAR_self', 'parset_dir') parser = argparse.ArgumentParser(description="Benchmark LOFAR software.") parser.add_argument("msfiles", type=str, help="Input ms files, glob-like string") parser.add_argument("--sourcedb", default='', help="If you want to provide a certain skymodel.") args = parser.parse_args() ############################################################################# # Clear
import casacore.tables as pt import lsmtool # Survey #if 'LBAsurvey' in os.getcwd(): # obs = os.getcwd().split('/')[-1] # if not os.path.exists('mss'): # os.makedirs('mss') # for i, tc in enumerate(glob.glob('/home/fdg/data/LBAsurvey/c*-o*/%s/mss/*' % obs)): # tc_ren = 'TC%02i.MS' % i # print('cp -r %s mss/%s' % (tc,tc_ren)) # os.system('cp -r %s mss/%s' % (tc,tc_ren)) ######################################################## from LiLF import lib_ms, lib_img, lib_util, lib_log logger_obj = lib_log.Logger('pipeline-self.logger') logger = lib_log.logger s = lib_util.Scheduler(log_dir=logger_obj.log_dir, dry=False) w = lib_util.Walker('pipeline-self.walker') parset = lib_util.getParset() parset_dir = parset.get('LOFAR_self', 'parset_dir') sourcedb = parset.get('model', 'sourcedb') apparent = parset.getboolean('model', 'apparent') userReg = parset.get('model', 'userReg') ############################################################################# # Clear if w.todo('cleaning'): logger.info('Cleaning...') lib_util.check_rm('img')
#!/usr/bin/env python # -*- coding: utf-8 -*- # Data preparation for selfcal, apply cal solutions # and split SB in time and concatenate in frequency. import sys, os, glob, re import numpy as np from astropy.time import Time import casacore.tables as pt ######################################################## from LiLF import lib_ms, lib_util, lib_log logger_obj = lib_log.Logger('pipeline-timesplit.logger') logger = lib_log.logger s = lib_util.Scheduler(log_dir=logger_obj.log_dir, dry=False) w = lib_util.Walker('pipeline-timesplit.walker') # parse parset parset = lib_util.getParset() parset_dir = parset.get('LOFAR_timesplit', 'parset_dir') data_dir = parset.get('LOFAR_timesplit', 'data_dir') cal_dir = parset.get('LOFAR_timesplit', 'cal_dir') ngroups = parset.getint('LOFAR_timesplit', 'ngroups') initc = parset.getint( 'LOFAR_timesplit', 'initc') # initial tc num (useful for multiple observation of same target) bl2flag = parset.get('flag', 'stations') ################################################# # Clean
#!/usr/bin/env python # -*- coding: utf-8 -*- import sys, os, re, glob, time import numpy as np import casacore.tables as pt from astropy.time import Time ########################################## from LiLF import lib_ms, lib_util, lib_log logger_obj = lib_log.Logger('pipeline-preprocess.logger') logger = lib_log.logger s = lib_util.Scheduler(log_dir=logger_obj.log_dir, dry=False) # parse parset parset = lib_util.getParset() parset_dir = parset.get('LOFAR_preprocess', 'parset_dir') fix_table = parset.getboolean('LOFAR_preprocess', 'fix_table') renameavg = parset.getboolean('LOFAR_preprocess', 'renameavg') keep_IS = parset.getboolean('LOFAR_preprocess', 'keep_IS') ########################################### if os.path.exists('html.txt'): download_file = 'html.txt' else: download_file = None # just renaming def nu2num(nu): """ Get the SB number from the freq
#!/usr/bin/env python # -*- coding: utf-8 -*- # Pipeline for direction dependent calibration import sys, os, glob, re, pickle import numpy as np import pyrap.tables as pt import lsmtool ####################################################### from LiLF import lib_ms, lib_img, lib_util, lib_log, lib_dd_parallel logger_obj = lib_log.Logger('pipeline-dd.logger') logger = lib_log.logger s = lib_util.Scheduler(log_dir=logger_obj.log_dir, dry=False) w = lib_util.Walker('pipeline-dd.walker') # parse parset parset = lib_util.getParset() parset_dir = parset.get('LOFAR_dd-parallel', 'parset_dir') maxniter = parset.getint('LOFAR_dd-parallel', 'maxniter') calFlux = parset.getfloat('LOFAR_dd-parallel', 'calFlux') userReg = parset.get('model', 'userReg') aterm_imaging = False MSs_self = lib_ms.AllMSs(glob.glob('mss/TC*[0-9].MS'), s) # make beam phasecentre = MSs_self.getListObj()[0].getPhaseCentre() fwhm = MSs_self.getListObj()[0].getFWHM(freq='mid')
#!/usr/bin/env python # -*- coding: utf-8 -*- # Pipeline to run on the calibrator observation. # It isolates various systematic effects and # prepare them for the transfer to the target field. import sys, os, glob, re import numpy as np ######################################################## from LiLF import lib_ms, lib_img, lib_util, lib_log logger_obj = lib_log.Logger('pipeline-cal.logger') logger = lib_log.logger s = lib_util.Scheduler(log_dir=logger_obj.log_dir, dry=False) w = lib_util.Walker('pipeline-cal.walker') # parse parset parset = lib_util.getParset() parset_dir = parset.get('LOFAR_cal', 'parset_dir') data_dir = parset.get('LOFAR_cal', 'data_dir') skymodel = parset.get('LOFAR_cal', 'skymodel') imaging = parset.getboolean('LOFAR_cal', 'imaging') bl2flag = parset.get('flag', 'stations') ############################################################# MSs = lib_ms.AllMSs(glob.glob(data_dir + '/*MS'), s, check_flags=False) if w.todo('copy'): # copy data logger.info('Copy data...')
#!/usr/bin/env python # -*- coding: utf-8 -*- import sys, os, re, glob, time import numpy as np import casacore.tables as pt from astropy.time import Time ########################################## from LiLF import lib_ms, lib_util, lib_log logger_obj = lib_log.Logger('pipeline-download.logger') logger = lib_log.logger s = lib_util.Scheduler(log_dir=logger_obj.log_dir, dry=False) # parse parset parset = lib_util.getParset() parset_dir = parset.get('LOFAR_download', 'parset_dir') fix_table = parset.getboolean('LOFAR_download', 'fix_table') renameavg = parset.getboolean('LOFAR_download', 'renameavg') keep_IS = parset.getboolean('LOFAR_download', 'keep_IS') ########################################### if os.path.exists('html.txt'): download_file = 'html.txt' else: download_file = None # just renaming def nu2num(nu): """
#!/usr/bin/python # demix of a set of SBs from a given dir, output is in the local dir import sys, os, glob import numpy as np ############################################### from LiLF import lib_ms, lib_util, lib_log logger_obj = lib_log.Logger('pipeline-demix.logger') logger = lib_log.logger s = lib_util.Scheduler(log_dir=logger_obj.log_dir, dry=False) # parse parset parset = lib_util.getParset() parset_dir = parset.get('LOFAR_demix', 'parset_dir') data_dir = parset.get('LOFAR_demix', 'data_dir') skydb = parset.get('LOFAR_demix', 'demix_model') ############################################## # Demix MSs = lib_ms.AllMSs(glob.glob(data_dir + '/*MS'), s) for MS in MSs.getListStr(): lib_util.check_rm(os.path.basename(MS) + '_' + os.path.basename(skydb)) print(('cp -r ' + skydb + ' ' + os.path.basename(MS) + '_' + os.path.basename(skydb))) os.system('cp -r ' + skydb + ' ' + os.path.basename(MS) + '_' + os.path.basename(skydb)) logger.info('Demixing...') MSs.run('DPPP '+parset_dir+'/DPPP_demix.parset msin=$pathMS msout=$nameMS demixer.skymodel=$nameMS.MS_'+os.path.basename(skydb)+' demixer.instrumentmodel=$nameMS/instrument_demix', \
# Pipeline for extraction of target region after LOFAR_dd-serial. # Provide a extractRegion in lilf.config. This pipeline will subtract # sources outside of the region and perform subsequent self-calibration. # A userReg may be specified as clean mask. # phSolMode can be used to solve either using phases or phaseandtec. import sys, os, glob, re import numpy as np import pyregion from losoto.h5parm import h5parm ####################################################### from LiLF import lib_ms, lib_img, lib_util, lib_log logger_obj = lib_log.Logger('pipeline-extract.logger') logger = lib_log.logger s = lib_util.Scheduler(log_dir=logger_obj.log_dir, dry=False) w = lib_util.Walker('pipeline-extract.walker') # parse parset parset = lib_util.getParset() parset_dir = parset.get('LOFAR_extract', 'parset_dir') maxniter = parset.getint('LOFAR_extract', 'maxniter') target_reg_file = parset.get('LOFAR_extract', 'extractRegion') # default 'target.reg' phSolMode = parset.get('LOFAR_extract', 'phSolMode') # default: tecandphase if phSolMode not in ['tecandphase', 'phase']: logger.error('phSolMode {} not supported. Choose tecandphase, phase.') sys.exit() userReg = parset.get('model', 'userReg')
# Pipeline for direction dependent calibration # TODO: remove regions and move to masks import sys, os, glob, re, pickle, collections import numpy as np from astropy.table import Table as astrotab from astropy.coordinates import SkyCoord from astropy import units as u import pyrap.tables as pt import lsmtool ####################################################### from LiLF import lib_ms, lib_img, lib_util, lib_log, lib_dd, lib_h5 logger_obj = lib_log.Logger('pipeline-dd-serial.logger') logger = lib_log.logger s = lib_util.Scheduler(log_dir=logger_obj.log_dir, dry=False) w = lib_util.Walker('pipeline-dd-serial.walker') # parse parset parset = lib_util.getParset() parset_dir = parset.get('LOFAR_dd-serial', 'parset_dir') userReg = parset.get('model', 'userReg') maxIter = parset.getint('LOFAR_dd-serial', 'maxIter') min_cal_flux60 = parset.getfloat('LOFAR_dd-serial', 'minCalFlux60') removeExtendedCutoff = parset.getfloat('LOFAR_dd-serial', 'removeExtendedCutoff') target_dir = parset.get('LOFAR_dd-serial', 'target_dir')
((np.log10(nu / 1.e9))**2) - 0.0705 * ( (np.log10(nu / 1.e9))**3)) # PB17 elif 'Cyg' in os.getcwd(): patch = 'CygA' nouseblrange = '' #f = lambda nu: 10690. * 10**(-0.67 * (np.log10(nu/150.e6))**1) * 10**(-0.204 * (np.log10(nu/150.e6))**2) * 10**(-0.021 * (np.log10(nu/150.e6))**3) f = lambda nu: 10**(3.3498 - 1.0022 * ((np.log10(nu / 1.e9))**1) - 0.2246 * ((np.log10(nu / 1.e9))**2) + 0.0227 * ( (np.log10(nu / 1.e9))**3) + 0.0425 * ( (np.log10(nu / 1.e9))**4)) # PB17 skymodel = '/home/fdg/scripts/model/A-team_4_CC.skydb' ######################################################## from LiLF import lib_ms, lib_img, lib_util, lib_log logger_obj = lib_log.Logger('pipeline-ateam.logger') logger = lib_log.logger s = lib_util.Scheduler(log_dir=logger_obj.log_dir, dry=False) # parse parset parset = lib_util.getParset() parset_dir = parset.get('LOFAR_ateam', 'parset_dir') bl2flag = parset.get('flag', 'stations') data_dir = '../tgts-bkp/' ########################################################## logger.info('Cleaning...') #lib_util.check_rm('cal*h5') lib_util.check_rm('plots*') lib_util.check_rm('img') os.makedirs('img')
# -*- coding: utf-8 -*- # Pipeline for single facet self calibration import sys, os, glob, re import numpy as np import pyrap.tables as pt import lsmtool #TODO, to extract from regionfile: lastcycle = 0 # get from somewhere target_reg = 'target.reg' ####################################################### from LiLF import lib_ms, lib_img, lib_util, lib_log, lib_dd logger_obj = lib_log.Logger('pipeline-facet_self.logger') logger = lib_log.logger s = lib_util.Scheduler(log_dir=logger_obj.log_dir, dry=False) #, maxThreads = 4) w = lib_util.Walker('pipeline-facet-self.walker') # parse parset parset = lib_util.getParset() parset_dir = parset.get('LOFAR_facet_self', 'parset_dir') maxniter = parset.getint('LOFAR_facet_self', 'maxniter') userReg = parset.get('model', 'userReg') mosaic_image = lib_img.Image('ddcal/images/c%02i/mos-MFS-image.fits' % lastcycle) ############################ if w.todo('cleaning'):
#!/usr/bin/env python # -*- coding: utf-8 -*- import sys, os, glob, re import pyrap.tables as pt from astropy.time import Time import numpy as np ######################################################## from LiLF import lib_ms, lib_img, lib_util, lib_log logger_obj = lib_log.Logger('pipeline-test.logger') logger = lib_log.logger s = lib_util.Scheduler(log_dir = logger_obj.log_dir, dry = False) # parse parset #parset = lib_util.getParset() parset_dir = '/home/baq1889/scripts/LiLF/parsets/LOFAR_dd' ############################################################# MSs = lib_ms.AllMSs( glob.glob('*MS'), s, check_flags=False ) skymodel_voro_skydb = 'skymodel00_voro.skydb' c=0 # Calibration - ms:SMOOTHED_DATA logger.info('Calibrating...') MSs.run('DPPP '+parset_dir+'/DPPP-solDD.parset msin=$pathMS ddecal.h5parm=$pathMS/cal-core.h5 \ ddecal.sourcedb='+skymodel_cl_skydb+' ddecal.solint=15 ddecal.nchan=30', \ log='$nameMS_solDD-core.log', commandType='DPPP') lib_util.run_losoto(s, 'core', [MS+'/cal-core.h5' for MS in MSs.getListStr()], \
#!/usr/bin/env python import os, sys, glob, getpass, socket, re from LiLF.surveys_db import SurveysDB from LiLF import lib_ms, lib_img, lib_util, lib_log logger_obj = lib_log.Logger('PiLL.logger') logger = lib_log.logger s = lib_util.Scheduler(log_dir=logger_obj.log_dir, dry=False) w = lib_util.Walker('PiLL.walker') LiLF_dir = os.path.dirname(os.path.dirname(lib_util.__file__)) parset = lib_util.getParset(parsetFile='lilf.config') survey_projects = 'LT14_002,LC12_017,LC9_016,LC8_031' # list of projects related with the LBA survey # get parameters # use lilf.config (this is also used by all other scripits) working_dir = os.path.abspath(parset.get('PiLL', 'working_dir')) redo_cal = parset.getboolean('PiLL', 'redo_cal') project = parset.get('PiLL', 'project') target = parset.get('PiLL', 'target') download_file = parset.get('PiLL', 'download_file') if download_file != '': download_file = os.path.abspath(download_file) def calibrator_tables_available(obsid): """ check if calibrator data exist in the database """ with SurveysDB(survey='lba', readonly=True) as sdb:
#!/usr/bin/env python # -*- coding: utf-8 -*- import sys, os, glob, re import numpy as np from casacore import tables ######################################################## from LiLF import lib_ms, lib_img, lib_util, lib_log logger_obj = lib_log.Logger('pipeline-init.logger') logger = lib_log.logger s = lib_util.Scheduler(log_dir=logger_obj.log_dir, dry=False) # parse parset parset = lib_util.getParset('lilf.config') parset_dir = parset.get('uGMRT_init', 'parset_dir') data_dir = parset.get('uGMRT_init', 'data_dir') bl2flag = parset.get('flag', 'antennas') ############################# # 1. Download data from NCRA server ############################# # 2. Convert into MS ############################# # 3. Set-up directory structure ans split MSs = lib_ms.AllMSs(glob.glob(data_dir + '/*MS'), s) numberOfMSs = len(MSs.getListObj())
#!/usr/bin/env python # -*- coding: utf-8 -*- # Pipeline for direction dependent calibration import sys, os, glob, re import numpy as np import lsmtool ####################################################### from LiLF import lib_ms, lib_img, lib_util, lib_log, lib_ddfacet from LiLF.make_mask import make_mask logger_obj = lib_log.Logger('pipeline-ddfacet.logger') logger = lib_log.logger s = lib_util.Scheduler(log_dir=logger_obj.log_dir, dry=False) #, maxThreads = 4) # parse parset parset = lib_util.getParset() parset_dir = parset.get('LOFAR_ddfacet', 'parset_dir') maxniter = parset.getint('LOFAR_ddfacet', 'maxniter') calFlux = parset.getfloat('LOFAR_ddfacet', 'calFlux') userReg = parset.get('model', 'userReg') uvrange = [0.03, 1e6] image_robust = -0.15 ncluster = 5 #################################################### MSs_self = lib_ms.AllMSs(glob.glob('mss/TC*[0-9].MS'), s) # make beam