#!/usr/bin/env python3
# -*- coding: utf-8 -*-
# A simple benchmark script for DP3 and wsclean
# they need to be in "./mss/"
import sys, os, glob, re, argparse
import numpy as np
import time
import lsmtool
import multiprocessing as mp
########################################################
from LiLF import lib_ms, lib_img, lib_util, lib_log
logger_obj = lib_log.Logger('script-benchmark.logger')
logger = lib_log.logger
s = lib_util.Scheduler(log_dir=logger_obj.log_dir, dry=False)
w = lib_util.Walker('script-benchmark.walker')
parset = lib_util.getParset()
parset_dir = parset.get('LOFAR_self', 'parset_dir')
parser = argparse.ArgumentParser(description="Benchmark LOFAR software.")
parser.add_argument("msfiles",
type=str,
help="Input ms files, glob-like string")
parser.add_argument("--sourcedb",
default='',
help="If you want to provide a certain skymodel.")
args = parser.parse_args()
#############################################################################
# Clear
import casacore.tables as pt
import lsmtool
# Survey
#if 'LBAsurvey' in os.getcwd():
# obs = os.getcwd().split('/')[-1]
# if not os.path.exists('mss'):
# os.makedirs('mss')
# for i, tc in enumerate(glob.glob('/home/fdg/data/LBAsurvey/c*-o*/%s/mss/*' % obs)):
# tc_ren = 'TC%02i.MS' % i
# print('cp -r %s mss/%s' % (tc,tc_ren))
# os.system('cp -r %s mss/%s' % (tc,tc_ren))
########################################################
from LiLF import lib_ms, lib_img, lib_util, lib_log
logger_obj = lib_log.Logger('pipeline-self.logger')
logger = lib_log.logger
s = lib_util.Scheduler(log_dir=logger_obj.log_dir, dry=False)
w = lib_util.Walker('pipeline-self.walker')
parset = lib_util.getParset()
parset_dir = parset.get('LOFAR_self', 'parset_dir')
sourcedb = parset.get('model', 'sourcedb')
apparent = parset.getboolean('model', 'apparent')
userReg = parset.get('model', 'userReg')
#############################################################################
# Clear
if w.todo('cleaning'):
logger.info('Cleaning...')
lib_util.check_rm('img')
#!/usr/bin/env python
# -*- coding: utf-8 -*-
# Data preparation for selfcal, apply cal solutions
# and split SB in time and concatenate in frequency.
import sys, os, glob, re
import numpy as np
from astropy.time import Time
import casacore.tables as pt
########################################################
from LiLF import lib_ms, lib_util, lib_log
logger_obj = lib_log.Logger('pipeline-timesplit.logger')
logger = lib_log.logger
s = lib_util.Scheduler(log_dir=logger_obj.log_dir, dry=False)
w = lib_util.Walker('pipeline-timesplit.walker')
# parse parset
parset = lib_util.getParset()
parset_dir = parset.get('LOFAR_timesplit', 'parset_dir')
data_dir = parset.get('LOFAR_timesplit', 'data_dir')
cal_dir = parset.get('LOFAR_timesplit', 'cal_dir')
ngroups = parset.getint('LOFAR_timesplit', 'ngroups')
initc = parset.getint(
'LOFAR_timesplit',
'initc') # initial tc num (useful for multiple observation of same target)
bl2flag = parset.get('flag', 'stations')
#################################################
# Clean
#!/usr/bin/env python
# -*- coding: utf-8 -*-
import sys, os, re, glob, time
import numpy as np
import casacore.tables as pt
from astropy.time import Time
##########################################
from LiLF import lib_ms, lib_util, lib_log
logger_obj = lib_log.Logger('pipeline-preprocess.logger')
logger = lib_log.logger
s = lib_util.Scheduler(log_dir=logger_obj.log_dir, dry=False)
# parse parset
parset = lib_util.getParset()
parset_dir = parset.get('LOFAR_preprocess', 'parset_dir')
fix_table = parset.getboolean('LOFAR_preprocess', 'fix_table')
renameavg = parset.getboolean('LOFAR_preprocess', 'renameavg')
keep_IS = parset.getboolean('LOFAR_preprocess', 'keep_IS')
###########################################
if os.path.exists('html.txt'):
download_file = 'html.txt'
else:
download_file = None # just renaming
def nu2num(nu):
"""
Get the SB number from the freq
#!/usr/bin/env python
# -*- coding: utf-8 -*-
# Pipeline for direction dependent calibration
import sys, os, glob, re, pickle
import numpy as np
import pyrap.tables as pt
import lsmtool
#######################################################
from LiLF import lib_ms, lib_img, lib_util, lib_log, lib_dd_parallel
logger_obj = lib_log.Logger('pipeline-dd.logger')
logger = lib_log.logger
s = lib_util.Scheduler(log_dir=logger_obj.log_dir, dry=False)
w = lib_util.Walker('pipeline-dd.walker')
# parse parset
parset = lib_util.getParset()
parset_dir = parset.get('LOFAR_dd-parallel', 'parset_dir')
maxniter = parset.getint('LOFAR_dd-parallel', 'maxniter')
calFlux = parset.getfloat('LOFAR_dd-parallel', 'calFlux')
userReg = parset.get('model', 'userReg')
aterm_imaging = False
MSs_self = lib_ms.AllMSs(glob.glob('mss/TC*[0-9].MS'), s)
# make beam
phasecentre = MSs_self.getListObj()[0].getPhaseCentre()
fwhm = MSs_self.getListObj()[0].getFWHM(freq='mid')
#!/usr/bin/env python
# -*- coding: utf-8 -*-
# Pipeline to run on the calibrator observation.
# It isolates various systematic effects and
# prepare them for the transfer to the target field.
import sys, os, glob, re
import numpy as np
########################################################
from LiLF import lib_ms, lib_img, lib_util, lib_log
logger_obj = lib_log.Logger('pipeline-cal.logger')
logger = lib_log.logger
s = lib_util.Scheduler(log_dir=logger_obj.log_dir, dry=False)
w = lib_util.Walker('pipeline-cal.walker')
# parse parset
parset = lib_util.getParset()
parset_dir = parset.get('LOFAR_cal', 'parset_dir')
data_dir = parset.get('LOFAR_cal', 'data_dir')
skymodel = parset.get('LOFAR_cal', 'skymodel')
imaging = parset.getboolean('LOFAR_cal', 'imaging')
bl2flag = parset.get('flag', 'stations')
#############################################################
MSs = lib_ms.AllMSs(glob.glob(data_dir + '/*MS'), s, check_flags=False)
if w.todo('copy'):
# copy data
logger.info('Copy data...')
#!/usr/bin/env python
# -*- coding: utf-8 -*-
import sys, os, re, glob, time
import numpy as np
import casacore.tables as pt
from astropy.time import Time
##########################################
from LiLF import lib_ms, lib_util, lib_log
logger_obj = lib_log.Logger('pipeline-download.logger')
logger = lib_log.logger
s = lib_util.Scheduler(log_dir=logger_obj.log_dir, dry=False)
# parse parset
parset = lib_util.getParset()
parset_dir = parset.get('LOFAR_download', 'parset_dir')
fix_table = parset.getboolean('LOFAR_download', 'fix_table')
renameavg = parset.getboolean('LOFAR_download', 'renameavg')
keep_IS = parset.getboolean('LOFAR_download', 'keep_IS')
###########################################
if os.path.exists('html.txt'):
download_file = 'html.txt'
else:
download_file = None # just renaming
def nu2num(nu):
"""
#!/usr/bin/python
# demix of a set of SBs from a given dir, output is in the local dir
import sys, os, glob
import numpy as np
###############################################
from LiLF import lib_ms, lib_util, lib_log
logger_obj = lib_log.Logger('pipeline-demix.logger')
logger = lib_log.logger
s = lib_util.Scheduler(log_dir=logger_obj.log_dir, dry=False)
# parse parset
parset = lib_util.getParset()
parset_dir = parset.get('LOFAR_demix', 'parset_dir')
data_dir = parset.get('LOFAR_demix', 'data_dir')
skydb = parset.get('LOFAR_demix', 'demix_model')
##############################################
# Demix
MSs = lib_ms.AllMSs(glob.glob(data_dir + '/*MS'), s)
for MS in MSs.getListStr():
lib_util.check_rm(os.path.basename(MS) + '_' + os.path.basename(skydb))
print(('cp -r ' + skydb + ' ' + os.path.basename(MS) + '_' +
os.path.basename(skydb)))
os.system('cp -r ' + skydb + ' ' + os.path.basename(MS) + '_' +
os.path.basename(skydb))
logger.info('Demixing...')
MSs.run('DPPP '+parset_dir+'/DPPP_demix.parset msin=$pathMS msout=$nameMS demixer.skymodel=$nameMS.MS_'+os.path.basename(skydb)+' demixer.instrumentmodel=$nameMS/instrument_demix', \
# Pipeline for extraction of target region after LOFAR_dd-serial.
# Provide a extractRegion in lilf.config. This pipeline will subtract
# sources outside of the region and perform subsequent self-calibration.
# A userReg may be specified as clean mask.
# phSolMode can be used to solve either using phases or phaseandtec.
import sys, os, glob, re
import numpy as np
import pyregion
from losoto.h5parm import h5parm
#######################################################
from LiLF import lib_ms, lib_img, lib_util, lib_log
logger_obj = lib_log.Logger('pipeline-extract.logger')
logger = lib_log.logger
s = lib_util.Scheduler(log_dir=logger_obj.log_dir, dry=False)
w = lib_util.Walker('pipeline-extract.walker')
# parse parset
parset = lib_util.getParset()
parset_dir = parset.get('LOFAR_extract', 'parset_dir')
maxniter = parset.getint('LOFAR_extract', 'maxniter')
target_reg_file = parset.get('LOFAR_extract',
'extractRegion') # default 'target.reg'
phSolMode = parset.get('LOFAR_extract', 'phSolMode') # default: tecandphase
if phSolMode not in ['tecandphase', 'phase']:
logger.error('phSolMode {} not supported. Choose tecandphase, phase.')
sys.exit()
userReg = parset.get('model', 'userReg')
# Pipeline for direction dependent calibration
# TODO: remove regions and move to masks
import sys, os, glob, re, pickle, collections
import numpy as np
from astropy.table import Table as astrotab
from astropy.coordinates import SkyCoord
from astropy import units as u
import pyrap.tables as pt
import lsmtool
#######################################################
from LiLF import lib_ms, lib_img, lib_util, lib_log, lib_dd, lib_h5
logger_obj = lib_log.Logger('pipeline-dd-serial.logger')
logger = lib_log.logger
s = lib_util.Scheduler(log_dir=logger_obj.log_dir, dry=False)
w = lib_util.Walker('pipeline-dd-serial.walker')
# parse parset
parset = lib_util.getParset()
parset_dir = parset.get('LOFAR_dd-serial', 'parset_dir')
userReg = parset.get('model', 'userReg')
maxIter = parset.getint('LOFAR_dd-serial', 'maxIter')
min_cal_flux60 = parset.getfloat('LOFAR_dd-serial', 'minCalFlux60')
removeExtendedCutoff = parset.getfloat('LOFAR_dd-serial',
'removeExtendedCutoff')
target_dir = parset.get('LOFAR_dd-serial', 'target_dir')
((np.log10(nu / 1.e9))**2) - 0.0705 * (
(np.log10(nu / 1.e9))**3)) # PB17
elif 'Cyg' in os.getcwd():
patch = 'CygA'
nouseblrange = ''
#f = lambda nu: 10690. * 10**(-0.67 * (np.log10(nu/150.e6))**1) * 10**(-0.204 * (np.log10(nu/150.e6))**2) * 10**(-0.021 * (np.log10(nu/150.e6))**3)
f = lambda nu: 10**(3.3498 - 1.0022 * ((np.log10(nu / 1.e9))**1) - 0.2246 *
((np.log10(nu / 1.e9))**2) + 0.0227 * (
(np.log10(nu / 1.e9))**3) + 0.0425 * (
(np.log10(nu / 1.e9))**4)) # PB17
skymodel = '/home/fdg/scripts/model/A-team_4_CC.skydb'
########################################################
from LiLF import lib_ms, lib_img, lib_util, lib_log
logger_obj = lib_log.Logger('pipeline-ateam.logger')
logger = lib_log.logger
s = lib_util.Scheduler(log_dir=logger_obj.log_dir, dry=False)
# parse parset
parset = lib_util.getParset()
parset_dir = parset.get('LOFAR_ateam', 'parset_dir')
bl2flag = parset.get('flag', 'stations')
data_dir = '../tgts-bkp/'
##########################################################
logger.info('Cleaning...')
#lib_util.check_rm('cal*h5')
lib_util.check_rm('plots*')
lib_util.check_rm('img')
os.makedirs('img')
# -*- coding: utf-8 -*-
# Pipeline for single facet self calibration
import sys, os, glob, re
import numpy as np
import pyrap.tables as pt
import lsmtool
#TODO, to extract from regionfile:
lastcycle = 0 # get from somewhere
target_reg = 'target.reg'
#######################################################
from LiLF import lib_ms, lib_img, lib_util, lib_log, lib_dd
logger_obj = lib_log.Logger('pipeline-facet_self.logger')
logger = lib_log.logger
s = lib_util.Scheduler(log_dir=logger_obj.log_dir,
dry=False) #, maxThreads = 4)
w = lib_util.Walker('pipeline-facet-self.walker')
# parse parset
parset = lib_util.getParset()
parset_dir = parset.get('LOFAR_facet_self', 'parset_dir')
maxniter = parset.getint('LOFAR_facet_self', 'maxniter')
userReg = parset.get('model', 'userReg')
mosaic_image = lib_img.Image('ddcal/images/c%02i/mos-MFS-image.fits' %
lastcycle)
############################
if w.todo('cleaning'):
#!/usr/bin/env python
# -*- coding: utf-8 -*-
import sys, os, glob, re
import pyrap.tables as pt
from astropy.time import Time
import numpy as np
########################################################
from LiLF import lib_ms, lib_img, lib_util, lib_log
logger_obj = lib_log.Logger('pipeline-test.logger')
logger = lib_log.logger
s = lib_util.Scheduler(log_dir = logger_obj.log_dir, dry = False)
# parse parset
#parset = lib_util.getParset()
parset_dir = '/home/baq1889/scripts/LiLF/parsets/LOFAR_dd'
#############################################################
MSs = lib_ms.AllMSs( glob.glob('*MS'), s, check_flags=False )
skymodel_voro_skydb = 'skymodel00_voro.skydb'
c=0
# Calibration - ms:SMOOTHED_DATA
logger.info('Calibrating...')
MSs.run('DPPP '+parset_dir+'/DPPP-solDD.parset msin=$pathMS ddecal.h5parm=$pathMS/cal-core.h5 \
ddecal.sourcedb='+skymodel_cl_skydb+' ddecal.solint=15 ddecal.nchan=30', \
log='$nameMS_solDD-core.log', commandType='DPPP')
lib_util.run_losoto(s, 'core', [MS+'/cal-core.h5' for MS in MSs.getListStr()], \
#!/usr/bin/env python
import os, sys, glob, getpass, socket, re
from LiLF.surveys_db import SurveysDB
from LiLF import lib_ms, lib_img, lib_util, lib_log
logger_obj = lib_log.Logger('PiLL.logger')
logger = lib_log.logger
s = lib_util.Scheduler(log_dir=logger_obj.log_dir, dry=False)
w = lib_util.Walker('PiLL.walker')
LiLF_dir = os.path.dirname(os.path.dirname(lib_util.__file__))
parset = lib_util.getParset(parsetFile='lilf.config')
survey_projects = 'LT14_002,LC12_017,LC9_016,LC8_031' # list of projects related with the LBA survey
# get parameters
# use lilf.config (this is also used by all other scripits)
working_dir = os.path.abspath(parset.get('PiLL', 'working_dir'))
redo_cal = parset.getboolean('PiLL', 'redo_cal')
project = parset.get('PiLL', 'project')
target = parset.get('PiLL', 'target')
download_file = parset.get('PiLL', 'download_file')
if download_file != '': download_file = os.path.abspath(download_file)
def calibrator_tables_available(obsid):
"""
check if calibrator data exist in the database
"""
with SurveysDB(survey='lba', readonly=True) as sdb:
#!/usr/bin/env python
# -*- coding: utf-8 -*-
import sys, os, glob, re
import numpy as np
from casacore import tables
########################################################
from LiLF import lib_ms, lib_img, lib_util, lib_log
logger_obj = lib_log.Logger('pipeline-init.logger')
logger = lib_log.logger
s = lib_util.Scheduler(log_dir=logger_obj.log_dir, dry=False)
# parse parset
parset = lib_util.getParset('lilf.config')
parset_dir = parset.get('uGMRT_init', 'parset_dir')
data_dir = parset.get('uGMRT_init', 'data_dir')
bl2flag = parset.get('flag', 'antennas')
#############################
# 1. Download data from NCRA server
#############################
# 2. Convert into MS
#############################
# 3. Set-up directory structure ans split
MSs = lib_ms.AllMSs(glob.glob(data_dir + '/*MS'), s)
numberOfMSs = len(MSs.getListObj())
#!/usr/bin/env python
# -*- coding: utf-8 -*-
# Pipeline for direction dependent calibration
import sys, os, glob, re
import numpy as np
import lsmtool
#######################################################
from LiLF import lib_ms, lib_img, lib_util, lib_log, lib_ddfacet
from LiLF.make_mask import make_mask
logger_obj = lib_log.Logger('pipeline-ddfacet.logger')
logger = lib_log.logger
s = lib_util.Scheduler(log_dir=logger_obj.log_dir,
dry=False) #, maxThreads = 4)
# parse parset
parset = lib_util.getParset()
parset_dir = parset.get('LOFAR_ddfacet', 'parset_dir')
maxniter = parset.getint('LOFAR_ddfacet', 'maxniter')
calFlux = parset.getfloat('LOFAR_ddfacet', 'calFlux')
userReg = parset.get('model', 'userReg')
uvrange = [0.03, 1e6]
image_robust = -0.15
ncluster = 5
####################################################
MSs_self = lib_ms.AllMSs(glob.glob('mss/TC*[0-9].MS'), s)
# make beam
