# -*- coding: utf-8 -*-
# Data preparation for selfcal, apply cal solutions
# and split SB in time and concatenate in frequency.
import sys, os, glob, re
import numpy as np
from astropy.time import Time
import casacore.tables as pt
########################################################
from LiLF import lib_ms, lib_util, lib_log
logger_obj = lib_log.Logger('pipeline-timesplit.logger')
logger = lib_log.logger
s = lib_util.Scheduler(log_dir=logger_obj.log_dir, dry=False)
w = lib_util.Walker('pipeline-timesplit.walker')
# parse parset
parset = lib_util.getParset()
parset_dir = parset.get('LOFAR_timesplit', 'parset_dir')
data_dir = parset.get('LOFAR_timesplit', 'data_dir')
cal_dir = parset.get('LOFAR_timesplit', 'cal_dir')
ngroups = parset.getint('LOFAR_timesplit', 'ngroups')
initc = parset.getint(
'LOFAR_timesplit',
'initc') # initial tc num (useful for multiple observation of same target)
bl2flag = parset.get('flag', 'stations')
#################################################
# Clean
with w.if_todo('clean'):
# A simple benchmark script for DP3 and wsclean
# they need to be in "./mss/"
import sys, os, glob, re, argparse
import numpy as np
import time
import lsmtool
import multiprocessing as mp
########################################################
from LiLF import lib_ms, lib_img, lib_util, lib_log
logger_obj = lib_log.Logger('script-benchmark.logger')
logger = lib_log.logger
s = lib_util.Scheduler(log_dir=logger_obj.log_dir, dry=False)
w = lib_util.Walker('script-benchmark.walker')
parset = lib_util.getParset()
parset_dir = parset.get('LOFAR_self', 'parset_dir')
parser = argparse.ArgumentParser(description="Benchmark LOFAR software.")
parser.add_argument("msfiles",
type=str,
help="Input ms files, glob-like string")
parser.add_argument("--sourcedb",
default='',
help="If you want to provide a certain skymodel.")
args = parser.parse_args()
#############################################################################
# Clear
with w.if_todo('cleaning'):
lib_util.check_rm('bench')
# -*- coding: utf-8 -*-
# Pipeline for direction dependent calibration
import sys, os, glob, re, pickle
import numpy as np
import pyrap.tables as pt
import lsmtool
#######################################################
from LiLF import lib_ms, lib_img, lib_util, lib_log, lib_dd_parallel
logger_obj = lib_log.Logger('pipeline-dd.logger')
logger = lib_log.logger
s = lib_util.Scheduler(log_dir=logger_obj.log_dir, dry=False)
w = lib_util.Walker('pipeline-dd.walker')
# parse parset
parset = lib_util.getParset()
parset_dir = parset.get('LOFAR_dd-parallel', 'parset_dir')
maxniter = parset.getint('LOFAR_dd-parallel', 'maxniter')
calFlux = parset.getfloat('LOFAR_dd-parallel', 'calFlux')
userReg = parset.get('model', 'userReg')
aterm_imaging = False
MSs_self = lib_ms.AllMSs(glob.glob('mss/TC*[0-9].MS'), s)
# make beam
phasecentre = MSs_self.getListObj()[0].getPhaseCentre()
fwhm = MSs_self.getListObj()[0].getFWHM(freq='mid')
# Survey
#if 'LBAsurvey' in os.getcwd():
# obs = os.getcwd().split('/')[-1]
# if not os.path.exists('mss'):
# os.makedirs('mss')
# for i, tc in enumerate(glob.glob('/home/fdg/data/LBAsurvey/c*-o*/%s/mss/*' % obs)):
# tc_ren = 'TC%02i.MS' % i
# print('cp -r %s mss/%s' % (tc,tc_ren))
# os.system('cp -r %s mss/%s' % (tc,tc_ren))
########################################################
from LiLF import lib_ms, lib_img, lib_util, lib_log
logger_obj = lib_log.Logger('pipeline-self.logger')
logger = lib_log.logger
s = lib_util.Scheduler(log_dir=logger_obj.log_dir, dry=False)
w = lib_util.Walker('pipeline-self.walker')
parset = lib_util.getParset()
parset_dir = parset.get('LOFAR_self', 'parset_dir')
sourcedb = parset.get('model', 'sourcedb')
apparent = parset.getboolean('model', 'apparent')
userReg = parset.get('model', 'userReg')
#############################################################################
# Clear
if w.todo('cleaning'):
logger.info('Cleaning...')
lib_util.check_rm('img')
os.makedirs('img')
# here images, models, solutions for each group will be saved
#!/usr/bin/env python
# -*- coding: utf-8 -*-
import sys, os, glob
import numpy as np
import lsmtool
from astropy.table import Table as astrotab
########################################################
from LiLF import lib_ms, lib_img, lib_util, lib_log
logger_obj = lib_log.Logger('pipeline-3c.logger')
logger = lib_log.logger
s = lib_util.Scheduler(log_dir=logger_obj.log_dir, dry=False)
w = lib_util.Walker('pipeline-3c.walker')
# parse parset
parset = lib_util.getParset()
parset_dir = parset.get('LOFAR_3c', 'parset_dir')
skydb_demix = parset.get('LOFAR_demix', 'demix_model')
bl2flag = parset.get('flag', 'stations')
target = os.getcwd().split('/')[-1]
data_dir = '/home/fdg/lofar5/3Csurvey/%s' % target
extended_targets = [
'3c223', '3c231', '3c236', '3c264', '3c274', '3c284', '3c285', '3c293',
'3c296', '3c31', '3c310', '3c326', '3c33', '3c35', '3c382', '3c386',
'3c442a', '3c449', '3c454.3', '3c465', '3c84'
]
very_extended_targets = ['3c138', 'da240']
#!/usr/bin/env python
# -*- coding: utf-8 -*-
# Pipeline to run on the calibrator observation.
# It isolates various systematic effects and
# prepare them for the transfer to the target field.
import sys, os, glob, re
import numpy as np
########################################################
from LiLF import lib_ms, lib_img, lib_util, lib_log
logger_obj = lib_log.Logger('pipeline-cal.logger')
logger = lib_log.logger
s = lib_util.Scheduler(log_dir=logger_obj.log_dir, dry=False)
w = lib_util.Walker('pipeline-cal.walker')
# parse parset
parset = lib_util.getParset()
parset_dir = parset.get('LOFAR_cal', 'parset_dir')
data_dir = parset.get('LOFAR_cal', 'data_dir')
skymodel = parset.get('LOFAR_cal', 'skymodel')
imaging = parset.getboolean('LOFAR_cal', 'imaging')
bl2flag = parset.get('flag', 'stations')
#############################################################
MSs = lib_ms.AllMSs(glob.glob(data_dir + '/*MS'), s, check_flags=False)
if w.todo('copy'):
# copy data
logger.info('Copy data...')
# sources outside of the region and perform subsequent self-calibration.
# A userReg may be specified as clean mask.
# phSolMode can be used to solve either using phases or phaseandtec.
import sys, os, glob, re
import numpy as np
import pyregion
from losoto.h5parm import h5parm
#######################################################
from LiLF import lib_ms, lib_img, lib_util, lib_log
logger_obj = lib_log.Logger('pipeline-extract.logger')
logger = lib_log.logger
s = lib_util.Scheduler(log_dir=logger_obj.log_dir, dry=False)
w = lib_util.Walker('pipeline-extract.walker')
# parse parset
parset = lib_util.getParset()
parset_dir = parset.get('LOFAR_extract', 'parset_dir')
maxniter = parset.getint('LOFAR_extract', 'maxniter')
target_reg_file = parset.get('LOFAR_extract',
'extractRegion') # default 'target.reg'
phSolMode = parset.get('LOFAR_extract', 'phSolMode') # default: tecandphase
if phSolMode not in ['tecandphase', 'phase']:
logger.error('phSolMode {} not supported. Choose tecandphase, phase.')
sys.exit()
userReg = parset.get('model', 'userReg')
############################
with w.if_todo('cleaning'):
# TODO: remove regions and move to masks
import sys, os, glob, re, pickle, collections
import numpy as np
from astropy.table import Table as astrotab
from astropy.coordinates import SkyCoord
from astropy import units as u
import pyrap.tables as pt
import lsmtool
#######################################################
from LiLF import lib_ms, lib_img, lib_util, lib_log, lib_dd, lib_h5
logger_obj = lib_log.Logger('pipeline-dd-serial.logger')
logger = lib_log.logger
s = lib_util.Scheduler(log_dir=logger_obj.log_dir, dry=False)
w = lib_util.Walker('pipeline-dd-serial.walker')
# parse parset
parset = lib_util.getParset()
parset_dir = parset.get('LOFAR_dd-serial', 'parset_dir')
userReg = parset.get('model', 'userReg')
maxIter = parset.getint('LOFAR_dd-serial', 'maxIter')
min_cal_flux60 = parset.getfloat('LOFAR_dd-serial', 'minCalFlux60')
removeExtendedCutoff = parset.getfloat('LOFAR_dd-serial',
'removeExtendedCutoff')
target_dir = parset.get('LOFAR_dd-serial', 'target_dir')
def clean(p, MSs, res='normal', size=[1, 1], empty=False, imagereg=None):
"""
p = patch name
#!/usr/bin/env python
import os, sys, glob, getpass, socket, re
from LiLF.surveys_db import SurveysDB
from LiLF import lib_ms, lib_img, lib_util, lib_log
logger_obj = lib_log.Logger('PiLL.logger')
logger = lib_log.logger
s = lib_util.Scheduler(log_dir=logger_obj.log_dir, dry=False)
w = lib_util.Walker('PiLL.walker')
LiLF_dir = os.path.dirname(os.path.dirname(lib_util.__file__))
parset = lib_util.getParset(parsetFile='lilf.config')
survey_projects = 'LT14_002,LC12_017,LC9_016,LC8_031' # list of projects related with the LBA survey
# get parameters
# use lilf.config (this is also used by all other scripits)
working_dir = os.path.abspath(parset.get('PiLL', 'working_dir'))
redo_cal = parset.getboolean('PiLL', 'redo_cal')
project = parset.get('PiLL', 'project')
target = parset.get('PiLL', 'target')
download_file = parset.get('PiLL', 'download_file')
if download_file != '': download_file = os.path.abspath(download_file)
def calibrator_tables_available(obsid):
"""
check if calibrator data exist in the database
"""
with SurveysDB(survey='lba', readonly=True) as sdb: