# parse parset
parset = lib_util.getParset()
parset_dir = parset.get('cal', 'parset_dir')
data_dir = parset.get('cal', 'data_dir')
skymodel = parset.get('cal', 'skymodel')
imaging = parset.getboolean('cal', 'imaging')
bl2flag = parset.get('flag', 'stations')
if 'LBAsurvey' in os.getcwd():
obs = os.getcwd().split('/')[-2] # assumes .../c??-o??/3c196
calname = os.getcwd().split('/')[-1] # assumes .../c??-o??/3c196
data_dir = '../../download/%s/%s' % (obs, calname)
#############################################################
MSs = lib_ms.AllMSs(glob.glob(data_dir + '/*MS'), s)
# copy data
logger.info('Copy data...')
for MS in MSs.getListObj():
MS.move(MS.nameMS + '.MS', keepOrig=True)
MSs = lib_ms.AllMSs(glob.glob('*MS'), s)
calname = MSs.getListObj()[0].getNameField()
obsmode = MSs.getListObj()[0].getObsMode()
if min(MSs.getFreqs()) < 40.e6:
iono3rd = True
logger.debug('Include iono 3rd order.')
else:
iono3rd = False
os.makedirs('extract/init')
os.system('cp ddcal/c01/images/wideDD-c01.app.restored.fits extract/init/'
) # copy ddcal image
os.system('cp ddcal/c01/images/wideDD-c01.DicoModel extract/init/'
) # copy dico model
os.system(
'cp ddcal/c01/solutions/interp.h5 extract/init/') # copy fnal dde sol
lib_util.check_rm('mss-extract')
if not os.path.exists('mss-extract'):
logger.info('Copy MS...')
os.system('cp -r mss-avg mss-extract')
# region must be a list of ds9 circles and polygons (other shapes can be implemented in lib_util.Rgion_helper()
target_reg = lib_util.Region_helper(target_reg_file)
center = target_reg.get_center() # center of the extract region
MSs = lib_ms.AllMSs(glob.glob('mss-extract/*MS'), s)
ch_out = MSs.getChout(4e6) # chout from dd-serial
fwhm = MSs.getListObj()[0].getFWHM(freq='mid')
phase_center = MSs.getListObj()[0].getPhaseCentre()
# read image, h5parm, make mask
wideDD_image = lib_img.Image('extract/init/wideDD-c01.app.restored.fits')
dde_h5parm = 'extract/init/interp.h5'
# make mask for subtraction
mask_ddcal = wideDD_image.imagename.replace(
'.fits', '_mask-ddcal.fits') # this is used to find calibrators
wideDD_image.makeMask(threshpix=5,
atrous_do=True,
maskname=mask_ddcal,
write_srl=True,
write_ds9=True)
logger = lib_log.logger
s = lib_util.Scheduler(log_dir=logger_obj.log_dir,
dry=False) #, maxThreads = 4)
# parse parset
parset = lib_util.getParset()
parset_dir = parset.get('LOFAR_ddfacet', 'parset_dir')
maxniter = parset.getint('LOFAR_ddfacet', 'maxniter')
calFlux = parset.getfloat('LOFAR_ddfacet', 'calFlux')
userReg = parset.get('model', 'userReg')
uvrange = [0.03, 1e6]
image_robust = -0.15
ncluster = 5
####################################################
MSs_self = lib_ms.AllMSs(glob.glob('mss/TC*[0-9].MS'), s)
# make beam
phasecentre = MSs_self.getListObj()[0].getPhaseCentre()
fwhm = MSs_self.getListObj()[0].getFWHM(freq='mid')
nchan = MSs_self.getListObj()[0].getNchan()
############################
logger.info('Cleaning...')
lib_util.check_rm('ddfcal')
os.makedirs('ddfcal/masks')
os.makedirs('ddfcal/plots')
os.makedirs('ddfcal/images')
os.makedirs('ddfcal/solutions')
os.makedirs('ddfcal/cache')
lib_util.check_rm('img')
import numpy as np
###############################################
from LiLF import lib_ms, lib_util, lib_log
logger_obj = lib_log.Logger('pipeline-demix.logger')
logger = lib_log.logger
s = lib_util.Scheduler(log_dir=logger_obj.log_dir, dry=False)
# parse parset
parset = lib_util.getParset()
parset_dir = parset.get('LOFAR_demix', 'parset_dir')
data_dir = parset.get('LOFAR_demix', 'data_dir')
skydb = parset.get('LOFAR_demix', 'demix_model')
##############################################
# Demix
MSs = lib_ms.AllMSs(glob.glob(data_dir + '/*MS'), s)
for MS in MSs.getListStr():
lib_util.check_rm(os.path.basename(MS) + '_' + os.path.basename(skydb))
print(('cp -r ' + skydb + ' ' + os.path.basename(MS) + '_' +
os.path.basename(skydb)))
os.system('cp -r ' + skydb + ' ' + os.path.basename(MS) + '_' +
os.path.basename(skydb))
logger.info('Demixing...')
MSs.run('DPPP '+parset_dir+'/DPPP_demix.parset msin=$pathMS msout=$nameMS demixer.skymodel=$nameMS.MS_'+os.path.basename(skydb)+' demixer.instrumentmodel=$nameMS/instrument_demix', \
log='$nameMS_demix.log', commandType='DPPP')
logger.info("Done.")
########################################################
from LiLF import lib_ms, lib_img, lib_util, lib_log, lib_dd
logger_obj = lib_log.Logger('pipeline-self.logger')
logger = lib_log.logger
s = lib_util.Scheduler(log_dir=logger_obj.log_dir, dry=False)
parset = lib_util.getParset()
parset_dir = parset.get('uGMRT_self', 'parset_dir')
sourcedb = parset.get('model',
'sourcedb') # relative to tgts dir "./tgts/xxx/model.txt
userReg = parset.get('model',
'userReg') # relative to tgts dir "./tgts/xxx/region.ref"
MSs = lib_ms.AllMSs(glob.glob('mss/*.MS'), s)
############################################################################
# Clear
logger.info('Cleaning...')
lib_util.check_rm('img')
os.makedirs('img')
lib_util.check_rm('self')
os.makedirs('self/images')
os.makedirs('self/solutions')
os.makedirs('self/plots')
is_wideband = len(
MSs.getListObj()[0].getFreqs()) > 1000 # if > 1000 chans, it is wideband
# make beam
cal_dir = parset.get('LOFAR_timesplit', 'cal_dir')
ngroups = parset.getint('LOFAR_timesplit', 'ngroups')
initc = parset.getint(
'LOFAR_timesplit',
'initc') # initial tc num (useful for multiple observation of same target)
bl2flag = parset.get('flag', 'stations')
#################################################
# Clean
with w.if_todo('clean'):
logger.info('Cleaning...')
lib_util.check_rm('mss*')
### DONE
with w.if_todo('copy'):
MSs = lib_ms.AllMSs(glob.glob(data_dir + '/*MS'), s)
logger.info('Copy data...')
for MS in MSs.getListObj():
#if min(MS.getFreqs()) > 30.e6:
# overwrite=True to prevent updating the weights twice
MS.move(MS.nameMS + '.MS', keepOrig=True, overwrite=True)
### DONE
MSs = lib_ms.AllMSs(glob.glob('*MS'), s)
##################################################
# Find solutions to apply
if cal_dir == '':
obsid = MSs.getListObj()[0].getObsID()
# try standard location
for i, line in enumerate(df):
ms = re.findall(r'L[0-9]*.*_SB[0-9]*_uv', line)[0]
if ms + '.MS' in downloaded or ms + '.dppp.MS' in downloaded:
continue
if ms + '.MS' in glob.glob('*MS') or ms + '.dppp.MS' in glob.glob(
'*MS'):
continue
s.add('wget -nv "' + line[:-1] + '" -O - | tar -x',
log=ms + '_download.log',
commandType='general')
# print 'wget -nv "'+line[:-1]+'" -O - | tar -x'
logger.debug('Queue download of: ' + line[:-1])
s.run(check=True, maxThreads=4)
MSs = lib_ms.AllMSs(glob.glob('*MS'), s)
if len(MSs.getListStr()) == 0:
logger.info('Done.')
sys.exit(0)
#######################################
if fix_table:
logger.info('Fix MS table...')
MSs.run('fixMS_TabRef.py $pathMS',
log='$nameMS_fixms.log',
commandType='python')
# only ms created in range (2/2013->2/2014)
with pt.table(MSs.getListStr()[0] + '/OBSERVATION',
readonly=True,
ack=False) as obs:
for i, line in enumerate(df):
ms = re.findall(r'L[0-9]*.*_SB[0-9]*_uv', line)[0]
if ms + '.MS' in downloaded or ms + '.dppp.MS' in downloaded:
continue
if ms + '.MS' in glob.glob('*MS') or ms + '.dppp.MS' in glob.glob(
'*MS'):
continue
s.add('wget -nv "' + line[:-1] + '" -O - | tar -x',
log=ms + '_download.log',
commandType='general')
# print 'wget -nv "'+line[:-1]+'" -O - | tar -x'
logger.debug('Queue download of: ' + line[:-1])
s.run(check=True, maxThreads=4)
MSs = lib_ms.AllMSs(glob.glob('*MS'), s, check_flags=False)
if len(MSs.getListStr()) == 0:
logger.info('Done.')
sys.exit(0)
######################################
with pt.table(MSs.getListStr()[0] + '/OBSERVATION', readonly=True,
ack=False) as obs:
t = Time(obs.getcell('TIME_RANGE', 0)[0] / (24 * 3600.), format='mjd')
time = np.int(t.iso.replace('-', '')[0:8])
if fix_table:
#logger.info('Fix MS table...')
#MSs.run('fixMS_TabRef.py $pathMS', log='$nameMS_fixms.log', commandType='python')
# only ms created in range (2/2013->2/2014)
from LiLF import lib_ms, lib_img, lib_util, lib_log
logger_obj = lib_log.Logger('pipeline-cal.logger')
logger = lib_log.logger
s = lib_util.Scheduler(log_dir=logger_obj.log_dir, dry=False)
w = lib_util.Walker('pipeline-cal.walker')
# parse parset
parset = lib_util.getParset()
parset_dir = parset.get('LOFAR_cal', 'parset_dir')
data_dir = parset.get('LOFAR_cal', 'data_dir')
skymodel = parset.get('LOFAR_cal', 'skymodel')
imaging = parset.getboolean('LOFAR_cal', 'imaging')
bl2flag = parset.get('flag', 'stations')
#############################################################
MSs = lib_ms.AllMSs(glob.glob(data_dir + '/*MS'), s, check_flags=False)
with w.if_todo('copy'):
# copy data
logger.info('Copy data...')
for MS in MSs.getListObj():
MS.move(MS.nameMS + '.MS', keepOrig=True, overwrite=False)
### DONE
MSs = lib_ms.AllMSs(glob.glob('*MS'), s, check_flags=False)
calname = MSs.getListObj()[0].getNameField()
for MS in MSs.getListObj():
os.system('cp -r %s %s' % (skymodel, MS.pathMS))
if min(MSs.getFreqs()) < 35.e6:
###############################################
from LiLF import lib_ms, lib_util, lib_log
logger_obj = lib_log.Logger('pipeline-demix.logger')
logger = lib_log.logger
s = lib_util.Scheduler(log_dir=logger_obj.log_dir, dry=False)
# parse parset
parset = lib_util.getParset()
parset_dir = parset.get('LOFAR_demix', 'parset_dir')
data_dir = parset.get('LOFAR_demix', 'data_dir')
demix_skymodel = parset.get('LOFAR_demix',
'demix_model') # skymodel to be converted in skydb
include_target = parset.getboolean('LOFAR_demix', 'include_target')
##############################################
MSs = lib_ms.AllMSs(glob.glob('mss-predemix/TC*[0-9].MS'), s)
ateams = ['VirA', 'TauA']
ateams_todemix = []
for ateam in ateams:
sep = MSs.getListObj()[0].distBrightSource(ateam)
if sep < 4 or sep > 15:
logger.debug('No demix of %s (sep: %.0f deg)' % (ateam, sep))
else:
ateams_todemix.append(ateam)
logger.warning('Demix of %s (sep: %.0f deg)' % (ateam, sep))
if len(ateams_todemix) < 0:
logger.info('Notingh to demix.')
sys.exit()
if os.path.exists('mss-predemix'):
s = lib_util.Scheduler(log_dir=logger_obj.log_dir, dry=False)
# parse parset
parset = lib_util.getParset()
parset_dir = parset.get('uGMRT_cal', 'parset_dir')
skymodel = parset.get('uGMRT_cal', 'skymodel')
bl2flag = parset.get('flag', 'stations')
#############################################################
# cicle on each msID assuming telescope properties are similar in all calibrator scans of the same msID
msIDs = np.unique([int(name[5:7]) for name in glob.glob('cals/*')])
for msID in msIDs:
logger.info('Working on MSid %02i' % msID)
MSs_cals = lib_ms.AllMSs(glob.glob('cals/%02i*/*MS' % msID), s)
calnames = [MS.getNameField() for MS in MSs_cals.getListObj()]
for MS in MSs_cals.getListObj():
os.system('cp -r %s %s' % (skymodel, MS.pathMS))
logger.info('Add columns...')
MSs_cals.run('addcol2ms.py -m $pathMS -c MODEL_DATA -i DATA',
log="$nameMS_addcol.log",
commandType="python")
# predict to save time ms:MODEL_DATA
for i, MS in enumerate(MSs_cals.getListObj()):
logger.info('%s: add model to MODEL_DATA (%s)...' %
(MS.pathMS, calnames[i]))
s.add("DPPP " + parset_dir + "/DPPP-predict.parset msin="+MS.pathMS+" pre.sourcedb="+MS.pathMS+"/" + os.path.basename(skymodel) + " pre.sources=" + calnames[i], \
log=MS.nameMS+"_pre.log", commandType="DPPP")
if timestamp in cal_timestamps:
logger.info('Calibrator found: %s (t=%s)' % (cal_dir, timestamp))
return cal_dir + '/cals'
logger.error('Missing calibrator.')
sys.exit()
##########################################################
logger.info('Cleaning...')
lib_util.check_rm('cal*h5')
lib_util.check_rm('plots*')
lib_util.check_rm('img')
os.makedirs('img')
MSs = lib_ms.AllMSs(glob.glob(data_dir + '/*MS'), s, check_flags=False)
for timestamp in set(
[os.path.basename(ms).split('_')[1][1:] for ms in MSs.getListStr()]):
mss_toconcat = glob.glob(data_dir + '/' + target + '_t' + timestamp +
'_SB*.MS')
MS_concat = target + '_t' + timestamp + '_concat.MS'
MS_concat_bkp = target + '_t' + timestamp + '_concat.MS-bkp'
if os.path.exists(MS_concat_bkp):
logger.info('Restoring bkp data...')
os.system('rm -r %s' % MS_concat)
os.system('cp -r %s %s' % (MS_concat_bkp, MS_concat))
else:
logger.info('Making %s...' % MS_concat)
s.add('DPPP '+parset_dir+'/DPPP-avg.parset msin=\"'+str(mss_toconcat)+'\" msout='+MS_concat,\
log=MS_concat+'_avg.log', commandType='DPPP')
# parse parset
parset = lib_util.getParset('lilf.config')
parset_dir = parset.get('uGMRT_init', 'parset_dir')
data_dir = parset.get('uGMRT_init', 'data_dir')
bl2flag = parset.get('flag', 'antennas')
#############################
# 1. Download data from NCRA server
#############################
# 2. Convert into MS
#############################
# 3. Set-up directory structure ans split
MSs = lib_ms.AllMSs(glob.glob(data_dir + '/*MS'), s)
numberOfMSs = len(MSs.getListObj())
if not os.path.exists('cals'): os.mkdir('cals')
if not os.path.exists('tgts'): os.mkdir('tgts')
for msID, MS in enumerate(MSs.getListObj()):
pathMS = MS.pathMS
logger.info("- Split-up of original MS '%s' (msID: %02i) -" %
(pathMS, msID))
scanIDs = tables.taql("select distinct SCAN_NUMBER from $pathMS").getcol(
"SCAN_NUMBER")
numberOfScans = len(scanIDs)
from LiLF import lib_ms, lib_img, lib_util, lib_log
logger_obj = lib_log.Logger('pipeline-cal.logger')
logger = lib_log.logger
s = lib_util.Scheduler(log_dir=logger_obj.log_dir, dry=False)
w = lib_util.Walker('pipeline-cal.walker')
# parse parset
parset = lib_util.getParset()
parset_dir = parset.get('LOFAR_cal', 'parset_dir')
data_dir = parset.get('LOFAR_cal', 'data_dir')
skymodel = parset.get('LOFAR_cal', 'skymodel')
imaging = parset.getboolean('LOFAR_cal', 'imaging')
bl2flag = parset.get('flag', 'stations')
#############################################################
MSs = lib_ms.AllMSs(glob.glob(data_dir + '/*MS'), s, check_flags=False)
if w.todo('copy'):
# copy data
logger.info('Copy data...')
for MS in MSs.getListObj():
MS.move(MS.nameMS + '.MS', keepOrig=True, overwrite=False)
w.done('copy')
### DONE
MSs = lib_ms.AllMSs(glob.glob('*MS'), s, check_flags=False)
calname = MSs.getListObj()[0].getNameField()
for MS in MSs.getListObj():
os.system('cp -r %s %s' % (skymodel, MS.pathMS))
import pyrap.tables as pt
from astropy.time import Time
import numpy as np
########################################################
from LiLF import lib_ms, lib_img, lib_util, lib_log
logger_obj = lib_log.Logger('pipeline-test.logger')
logger = lib_log.logger
s = lib_util.Scheduler(log_dir = logger_obj.log_dir, dry = False)
# parse parset
#parset = lib_util.getParset()
parset_dir = '/home/baq1889/scripts/LiLF/parsets/LOFAR_dd'
#############################################################
MSs = lib_ms.AllMSs( glob.glob('*MS'), s, check_flags=False )
skymodel_voro_skydb = 'skymodel00_voro.skydb'
c=0
# Calibration - ms:SMOOTHED_DATA
logger.info('Calibrating...')
MSs.run('DPPP '+parset_dir+'/DPPP-solDD.parset msin=$pathMS ddecal.h5parm=$pathMS/cal-core.h5 \
ddecal.sourcedb='+skymodel_cl_skydb+' ddecal.solint=15 ddecal.nchan=30', \
log='$nameMS_solDD-core.log', commandType='DPPP')
lib_util.run_losoto(s, 'core', [MS+'/cal-core.h5' for MS in MSs.getListStr()], \
[parset_dir+'/losoto-core.parset'])
logger.info('re-calibration...')
# predict and corrupt each facet
return cal_dir + '/cals'
logger.error('Missing calibrator.')
sys.exit()
##########################################################
with w.if_todo('setup'):
logger.info('Cleaning...')
lib_util.check_rm('cal*h5')
lib_util.check_rm('plots*')
lib_util.check_rm('peel*')
lib_util.check_rm('img')
os.makedirs('img')
MSs = lib_ms.AllMSs(glob.glob(data_dir + '/*MS'), s, check_flags=False)
for timestamp in set(
[os.path.basename(ms).split('_')[1][1:] for ms in MSs.getListStr()]):
mss_toconcat = sorted(
glob.glob(data_dir + '/' + target + '_t' + timestamp + '_SB*.MS'))
MS_concat = target + '_t' + timestamp + '_concat.MS'
MS_concat_bkp = target + '_t' + timestamp + '_concat.MS-bkp'
if os.path.exists(MS_concat_bkp):
logger.info('Restoring bkp data: %s...' % MS_concat_bkp)
lib_util.check_rm(MS_concat)
os.system('cp -r %s %s' % (MS_concat_bkp, MS_concat))
else:
logger.info('Making %s...' % MS_concat)
s.add('DPPP '+parset_dir+'/DPPP-avg.parset msin=\"'+str(mss_toconcat)+'\" msout='+MS_concat,\
if w.todo('cleaning'):
logger.info('Cleaning...')
lib_util.check_rm('plot*')
lib_util.check_rm('cal*h5')
lib_util.check_rm('img')
lib_util.check_rm('facet')
os.makedirs('img')
os.makedirs('facet')
lib_util.check_rm('mss-facet')
if not os.path.exists('mss-facet'): os.system('cp -r mss-dd mss-facet')
w.done('cleaning')
### DONE
MSs = lib_ms.AllMSs(glob.glob('mss-facet/*MS'), s)
def clean(p, MSs, size, res='normal', apply_beam=False):
"""
p = patch name
mss = list of mss to clean
size = in deg of the image
"""
# set pixscale and imsize
pixscale = MSs.getListObj()[0].getResolution()
if res == 'normal':
pixscale = float('%.1f' % (pixscale / 2.5))
elif res == 'high':
pixscale = float('%.1f' % (pixscale / 3.5))
elif res == 'low':
from LiLF import lib_ms, lib_img, lib_util, lib_log, lib_dd_parallel
logger_obj = lib_log.Logger('pipeline-dd.logger')
logger = lib_log.logger
s = lib_util.Scheduler(log_dir=logger_obj.log_dir, dry=False)
w = lib_util.Walker('pipeline-dd.walker')
# parse parset
parset = lib_util.getParset()
parset_dir = parset.get('LOFAR_dd-parallel', 'parset_dir')
maxniter = parset.getint('LOFAR_dd-parallel', 'maxniter')
calFlux = parset.getfloat('LOFAR_dd-parallel', 'calFlux')
userReg = parset.get('model', 'userReg')
aterm_imaging = False
MSs_self = lib_ms.AllMSs(glob.glob('mss/TC*[0-9].MS'), s)
# make beam
phasecentre = MSs_self.getListObj()[0].getPhaseCentre()
fwhm = MSs_self.getListObj()[0].getFWHM(freq='mid')
def clean(p, MSs, size, res='normal', apply_beam=False):
"""
p = patch name
mss = list of mss to clean
size = in deg of the image
"""
# set pixscale and imsize
pixscale = MSs.resolution
if res == 'normal':
'.log | grep "background noise"')
#############################################################
with w.if_todo('cleaning'):
logger.info('Cleaning...')
lib_util.check_rm('ddcal')
os.makedirs('ddcal/init')
lib_util.check_rm('img')
os.makedirs('img')
lib_util.check_rm('mss-avg')
### DONE
# goes down to 8 seconds/4 chans
if not os.path.exists('mss-avg'):
MSs = lib_ms.AllMSs(glob.glob('mss/TC*[0-9].MS'), s)
timeint = MSs.getListObj()[0].getTimeInt()
avgtimeint = int(round(8 / timeint)) # to 8 seconds
nchan_init = MSs.getListObj()[0].getNchan()
# chan: avg (x8) sol (x6) - we need a multiple of 8x6=48, the largest that is <nchan
# survey after avg (x8): 60, final number of sol 10
# pointed after avg (x8): 120, final number of sol 20
nchan = nchan_init - nchan_init % 48
os.makedirs('mss-avg')
logger.info('Averaging in time (%is -> %is), channels: %ich -> %ich)' %
(timeint, timeint * avgtimeint, nchan_init, nchan))
MSs.run(
'DPPP ' + parset_dir +
'/DPPP-avg.parset msin=$pathMS msout=mss-avg/$nameMS.MS msin.datacolumn=CORRECTED_DATA msin.nchan='
+ str(nchan) + ' \
avg.timestep=' + str(avgtimeint) + ' avg.freqstep=1',
# Clear
if w.todo('cleaning'):
logger.info('Cleaning...')
lib_util.check_rm('img')
os.makedirs('img')
# here images, models, solutions for each group will be saved
lib_util.check_rm('self')
if not os.path.exists('self/images'): os.makedirs('self/images')
if not os.path.exists('self/solutions'): os.makedirs('self/solutions')
if not os.path.exists('self/plots'): os.makedirs('self/plots')
w.done('cleaning')
### DONE
MSs = lib_ms.AllMSs(glob.glob('mss/TC*[0-9].MS'), s)
try:
MSs.print_HAcov()
except:
logger.error('Problem with HAcov, continue anyway.')
# make beam to the first mid null
phasecentre = MSs.getListObj()[0].getPhaseCentre()
MSs.getListObj()[0].makeBeamReg('self/beam.reg', freq='mid', to_null=True)
beamReg = 'self/beam.reg'
# set image size
imgsizepix = int(2.1 * MSs.getListObj()[0].getFWHM(freq='mid') * 3600 / 10.)
if imgsizepix % 2 != 0: imgsizepix += 1 # prevent odd img sizes
#################################################################
logger = lib_log.logger
s = lib_util.Scheduler(log_dir=logger_obj.log_dir, dry=False)
# parse parset
parset = lib_util.getParset()
parset_dir = parset.get('LOFAR_ateam', 'parset_dir')
bl2flag = parset.get('flag', 'stations')
data_dir = '../tgts-bkp/'
##########################################################
logger.info('Cleaning...')
#lib_util.check_rm('cal*h5')
lib_util.check_rm('plots*')
lib_util.check_rm('img')
os.makedirs('img')
MSs = lib_ms.AllMSs(sorted(glob.glob(data_dir + '/*MS')), s)
# copy data (avg to 1ch/sb and 10 sec)
nchan = int(MSs.getListObj()[0].getNchan()) # average 4 chans
timeint = MSs.getListObj()[0].getTimeInt()
avg_time = int(np.rint(30. / timeint)) # average to 30 s
logger.info('Copy data...')
for obs in set([os.path.basename(ms).split('_')[0]
for ms in MSs.getListStr()]):
mss_toconcat = sorted(glob.glob(data_dir + '/' + obs + '*MS'))
# add missing SB with a fake name not to leave frequency holes
num_init = int(re.findall(r'\d+', mss_toconcat[0])[-1])
num_fin = int(re.findall(r'\d+', mss_toconcat[-1])[-1])
help="If you want to provide a certain skymodel.")
args = parser.parse_args()
#############################################################################
# Clear
with w.if_todo('cleaning'):
lib_util.check_rm('bench')
if not os.path.exists('bench/img'): os.makedirs('bench/img')
logger.info('Cleaning...')
lib_util.check_rm('bench/img')
os.makedirs('bench/img')
lib_util.check_rm('bench/plots')
os.makedirs('bench/plots')
### DONE
print(args.msfiles)
MSs = lib_ms.AllMSs(glob.glob(args.msfiles), s)
sourcedb = args.sourcedb
ncpu = mp.cpu_count()
nms = len(MSs.getListObj())
logger.info(f'Found {ncpu} cpus and {nms} MS files')
# set image size
imgsizepix = int(2.1 * MSs.getListObj()[0].getFWHM(freq='mid') * 3600 / 10.)
if imgsizepix % 2 != 0: imgsizepix += 1 # prevent odd img sizes
# set clean componet fit order (use 5 for large BW)
bandwidth = MSs.getBandwidth()
# make beam to the first mid null
phasecentre = MSs.getListObj()[0].getPhaseCentre()
# MSs.getListObj()[0].makeBeamReg('bench/beam.reg', freq='mid', to_null=True)
from LiLF import lib_ms, lib_img, lib_util, lib_log
lib_log.Logger('pipeline-ateam.logger')
logger = lib_log.logger
s = lib_util.Scheduler(dry=False)
# parse parset
parset = lib_util.getParset()
parset_dir = parset.get('ateam', 'parset_dir')
bl2flag = parset.get('flag', 'stations')
data_dir = '../tgts-bkp/'
##########################################################
lib_util.check_rm('img')
os.makedirs('img')
mss = sorted(glob.glob(data_dir + '/*MS'))
MSs = lib_ms.AllMSs(mss, s)
# HBA/LBA
if min(MSs.getFreqs()) < 80.e6: hba = False
else: hba = True
# wsclean temp-dir
temp_dir = '.'
if s.get_cluster() == 'Hamburg_fat': temp_dir = '/localwork.ssd'
# copy data (avg to 1ch/sb and 10 sec)
nchan = MSs.getListObj()[0].getNchan()
timeint = MSs.getListObj()[0].getTimeInt()
avg_time = int(np.rint(10. / timeint))
logger.info('Copy data...')
