def do_verify_subtract(mslist, res_val, source, numchanperms=20):
''' main function for verify_subtract '''
msavglist = []
for ms_id, ms in enumerate(mslist):
msavglist.append(ms.split('.')[0] + '.' + source + '.ms.avgcheck')
username = pwd.getpwuid(os.getuid())[0]
###########################################################################
# NDPPP phase shift, less averaging (NEW: run 2 in parallel)
b = bg(maxp=2)
for ms_id, ms in enumerate(mslist):
parset = create_phaseshift_parset_field(ms,
msavglist[ms_id],
numchanperms=numchanperms)
#ncmd='NDPPP ' + parset+' &>'+ms.split('.')[0] +'.ndppp_avgphaseshift_check.log'
ncmd = 'NDPPP ' + parset + ' >' + ms.split(
'.')[0] + '.ndppp_avgphaseshift_check.log' + ' 2>&1'
print 'Running', ncmd
b.run(ncmd)
# Check if all NDPPP processes are finished
b.wait()
###########################################################################
imsize = 2048
###########################################################################
# IMAGE IN PARALLEL
b = bg(maxp=8)
for ms in msavglist:
imout = 'im' + '_residual_' + source + '_' + ms.split('.')[0]
b.run('casapy --nogui -c '+SCRIPTPATH+'/casapy_cleanv4_checksubtract.py ' +\
ms + ' ' + imout + ' ' + str(imsize))
time.sleep(20)
# Check if all NDPPP processes are finished
b.wait()
#conver the images to FITS format
for ms in msavglist:
imout = 'im' + '_residual_' + source + '_' + ms.split('.')[0]
run('image2fits in=' + imout + '.image' + ' ' + 'out=' + imout +
'.fits')
stopcal = False
for ms in msavglist:
# find the source which was done before the current one
g = sorted(glob.glob('im_residual_*_' + ms.split('.')[0] + '.image'),
key=os.path.getmtime,
reverse=True)
if len(g) > 1:
pre_sourcename = g[1]
#print 'Previous image was', pre_sourcename
else:
pre_sourcename = None
image = 'im' + '_residual_' + source + '_' + ms.split(
'.')[0] + '.image'
img = pyrap.images.image(image)
pixels = numpy.copy(img.getdata())
maxval = numpy.max(pixels)
minval = numpy.min(pixels)
maxvalpre = 1e9
minvalpre = -maxvalpre
if pre_sourcename is not None:
imgpre = pyrap.images.image(pre_sourcename)
pixelspre = numpy.copy(imgpre.getdata())
maxvalpre = numpy.max(pixelspre)
minvalpre = numpy.min(pixelspre)
logging.info('verify_subtract: For %s max, min residual were %f, %f' %
(image, maxval, minval))
logging.debug("{} {} {}".format(maxval, minval, image))
if pre_sourcename is not None:
logging.debug("{} {} {}".format(maxvalpre, minvalpre,
pre_sourcename))
if (maxval > res_val) or ((maxval * 0.92) > maxvalpre):
stopcal = True
logging.warning('RESIDUAL TOO LARGE, STOPPING {} {}'.format(
maxval, res_val))
logging.warning(
'RESIDUAL TOO LARGE, STOPPING, previous max in image {}'.
format(maxvalpre))
return stopcal
def do_selfcal(mslist, cluster, atrous_do, imsize, nterms, cellsizetime_a, cellsizetime_p,
TECi, clocki, HRi, region, clusterdesc, dbserver, dbuser, dbname, SCRIPTPATH,
ncores=8, config=None):
TEC = False
FFT = False
clock= False
HR = False # high dynamic range
if TECi == "True":
TEC = True
if clocki == "True":
clock = True
if HRi == "HD":
HR = True
if imsize <=2048:
wplanes = 1
FFT = True # FFT image into MODEL_DATA
if imsize >= 512:
wplanes = 64
if imsize > 799:
wplanes = 96
if imsize > 1023:
wplanes = 128
if imsize > 1599:
wplanes = 180
if imsize > 1800:
wplanes = 196
if imsize > 2049:
wplanes = 256
#if imsize > 3000:
# wplanes = 448
#if imsize > 4095:
# wplanes = 512
logging.info('mslist {}'.format(mslist))
logging.info('source {}'.format(cluster))
logging.info('atrous_do {}'.format(atrous_do))
logging.info('imsize {} '.format(imsize))
logging.info('TEC is {} and clock is {}'.format(TEC, clock))
msinputlist = ''
for m in mslist:
msinputlist = msinputlist + ' ' + m
#if len(mslist) == 29:
#group = "9,10,10"
#elif len(mslist) == 28:
#group = "7,7,7,7"
#elif len(mslist) == 20:
##group = "10,10"
#group = "7,7,6"
#elif len(mslist) == 16:
#group = "8,8"
#else:
#group = str(len(mslist))
group = get_group(mslist)
logging.info('GROUP {}'.format(group))
uvrange = '80'
#uvrange = '400'
merge_parmdb = True
phasors = False # if true only solve for amps on long timescales
smooth = False # seems that smooth does not help the selfcal (various reasons for that)
# 1. boundaries of flagged vs non-flagged data are sharp (should not be smoothed)
# 2. there sre some strong ampl various at low elevations
smooth = True # sometimes almost 0.0 amplitude, causes ripples
phasezero = True # reset phases from ap calibration
## Loop parameters
rms_old = 1.e9 # bad values to start with
dynamicrange_old = 1. # low value to start with so we get into the while loop
im_count = 4
number_forced_selfcalcycles = 8
factor = 1.0125 # demand 1.25% improvement
max_selfcalcycles = 16
if config is not None:
number_forced_selfcalcycles = config.get("selfcal_forced_cycles", 8)
factor = config.get("selfcal_factor", 1.0125)
max_selfcalcycles = config.get("selfcal_max_cycles", 16)
empty_mask_cycle = config.get("selfcal_empty_cycle", 5)
logging.info('Selfcal loop params: forced {}; empty mask {}; max {}; factor {}'.format(
number_forced_selfcalcycles, empty_mask_cycle, max_selfcalcycles, factor))
#####################
#####################
#### MAKE IMAGE 0 ###
logging.info('Make image 0')
imout,mask = make_image(mslist, cluster, '0', 10, 6, nterms, atrous_do, imsize, region, ncores, SCRIPTPATH)
#####################
### CALIBRATE WITH BBS PHASE ONLY 1 ###
# create skymodel for BBS
run(SCRIPTPATH+'/casapy2bbs.py -m '+ mask + ' ' +'-t ' + str(nterms)+ ' ' + imout+'.model ' + imout+'.skymodel')
if FFT:
os.system('casapy --nogui -c '+SCRIPTPATH+'/ft_v2.py ' + msinputlist + ' ' + imout+'.model' \
+ ' ' + str(nterms) + ' '+ str(wplanes))
# phase only calibrate
skymodel = imout+'.skymodel'
parset = create_scalarphase_parset(cellsizetime_p, TEC, clock, group, False, uvrange)
runbbs(mslist, '3C244.1.skymodel', parset, 'instrument', False, TEC, clusterdesc, dbserver, dbuser, dbname)
#NOTE WORK FROM MODEL_DATA (contains correct phase data from 10SB calibration)
######################################
### MAKE IMAGE 1 ###
logging.info('Make image 1')
imout,mask = make_image(mslist, cluster, '1', 15, 15, nterms, atrous_do, imsize, region, ncores, SCRIPTPATH)
####################
### CALIBRATE WITH BBS PHASE ONLY 2 ###
# create skymodel for BBS
run(SCRIPTPATH+'/casapy2bbs.py -m '+ mask + ' ' +'-t ' + str(nterms)+ ' ' + imout+'.model ' + imout+'.skymodel')
if FFT:
run('casapy --nogui -c '+SCRIPTPATH+'/ft_v2.py ' + msinputlist + ' ' + imout+'.model' \
+ ' ' + str(nterms) + ' '+ str(wplanes))
# phase only calibrate
skymodel = imout+'.skymodel'
parset = create_scalarphase_parset(cellsizetime_p, TEC, clock, group, FFT, uvrange)
runbbs(mslist, skymodel, parset, 'instrument', False, TEC,clusterdesc, dbserver, dbuser, dbname) #NOTE WORK FROM MODEL_DATA (contains correct phase data from 10SB calibration)
######################################
### MAKE IMAGE 2 ###
logging.info('Make image 2')
imout,mask = make_image(mslist, cluster, '2', 15, 15, nterms, atrous_do, imsize, region, ncores, SCRIPTPATH)
####################
### CALIBRATE WITH BBS PHASE+AMP 1 ###
run(SCRIPTPATH+'/casapy2bbs.py -m '+ mask + ' ' +'-t ' + str(nterms)+ ' ' + imout+'.model ' + imout+'.skymodel')
if FFT:
run('casapy --nogui -c '+SCRIPTPATH+'/ft_v2.py ' + msinputlist + ' ' + imout+'.model' \
+ ' ' + str(nterms) + ' '+ str(wplanes))
skymodel = imout+'.skymodel'
parset = create_scalarphase_parset_p(cellsizetime_p, TEC, clock, group, FFT, uvrange)
# solve +apply phases
runbbs(mslist, skymodel, parset, 'instrument_phase0', False, TEC, clusterdesc, dbserver, dbuser, dbname)
# solve amps
parmdb = 'instrument_amps0'
parset = create_amponly_parset(cellsizetime_a, FFT, uvrange)
runbbs(mslist, skymodel, parset, parmdb, False, False, clusterdesc, dbserver, dbuser, dbname)
for ms in mslist:
# remove outliers from the solutions
if phasors:
run('python '+SCRIPTPATH+'/smoothcal_rx42.py ' + ms + ' ' + ms+'/'+parmdb + ' ' + ms+'/'+parmdb+'_smoothed'+' > '+ms+'_'+parmdb+'_smoothed.log')
else:
run('python '+SCRIPTPATH+'/smoothcal_a2256_nophasors.py ' + ms + ' ' + ms+'/'+parmdb + ' ' + ms+'/'+parmdb+'_smoothed'+' > '+ms+'_'+parmdb+'_smoothed.log')
# apply amps
if smooth:
runbbs(mslist, skymodel,SCRIPTPATH+'/apply_amplitudeonly.parset', parmdb+'_smoothed', True, False, clusterdesc, dbserver, dbuser, dbname)
else:
runbbs(mslist, skymodel,SCRIPTPATH+'/apply_amplitudeonly.parset', parmdb, True, False, clusterdesc, dbserver, dbuser, dbname)
### MAKE IMAGE 3 ###
logging.info('Make image 3')
imout,mask = make_image(mslist, cluster, '3', 10, 10, nterms, atrous_do, imsize, region, ncores, SCRIPTPATH)
####################
# LOOP
####################
rms, dynamicrange = find_imagenoise(imout + '.image')
while (((dynamicrange/factor) > dynamicrange_old) or ((rms*factor) < rms_old)):
logging.info('Starting selfcal loop {}'.format(im_count))
#### CALIBRATE BBS PHASE+AMP 2 (LOOP) ###
# make model
run(SCRIPTPATH+'/casapy2bbs.py -m '+ mask + ' ' +'-t ' + str(nterms)+ ' ' + imout+'.model ' + imout+'.skymodel')
if FFT:
run('casapy --nogui -c '+SCRIPTPATH+'/ft_v2.py ' + msinputlist + ' ' + imout+'.model' \
+ ' ' + str(nterms) + ' '+ str(wplanes))
#parmdb keep from previous step
skymodel = imout+'.skymodel'
# reset the phases from instrument_amps0 to zero to prevent large phase corrections from incorrect AP solve
# FIXME: Is this used?
if phasezero:
inputparmdb = parmdb +'_smoothed'
outputparmdb = parmdb +'_smoothed_phasezero'
for ms in mslist:
run('python '+SCRIPTPATH+'/setphasezero.py ' + ms + ' ' + ms+'/'+inputparmdb +' ' + ms+'/'+outputparmdb)
else:
outputparmdb = parmdb +'_smoothed'
# phase only cal
skymodel = imout+'.skymodel'
parset = create_scalarphase_parset_p(cellsizetime_p, TEC, clock, group, FFT, uvrange)
runbbs(mslist, skymodel, parset, 'instrument_phase1', False, TEC, clusterdesc, dbserver, dbuser, dbname)
# solve amps
parmdb = 'instrument_amps1'
parset = create_amponly_parset(cellsizetime_a, FFT, uvrange)
runbbs(mslist, skymodel, parset,parmdb, False, False, clusterdesc, dbserver, dbuser, dbname)
for ms in mslist:
# remove outliers from the solutions
if phasors:
run('python '+SCRIPTPATH+'/smoothcal_rx42.py ' + ms + ' ' + ms+'/'+parmdb + ' ' + ms+'/'+parmdb+'_smoothed'+' > '+ms+'_'+parmdb+'_smoothed.log')
else:
run('python '+SCRIPTPATH+'/smoothcal_a2256_nophasors.py ' + ms + ' ' + ms+'/'+parmdb + ' ' + ms+'/'+parmdb+'_smoothed'+' > '+ms+'_'+parmdb+'_smoothed.log')
# apply amps
if smooth:
runbbs(mslist, skymodel,SCRIPTPATH+'/apply_amplitudeonly.parset',parmdb+'_smoothed', True, False, clusterdesc, dbserver, dbuser, dbname)
else:
runbbs(mslist, skymodel,SCRIPTPATH+'/apply_amplitudeonly.parset',parmdb, True, False, clusterdesc, dbserver, dbuser, dbname)
### MAKE IMAGE #N ###
logging.info('Make image {}'.format(im_count))
# Do not use the initial mask in the final cycles
if im_count >= empty_mask_cycle:
region_im = "empty"
else:
region_im = region
imout,mask = make_image(mslist, cluster, str(im_count), 10, 10, nterms, atrous_do, imsize, region_im, ncores, SCRIPTPATH)
## Prepare the next iteration
im_count += 1
# save previous values to compare with
rms_old = rms
dynamicrange_old = dynamicrange
if nterms < 2:
rms, dynamicrange = find_imagenoise(imout + '.image')
else:
rms, dynamicrange = find_imagenoise(imout + '.image.tt0')
logging.info('IMAGE STATISTICS {}, {}'.format(rms, dynamicrange))
if im_count < number_forced_selfcalcycles:
rms_old = 1.e9 # bad values to start with
dynamicrange_old = 1.
logging.debug("Count below the number of forced selfcal cycles. Force new loop.")
if im_count >= max_selfcalcycles:
logging.info("Maximum number of cycles ({}) reached. Leaving selfcal loop.".format(max_selfcalcycles))
break
### CREATE FINAL MODEL ###
logging.info('Create final model')
skymodelf= 'im_cluster'+cluster+ '.final.skymodel'
run(SCRIPTPATH+'/casapy2bbs.py -m '+ mask + ' ' +'-t ' + str(nterms)+ ' ' + imout+'.model ' + skymodelf)
if FFT:
run('casapy --nogui -c '+SCRIPTPATH+'/ft_v2.py ' + msinputlist + ' ' + imout+'.model' \
+ ' ' + str(nterms) + ' '+ str(wplanes))
### CREATED MERGED PARMDB SCALARPHASE+AMPS ###
### INCLUDES SPLINE INTERPOLARION OF AMPS ###
if merge_parmdb:
logging.info('Merge parmdb')
if phasors:
dummyparset = SCRIPTPATH+'/scalarphase+amp.parset'
else:
if TEC:
dummyparset = SCRIPTPATH+'/scalarphase+ap+TEC.parset'
else:
dummyparset = SCRIPTPATH+'/scalarphase+ap.parset'
if TEC:
if clock:
dummyparmdb = 'instrument_template_TECclock'
else:
dummyparmdb = 'instrument_template_Gain_TEC_CSphase'
#if not os.path.isdir(dummyparmdb):
#runbbs([mslist[0]], skymodel,dummyparset, dummyparmdb, True, False)
# TO SPEED THINGS UP, hard coded for BOOTES - i.e. the above has already been run
for ms in mslist:
os.system('rm -rf ' + ms +'/' + dummyparmdb)
os.system('cp -r ' + dummyparmdb + ' ' + ms + '/instrument_template')
if smooth:
parmdb_a = 'instrument_amps1_smoothed' # last/best ampplitude(+phase) parmdb
else:
parmdb_a = 'instrument_amps1' # last/best ampplitude(+phase) parmdb
parmdb_p = 'instrument_phase1' # last/best CommonScalarPhase parmdb
parmdbout = 'instrument_merged'
#reset in case of instrument_template_TECclock
dummyparmdb = 'instrument_template'
for ms in mslist:
create_merged_parmdb(ms, ms+'/'+parmdb_a, ms+'/'+parmdb_p, ms+'/'+dummyparmdb,ms+'/'+parmdbout,cellsizetime_a,cellsizetime_p)
os.system('rm -r '+imlow+'*')
os.system('rm '+lr_bbslog+' '+hr_bbslog+' '+finalsky)
if os.path.isfile(imhigh+'-image.fits'):
logging.warning('High-resolution image exists, NOT remaking it')
else:
# ---------------------
# image without mask
cmd = wsclean + ' -reorder -name ' + imhigh + ' -size ' + str(imsizeh) + ' ' + str(imsizeh) + ' '
if tempdir is not None:
cmd+='-tempdir '+tempdir+' '
cmd+= '-scale ' + cellh + ' -weight briggs 0.0 -niter ' + str(niterh) + ' '
cmd+= '-maxuv-l 7e3 -mgain 0.65 -fitbeam -datacolumn CORRECTED_DATA -no-update-model-required ' + ms
wsclean_wait()
run(cmd)
# create the mask
mask_name = imhigh + '.fitsmask'
casa_mask = imhigh + '.casamask'
if os.path.isfile(mask_name):
logging.warning('Mask exists, NOT remaking it')
else:
cmd='python '+SCRIPTPATH+'/makecleanmask_10sb_wsclean.py --threshpix '+str(threshpix)+\
' --threshisl '+str(threshisl) +' --atrous_do '+ str(atrous_do)+' '
if casaregion!='':
cmd+='--casaregion '+ casaregion +' '
cmd+=imhigh + '-image.fits'
def runbbs(mslist, skymodel, parset, parmdb, applycal, TEC, clusterdesc, db,
dbuser, dbname):
#NOTE WORK FROM MODEL_DATA (contains correct phase data from 10SB calibration)
#NOTE WORK FROM DATA if TEC
if TEC:
key = str(uuid.uuid4())
key = key[0:12]
if clusterdesc == 'default':
# allow 'default' which gives the old behaviour, for now
username = pwd.getpwuid(os.getuid())[0]
cdparset = '/home/' + username + '/pgsql-setup.txt'
if os.path.isfile(cdparset):
logging.debug('Getting pgsql setup from {}'.format(cdparset))
lines = [line.strip() for line in open(cdparset)]
clusterdesc = lines[0]
db = lines[1]
dbuser = lines[2]
dbname = lines[3]
elif 'para' in os.uname()[1]:
clusterdesc = '/home/wwilliams/para/paranew.clusterdesc'
db = 'kolkje'
dbuser = '******'
dbname = 'shimwell'
else:
clusterdesc = '/home/williams/clusterdesc/cep3.clusterdesc'
db = 'ldb002.offline.lofar'
dbuser = '******'
dbname = username
if len(mslist) == 10000: # 34 does not work now!!!!!!
# this is a very special case for a full run, manually here to run with 3 solver controls
vdslist = ''
run('makevds ' + clusterdesc + ' ' + ms)
vdslist = vdslist + ms + '.vds '
else:
vdslist = ''
for ms in mslist:
run('makevds ' + clusterdesc + ' ' + ms)
vdslist = vdslist + ms + '.vds '
gds = 'tec.gds'
logging.debug(vdslist)
run('combinevds ' + gds + ' ' + vdslist)
cmd1 = 'calibrate -f --key ' + key + ' --cluster-desc ' + clusterdesc + ' --instrument-name ' + parmdb + ' '
cmd2 = '--db ' + db + ' --db-user ' + dbuser + ' ' + ' --db-name ' + dbname + ' ' + gds + ' ' + parset + ' ' + skymodel + ' ' + '. > ' + gds + '.bbslog'
bbscmd = cmd1 + cmd2
ntries = 0
done = 0
while (ntries < 10) and (done < 1):
logging.debug('calibrate try {}'.format(ntries))
# let run 'proceed on failure' as the test for failure is done here to allow retries
run(bbscmd, proceed=True)
#done = 0
#while(done < 1):
cmd = "grep '\[OK\] done!' " + gds + ".bbslog"
output = Popen(cmd, shell=True, stdout=PIPE).communicate()[0]
if 'OK' in output:
done += 1
logging.debug('{} is done'.format(gds))
else:
#cmd = "grep 'FAIL' " + gds + ".bbslog"
#output=Popen(cmd, shell=True, stdout=PIPE).communicate()[0]
#if 'FAIL' in output:
logging.warning('calibrate failed, trying again...')
ntries += 1
time.sleep(5)
# after ntries exceeded is it still at FAIL
cmd = "grep 'FAIL' " + gds + ".bbslog"
output = Popen(cmd, shell=True, stdout=PIPE).communicate()[0]
if 'FAIL' in output:
raise Exception('calibrate has failed too many times')
else:
# Normal calibrate for amplitude
b = bg()
for ms in mslist:
log = ms + '.bbslog'
if applycal:
cmd = 'calibrate-stand-alone --parmdb-name ' + parmdb + ' ' + ms + ' ' + parset + ' ' + skymodel + '>' + log + ' 2>&1'
#cmd = 'calibrate-stand-alone -t 4 --parmdb-name ' + parmdb + ' ' + ms + ' ' + parset + ' ' + skymodel + '>' + log
else:
cmd = 'calibrate-stand-alone -f --parmdb-name ' + parmdb + ' ' + ms + ' ' + parset + ' ' + skymodel + '>' + log + ' 2>&1'
#cmd = 'calibrate-stand-alone -t 4 -f --parmdb-name ' + parmdb + ' ' + ms + ' ' + parset + ' ' + skymodel + '>' + log
b.run(cmd)
time.sleep(10)
b.wait()
return
def do_selfcal(mslist,
cluster,
atrous_do,
imsize,
nterms,
cellsizetime_a,
cellsizetime_p,
TECi,
clocki,
HRi,
region,
clusterdesc,
dbserver,
dbuser,
dbname,
SCRIPTPATH,
ncores=8,
config=None,
model=None):
TEC = False
FFT = False
clock = False
HR = False # high dynamic range
if TECi == "True":
TEC = True
if clocki == "True":
clock = True
if HRi == "HD":
HR = True
if imsize <= 2048:
wplanes = 1
FFT = True # FFT image into MODEL_DATA
if imsize >= 512:
wplanes = 64
if imsize > 799:
wplanes = 96
if imsize > 1023:
wplanes = 128
if imsize > 1599:
wplanes = 180
if imsize > 1800:
wplanes = 196
if imsize > 2049:
wplanes = 256
#if imsize > 3000:
# wplanes = 448
#if imsize > 4095:
# wplanes = 512
logging.info('mslist {}'.format(mslist))
logging.info('source {}'.format(cluster))
logging.info('atrous_do {}'.format(atrous_do))
logging.info('imsize {} '.format(imsize))
logging.info('TEC is {} and clock is {}'.format(TEC, clock))
msinputlist = ''
for m in mslist:
msinputlist = msinputlist + ' ' + m
#if len(mslist) == 29:
#group = "9,10,10"
#elif len(mslist) == 28:
#group = "7,7,7,7"
#elif len(mslist) == 20:
##group = "10,10"
#group = "7,7,6"
#elif len(mslist) == 16:
#group = "8,8"
#else:
#group = str(len(mslist))
group = get_group(mslist)
logging.info('GROUP {}'.format(group))
uvrange = '80'
#uvrange = '400'
merge_parmdb = True
phasors = False # if true only solve for amps on long timescales
smooth = False # seems that smooth does not help the selfcal (various reasons for that)
# 1. boundaries of flagged vs non-flagged data are sharp (should not be smoothed)
# 2. there sre some strong ampl various at low elevations
smooth = True # sometimes almost 0.0 amplitude, causes ripples
phasezero = True # reset phases from ap calibration
if model is None:
#### MAKE IMAGE 0 ###
logging.info('Make image 0')
imout, mask = make_image(mslist, cluster, '0', 10, 6, nterms,
atrous_do, imsize, region, ncores, SCRIPTPATH)
#####################
### CALIBRATE WITH BBS PHASE ONLY 1 ###
# create skymodel for BBS
run(SCRIPTPATH + '/casapy2bbs.py -m ' + mask + ' ' + '-t ' +
str(nterms) + ' ' + imout + '.model ' + imout + '.skymodel')
if FFT:
os.system('casapy --nogui -c '+SCRIPTPATH+'/ft_v2.py ' + msinputlist + ' ' + imout+'.model' \
+ ' ' + str(nterms) + ' '+ str(wplanes))
# phase only calibrate
skymodel = imout + '.skymodel'
parset = create_scalarphase_parset(cellsizetime_p, TEC, clock, group,
FFT, uvrange)
runbbs(mslist, skymodel, parset, 'instrument', False, TEC, clusterdesc,
dbserver, dbuser, dbname)
#NOTE WORK FROM MODEL_DATA (contains correct phase data from 10SB calibration)
######################################
### MAKE IMAGE 1 ###
logging.info('Make image 1')
imout, mask = make_image(mslist, cluster, '1', 15, 15, nterms,
atrous_do, imsize, region, ncores, SCRIPTPATH)
####################
### CALIBRATE WITH BBS PHASE ONLY 2 ###
# create skymodel for BBS
run(SCRIPTPATH + '/casapy2bbs.py -m ' + mask + ' ' + '-t ' +
str(nterms) + ' ' + imout + '.model ' + imout + '.skymodel')
if FFT:
run('casapy --nogui -c '+SCRIPTPATH+'/ft_v2.py ' + msinputlist + ' ' + imout+'.model' \
+ ' ' + str(nterms) + ' '+ str(wplanes))
# phase only calibrate
skymodel = imout + '.skymodel'
parset = create_scalarphase_parset(cellsizetime_p, TEC, clock, group,
FFT, uvrange)
runbbs(
mslist, skymodel, parset, 'instrument', False, TEC, clusterdesc,
dbserver, dbuser, dbname
) #NOTE WORK FROM MODEL_DATA (contains correct phase data from 10SB calibration)
######################################
### MAKE IMAGE 2 ###
logging.info('Make image 2')
imout, mask = make_image(mslist, cluster, '2', 15, 15, nterms,
atrous_do, imsize, region, ncores, SCRIPTPATH)
####################
### CALIBRATE WITH BBS PHASE+AMP 1 ###
run(SCRIPTPATH + '/casapy2bbs.py -m ' + mask + ' ' + '-t ' +
str(nterms) + ' ' + imout + '.model ' + imout + '.skymodel')
if FFT:
run('casapy --nogui -c '+SCRIPTPATH+'/ft_v2.py ' + msinputlist + ' ' + imout+'.model' \
+ ' ' + str(nterms) + ' '+ str(wplanes))
skymodel = imout + '.skymodel'
parset = create_scalarphase_parset_p(cellsizetime_p, TEC, clock, group,
FFT, uvrange)
# solve +apply phases
runbbs(mslist, skymodel, parset, 'instrument_phase0', False, TEC,
clusterdesc, dbserver, dbuser, dbname)
# solve amps
parmdb = 'instrument_amps0'
parset = create_amponly_parset(cellsizetime_a, FFT, uvrange)
runbbs(mslist, skymodel, parset, parmdb, False, False, clusterdesc,
dbserver, dbuser, dbname)
for ms in mslist:
# remove outliers from the solutions
if phasors:
run('python ' + SCRIPTPATH + '/smoothcal_rx42.py ' + ms + ' ' +
ms + '/' + parmdb + ' ' + ms + '/' + parmdb + '_smoothed' +
' > ' + ms + '_' + parmdb + '_smoothed.log')
else:
run('python ' + SCRIPTPATH + '/smoothcal_a2256_nophasors.py ' +
ms + ' ' + ms + '/' + parmdb + ' ' + ms + '/' + parmdb +
'_smoothed' + ' > ' + ms + '_' + parmdb + '_smoothed.log')
# apply amps
if smooth:
runbbs(mslist, skymodel,
SCRIPTPATH + '/apply_amplitudeonly.parset',
parmdb + '_smoothed', True, False, clusterdesc, dbserver,
dbuser, dbname)
else:
runbbs(mslist, skymodel,
SCRIPTPATH + '/apply_amplitudeonly.parset', parmdb, True,
False, clusterdesc, dbserver, dbuser, dbname)
### MAKE IMAGE 3 ###
logging.info('Make image 3')
imout, mask = make_image(mslist, cluster, '3', 10, 10, nterms,
atrous_do, imsize, region, ncores, SCRIPTPATH)
#### CALIBRATE BBS PHASE+AMP 2 ###
# make model
run(SCRIPTPATH + '/casapy2bbs.py -m ' + mask + ' ' + '-t ' +
str(nterms) + ' ' + imout + '.model ' + imout + '.skymodel')
if FFT:
run('casapy --nogui -c '+SCRIPTPATH+'/ft_v2.py ' + msinputlist + ' ' + imout+'.model' \
+ ' ' + str(nterms) + ' '+ str(wplanes))
#parmdb keep from previous step
skymodel = imout + '.skymodel'
# reset the phases from instrument_amps0 to zero to prevent large phase corrections from incorrect AP solve
if phasezero:
inputparmdb = parmdb + '_smoothed'
outputparmdb = parmdb + '_smoothed_phasezero'
for ms in mslist:
run('python ' + SCRIPTPATH + '/setphasezero.py ' + ms + ' ' +
ms + '/' + inputparmdb + ' ' + ms + '/' + outputparmdb)
else:
outputparmdb = parmdb + '_smoothed'
# phase only cal
skymodel = imout + '.skymodel'
parset = create_scalarphase_parset_p(cellsizetime_p, TEC, clock, group,
FFT, uvrange)
runbbs(mslist, skymodel, parset, 'instrument_phase1', False, TEC,
clusterdesc, dbserver, dbuser, dbname)
# solve amps
parmdb = 'instrument_amps1'
parset = create_amponly_parset(cellsizetime_a, FFT, uvrange)
runbbs(mslist, skymodel, parset, parmdb, False, False, clusterdesc,
dbserver, dbuser, dbname)
for ms in mslist:
# remove outliers from the solutions
if phasors:
run('python ' + SCRIPTPATH + '/smoothcal_rx42.py ' + ms + ' ' +
ms + '/' + parmdb + ' ' + ms + '/' + parmdb + '_smoothed' +
' > ' + ms + '_' + parmdb + '_smoothed.log')
else:
run('python ' + SCRIPTPATH + '/smoothcal_a2256_nophasors.py ' +
ms + ' ' + ms + '/' + parmdb + ' ' + ms + '/' + parmdb +
'_smoothed' + ' > ' + ms + '_' + parmdb + '_smoothed.log')
# apply amps
if smooth:
runbbs(mslist, skymodel,
SCRIPTPATH + '/apply_amplitudeonly.parset',
parmdb + '_smoothed', True, False, clusterdesc, dbserver,
dbuser, dbname)
else:
runbbs(mslist, skymodel,
SCRIPTPATH + '/apply_amplitudeonly.parset', parmdb, True,
False, clusterdesc, dbserver, dbuser, dbname)
### MAKE IMAGE 4 ###
logging.info('Make image 4')
imout, mask = make_image(mslist, cluster, '4', 10, 10, nterms,
atrous_do, imsize, region, ncores, SCRIPTPATH)
else:
### As for image 4 ###
### the model variable contains a full path to a CASA sky model for the field
logging.info('Calibrating using pre-existing model ' + model)
imout = 'model_image_' + cluster
mask = imout + '.mask'
origmask = model.replace('model', 'mask')
# delete any old versions, and copy over the model
for suffix in ['model', 'mask']:
os.system('rm -rf ' + imout + '.' + suffix)
run('cp -r ' + model + ' ' + imout + '.model')
run('cp -r ' + origmask + ' ' + imout + '.mask')
# now proceed as though this was the previous image!
run(SCRIPTPATH + '/casapy2bbs.py -m ' + mask + ' ' + '-t ' +
str(nterms) + ' ' + imout + '.model' + ' ' + imout + '.skymodel')
if FFT:
run('casapy --nogui -c '+SCRIPTPATH+'/ft_v2.py ' + msinputlist + ' ' + imout+'.model' \
+ ' ' + str(nterms) + ' '+ str(wplanes))
skymodel = imout + '.skymodel'
parset = create_scalarphase_parset_p(cellsizetime_p, TEC, clock, group,
FFT, uvrange)
# solve +apply phases
runbbs(mslist, skymodel, parset, 'instrument_phase1', False, TEC,
clusterdesc, dbserver, dbuser, dbname)
# solve amps
parmdb = 'instrument_amps1'
parset = create_amponly_parset(cellsizetime_a, FFT, uvrange)
runbbs(mslist, skymodel, parset, parmdb, False, False, clusterdesc,
dbserver, dbuser, dbname)
for ms in mslist:
# remove outliers from the solutions
if phasors:
run('python ' + SCRIPTPATH + '/smoothcal_rx42.py ' + ms + ' ' +
ms + '/' + parmdb + ' ' + ms + '/' + parmdb + '_smoothed' +
' > ' + ms + '_' + parmdb + '_smoothed.log')
else:
run('python ' + SCRIPTPATH + '/smoothcal_a2256_nophasors.py ' +
ms + ' ' + ms + '/' + parmdb + ' ' + ms + '/' + parmdb +
'_smoothed' + ' > ' + ms + '_' + parmdb + '_smoothed.log')
# apply amps
if smooth:
runbbs(mslist, skymodel,
SCRIPTPATH + '/apply_amplitudeonly.parset',
parmdb + '_smoothed', True, False, clusterdesc, dbserver,
dbuser, dbname)
else:
runbbs(mslist, skymodel,
SCRIPTPATH + '/apply_amplitudeonly.parset', parmdb, True,
False, clusterdesc, dbserver, dbuser, dbname)
### MAKE IMAGE 4 ###
logging.info('Make image 4')
imout, mask = make_image(mslist, cluster, '4', 10, 10, nterms,
atrous_do, imsize, region, ncores, SCRIPTPATH)
### CREATE FINAL MODEL ###
logging.info('Create final model')
skymodelf = 'im_cluster' + cluster + '.final.skymodel'
run(SCRIPTPATH + '/casapy2bbs.py -m ' + mask + ' ' + '-t ' + str(nterms) +
' ' + imout + '.model ' + skymodelf)
if FFT:
run('casapy --nogui -c '+SCRIPTPATH+'/ft_v2.py ' + msinputlist + ' ' + imout+'.model' \
+ ' ' + str(nterms) + ' '+ str(wplanes))
### CREATED MERGED PARMDB SCALARPHASE+AMPS ###
### INCLUDES SPLINE INTERPOLARION OF AMPS ###
if merge_parmdb:
logging.info('Merge parmdb')
if phasors:
dummyparset = SCRIPTPATH + '/scalarphase+amp.parset'
else:
if TEC:
dummyparset = SCRIPTPATH + '/scalarphase+ap+TEC.parset'
else:
dummyparset = SCRIPTPATH + '/scalarphase+ap.parset'
if TEC:
if clock:
dummyparmdb = 'instrument_template_TECclock'
else:
dummyparmdb = 'instrument_template_Gain_TEC_CSphase'
#if not os.path.isdir(dummyparmdb):
#runbbs([mslist[0]], skymodel,dummyparset, dummyparmdb, True, False)
# TO SPEED THINGS UP, hard coded for BOOTES - i.e. the above has already been run
for ms in mslist:
os.system('rm -rf ' + ms + '/' + dummyparmdb)
os.system('cp -r ' + dummyparmdb + ' ' + ms +
'/instrument_template')
if smooth:
parmdb_a = 'instrument_amps1_smoothed' # last/best ampplitude(+phase) parmdb
else:
parmdb_a = 'instrument_amps1' # last/best ampplitude(+phase) parmdb
parmdb_p = 'instrument_phase1' # last/best CommonScalarPhase parmdb
parmdbout = 'instrument_merged'
#reset in case of instrument_template_TECclock
dummyparmdb = 'instrument_template'
for ms in mslist:
create_merged_parmdb(ms, ms + '/' + parmdb_a, ms + '/' + parmdb_p,
ms + '/' + dummyparmdb, ms + '/' + parmdbout,
cellsizetime_a, cellsizetime_p)
if os.path.isfile(imhigh + '-image.fits'):
logging.warning('High-resolution image exists, NOT remaking it')
else:
# ---------------------
# image without mask
cmd = wsclean + ' -reorder -name ' + imhigh + ' -size ' + str(
imsizeh) + ' ' + str(imsizeh) + ' '
if tempdir is not None:
cmd += '-tempdir ' + tempdir + ' '
cmd += '-scale ' + cellh + ' -weight briggs 0.0 -niter ' + str(
niterh) + ' '
cmd += '-maxuv-l 7e3 -mgain 0.65 -fitbeam -datacolumn CORRECTED_DATA -no-update-model-required ' + ms
wsclean_wait()
run(cmd)
# create the mask
mask_name = imhigh + '.fitsmask'
casa_mask = imhigh + '.casamask'
if os.path.isfile(mask_name):
logging.warning('Mask exists, NOT remaking it')
else:
cmd='python '+SCRIPTPATH+'/makecleanmask_10sb_wsclean.py --threshpix '+str(threshpix)+\
' --threshisl '+str(threshisl) +' --atrous_do '+ str(atrous_do)+' '
if casaregion != '':
cmd += '--casaregion ' + casaregion + ' '
cmd += imhigh + '-image.fits'
def make_image(mslist, cluster, callnumber, threshpix, threshisl, nterms,
atrous_do, imsize, region, ncores, SCRIPTPATH):
do_mask = True
niter = 1000 # 7500 causes nasty clean artifacts
mscale = 'False'
if atrous_do:
mscale = 'True'
average = True # average the data a lot to speed up the imaging,
# ONLY average for small FOV, otherwise timesmearing is a problem
#average data
if average:
b = bg(maxp=2)
for ms in (mslist):
ndppp_parset = ms + '_NDPPP.parset'
ndppplog = ndppp_parset.replace('.parset', '.log')
os.system('rm -f ' + ndppp_parset)
output = ms + '.tmpavg'
os.system('rm -rf ' + output)
f = open(ndppp_parset, 'w')
f.write('msin = %s\n' % ms)
f.write('msin.datacolumn = CORRECTED_DATA\n')
f.write('msout = %s\n' % output)
f.write('msout.writefullresflag=False\n')
f.write('steps=[avg]\n')
f.write('rficonsole.type=aoflagger\n')
f.write('avg.type = squash\n')
f.write('avg.freqstep = 1\n')
f.write('avg.timestep = 12\n') # is the default
f.close()
ndpppcmd = 'NDPPP ' + ndppp_parset + ' >' + ndppplog + ' 2>&1'
b.run(ndpppcmd)
#os.system (ndpppcmd)
b.wait()
#output=numpy.int(Popen(cmd, shell=True, stdout=PIPE).communicate()[0])
#while output > 0 :
#time.sleep(10)
#output=numpy.int(Popen(cmd, shell=True, stdout=PIPE).communicate()[0])
#pid = (Popen('pgrep NDPPP', shell=True, stdout=PIPE).communicate()[0])
##print pid
#pid_list = pid.split()
#for pidnr in pid_list:
##print 'PID', numpy.int(pidnr)
#os.system('kill -s CONT ' + str(pidnr))
ms = ''
for m in mslist:
if average:
ms = ms + ' ' + m + '.tmpavg'
else:
ms = ms + ' ' + m
imout = 'im' + callnumber + '_cluster' + cluster + 'nm'
logging.debug(ms + ' ' + imout + ' ' + 'None' + ' ' + '1mJy' + ' ' +
str(niter) + ' ' + str(nterms))
if do_mask:
if cluster == 'a2256': ## special case for a2256
niter = niter * 15 # clean very deep here
run('casapy --nogui --logfile casapy-'+imout+'.log -c '+SCRIPTPATH+'/casapy_cleanv4.py ' + ms + ' ' + imout + ' ' + 'None' +\
' ' + '1mJy' + ' ' + str(niter) + ' ' + str(nterms) + ' ' + str(imsize) + ' ' + mscale)
# make mask
if nterms > 1:
do_makecleanmask(imout + '.image.tt0',
threshpix,
threshisl,
atrous_do,
ncores=ncores)
# run('python '+SCRIPTPATH+'/makecleanmask.py --threshpix '+str(threshpix)+\
# ' --threshisl '+str(threshisl) +' --atrous_do '+ str(atrous_do) +' ' +imout +'.image.tt0')
else:
do_makecleanmask(imout + '.image',
threshpix,
threshisl,
atrous_do,
ncores=ncores)
# run('python '+SCRIPTPATH+'/makecleanmask.py --threshpix '+str(threshpix)+\
# ' --threshisl '+str(threshisl) +' --atrous_do '+ str(atrous_do) + ' ' + imout +'.image')
# clean image with manual mask
mask = imout + '.cleanmask'
niter = 1000
imout = 'im' + callnumber + '_cluster' + cluster
if region != 'empty': ## special cases
niter = niter * 3
run('casapy --nogui --logfile casapy-'+imout+'.log -c '+SCRIPTPATH+'/casapy_cleanv4.py '+ ms + ' ' + imout + ' ' + mask+','+region + \
' ' + '1mJy' + ' ' + str(niter) + ' ' + str(nterms) + ' ' + str(imsize) + ' ' + mscale)
else:
run('casapy --nogui --logfile casapy-'+imout+'.log -c '+SCRIPTPATH+'/casapy_cleanv4.py '+ ms + ' ' + imout + ' ' + mask + \
' ' + '1mJy' + ' ' + str(niter) + ' ' + str(nterms) + ' ' + str(imsize) + ' ' + mscale)
# convert to FITS
if nterms > 1:
run('image2fits in=' + imout + '.image.tt0' + ' ' + 'out=' + imout +
'.fits')
else:
run('image2fits in=' + imout + '.image' + ' ' + 'out=' + imout +
'.fits')
if not os.path.exists(imout + '.fits'):
raise Exception('Imaging Error: ' + imout + '.fits does not exist')
if average:
logging.debug('rm -rf {}'.format(ms))
os.system('rm -rf ' + ms)
return imout, imout + '.mask'
#continue
## NOTE: addback now needs to be done as subtract in reverse... with FT and allbands concat
## STEP 1: check ##
if StartAtStep in ['preSC']:
# if we didn't keep the allbands.concat.shifted
if not os.path.exists('allbands.concat.shifted_'+source+'.ms'):
parset = create_phaseshift_parset_full('allbands.concat.ms', 'allbands.concat.shifted_'+source+'.ms', facetdir,'DATA')
ndppplog = parset.replace('.parset','.log')
#ndpppcmd = 'NDPPP ' + parset + ' > '+ ndppplog + ' 2>&1'
#ndppprc = os.system(ndpppcmd)
ndpppcmd = 'NDPPP ' + parset + ' > '+ ndppplog + ' 2>&1'
run(ndpppcmd)
### STEP 3: prep for facet ##
if StartAtStep in ['preSC','preFACET']:
# image model to add back
imsizef = image_size_from_mask(output_template_im)
imout = 'im'+ 'field0' +'_cluster'+source
if not os.path.exists(imout+'-model.fits'):
raise Exception(imout+'-model.fits is missing')
logging.info('running ft: '+imout)
def make_image_wsclean_nomask(mslist, cluster, callnumber, threshpix, threshisl, nterms, atrous_do, imsize, inputmask, mscale, region,cellsize,uvrange,wsclean,WSCleanRobust):
if imsize is None:
imsize = image_size_from_mask(inputmask)
niter = numpy.int(20000 * (numpy.sqrt(numpy.float(len(mslist)))))
cellsizeim = str(cellsize) +'arcsec'
depth = 1e-3*0.7 / (numpy.sqrt(numpy.float(len(mslist))))
print 'Cleaning to a noise level of',depth,'Jy: niter is',niter
cleandepth2 = str(depth) #+ 'mJy'
wideband = False
if len(mslist) > 5:
wideband = True
ms = ''
for m in mslist:
ms = ms + ' ' + m
imout = 'im'+ callnumber +'_cluster'+cluster
os.system('rm -rf ' + imout + '-*')
outms = 'field-'+cluster+'.ms'
parsetname = 'concatforwsclean-'+cluster+'.parset'
msinstr = ""
for ms_id, ms in enumerate(mslist):
msinstr = msinstr + "'" + ms + "'"
if ms_id < len(mslist)-1:
msinstr = msinstr + ", "
os.system('rm -rf ' + parsetname)
f=open(parsetname, 'w')
f.write('msin = [%s]\n' % msinstr)
f.write('msin.datacolumn = DATA\n')
f.write('msin.missingdata=True\n')
f.write('msin.orderms=False\n')
f.write('msout=%s\n' % outms)
f.write('steps=[]\n')
f.close()
os.system('rm -rf ' + outms)
run('NDPPP ' + parsetname)
if wideband:
channelsout = 1 # there is a factor of 5 averaging
cmd1 = wsclean + ' -reorder -name ' + imout + ' -size ' + str(imsize) + ' ' + str(imsize) + ' '
cmd2 = '-scale ' + cellsizeim + ' -weight briggs '+str(WSCleanRobust)+' -niter ' + str(niter) + '-cleanborder 0 -threshold '+ cleandepth2 + ' '
cmd3 = '-minuv-l '+ str(uvrange) + ' -casamask ' + inputmask + ' '\
+' -mgain 0.6 -fitbeam -datacolumn DATA -no-update-model-required -joinchannels -channelsout ' +\
str(channelsout) + ' ' + outms
else:
cmd1 = wsclean + ' -reorder -name ' + imout + ' -size ' + str(imsize) + ' ' + str(imsize) + ' '
cmd2 = '-scale ' + cellsizeim + ' -weight briggs '+str(WSCleanRobust)+' -niter ' + str(niter) + ' -cleanborder 0 -threshold '+ cleandepth2 + ' '
cmd3 = '-minuv-l '+ str(uvrange) +' -casamask ' + inputmask + ' -mgain 0.6 -fitbeam -datacolumn DATA -no-update-model-required ' + outms
print cmd1+cmd2+cmd3
run(cmd1+cmd2+cmd3)
finalim=pyrap.images.image(imout+'-image.fits')
finalim.saveas(imout +'.image')
return imout, None, imsize
def runbbs(mslist, skymodel, parset, parmdb, applycal, TEC, clusterdesc, db, dbuser, dbname):
#NOTE WORK FROM MODEL_DATA (contains correct phase data from 10SB calibration)
#NOTE WORK FROM DATA if TEC
if TEC:
key = str(uuid.uuid4())
key = key[0:12]
if clusterdesc=='default':
# allow 'default' which gives the old behaviour, for now
username = pwd.getpwuid(os.getuid())[0]
cdparset='/home/'+username+'/pgsql-setup.txt'
if os.path.isfile(cdparset):
logging.debug('Getting pgsql setup from {}'.format(cdparset))
lines=[line.strip() for line in open(cdparset)]
clusterdesc=lines[0]
db=lines[1]
dbuser=lines[2]
dbname=lines[3]
elif 'para' in os.uname()[1]:
clusterdesc = '/home/wwilliams/para/paranew.clusterdesc'
db = 'kolkje'
dbuser = '******'
dbname = 'shimwell'
else:
clusterdesc = '/home/williams/clusterdesc/cep3.clusterdesc'
db = 'ldb002.offline.lofar'
dbuser = '******'
dbname = username
if len(mslist) == 10000: # 34 does not work now!!!!!!
# this is a very special case for a full run, manually here to run with 3 solver controls
vdslist = ''
run('makevds '+clusterdesc+' ' + ms)
vdslist = vdslist + ms + '.vds '
else:
vdslist = ''
for ms in mslist:
run('makevds '+clusterdesc+' ' + ms)
vdslist = vdslist + ms + '.vds '
gds = 'tec.gds'
logging.debug(vdslist)
run('combinevds ' + gds + ' '+ vdslist)
cmd1= 'calibrate -f --key ' + key + ' --cluster-desc '+clusterdesc +' --instrument-name ' + parmdb + ' '
cmd2= '--db '+db + ' --db-user '+ dbuser + ' '+ ' --db-name '+ dbname + ' ' + gds + ' ' + parset + ' ' + skymodel + ' ' + '. > ' + gds + '.bbslog'
bbscmd = cmd1 + cmd2
ntries = 0
done = 0
while (ntries < 10) and (done < 1):
logging.debug('calibrate try {}'.format(ntries))
# let run 'proceed on failure' as the test for failure is done here to allow retries
run(bbscmd, proceed=True)
#done = 0
#while(done < 1):
cmd = "grep '\[OK\] done!' " + gds + ".bbslog"
output=Popen(cmd, shell=True, stdout=PIPE).communicate()[0]
if 'OK' in output:
done += 1
logging.debug('{} is done'.format(gds))
else:
#cmd = "grep 'FAIL' " + gds + ".bbslog"
#output=Popen(cmd, shell=True, stdout=PIPE).communicate()[0]
#if 'FAIL' in output:
logging.warning('calibrate failed, trying again...')
ntries += 1
time.sleep(5)
# after ntries exceeded is it still at FAIL
cmd = "grep 'FAIL' " + gds + ".bbslog"
output=Popen(cmd, shell=True, stdout=PIPE).communicate()[0]
if 'FAIL' in output:
raise Exception('calibrate has failed too many times' )
else:
# Normal calibrate for amplitude
b=bg()
for ms in mslist:
log = ms + '.bbslog'
if applycal:
cmd = 'calibrate-stand-alone --parmdb-name ' + parmdb + ' ' + ms + ' ' + parset + ' ' + skymodel + '>' + log + ' 2>&1'
#cmd = 'calibrate-stand-alone -t 4 --parmdb-name ' + parmdb + ' ' + ms + ' ' + parset + ' ' + skymodel + '>' + log
else:
cmd = 'calibrate-stand-alone -f --parmdb-name ' + parmdb + ' ' + ms + ' ' + parset + ' ' + skymodel + '>' + log + ' 2>&1'
#cmd = 'calibrate-stand-alone -t 4 -f --parmdb-name ' + parmdb + ' ' + ms + ' ' + parset + ' ' + skymodel + '>' + log
b.run(cmd)
time.sleep(10)
b.wait()
return
def do_selfcal(mslist, cluster, atrous_do, imsize, nterms,
cellsizetime_a, cellsizetime_p, TECi, clocki, HRi,
region, clusterdesc, dbserver, dbuser, dbname,
SCRIPTPATH, ncores=8, config=None, model=None):
TEC = False
FFT = False
clock= False
HR = False # high dynamic range
if TECi == "True":
TEC = True
if clocki == "True":
clock = True
if HRi == "HD":
HR = True
if imsize <=2048:
wplanes = 1
FFT = True # FFT image into MODEL_DATA
if imsize >= 512:
wplanes = 64
if imsize > 799:
wplanes = 96
if imsize > 1023:
wplanes = 128
if imsize > 1599:
wplanes = 180
if imsize > 1800:
wplanes = 196
if imsize > 2049:
wplanes = 256
#if imsize > 3000:
# wplanes = 448
#if imsize > 4095:
# wplanes = 512
logging.info('mslist {}'.format(mslist))
logging.info('source {}'.format(cluster))
logging.info('atrous_do {}'.format(atrous_do))
logging.info('imsize {} '.format(imsize))
logging.info('TEC is {} and clock is {}'.format(TEC, clock))
msinputlist = ''
for m in mslist:
msinputlist = msinputlist + ' ' + m
#if len(mslist) == 29:
#group = "9,10,10"
#elif len(mslist) == 28:
#group = "7,7,7,7"
#elif len(mslist) == 20:
##group = "10,10"
#group = "7,7,6"
#elif len(mslist) == 16:
#group = "8,8"
#else:
#group = str(len(mslist))
group = get_group(mslist)
logging.info('GROUP {}'.format(group))
uvrange = '80'
#uvrange = '400'
merge_parmdb = True
phasors = False # if true only solve for amps on long timescales
smooth = False # seems that smooth does not help the selfcal (various reasons for that)
# 1. boundaries of flagged vs non-flagged data are sharp (should not be smoothed)
# 2. there sre some strong ampl various at low elevations
smooth = True # sometimes almost 0.0 amplitude, causes ripples
phasezero = True # reset phases from ap calibration
#### MAKE IMAGE 0 ###
logging.info('Make image 0')
imout,mask = make_image(mslist, cluster, '0', 10, 6, nterms, atrous_do, imsize, region, ncores, SCRIPTPATH)
#####################
### CALIBRATE WITH BBS PHASE ONLY 1 ###
# create skymodel for BBS
run(SCRIPTPATH+'/casapy2bbs.py -m '+ mask + ' ' +'-t ' + str(nterms)+ ' ' + imout+'.model ' + imout+'.skymodel')
if FFT:
os.system('casapy --nogui -c '+SCRIPTPATH+'/ft_v2.py ' + msinputlist + ' ' + imout+'.model' \
+ ' ' + str(nterms) + ' '+ str(wplanes))
# phase only calibrate
skymodel = imout+'.skymodel'
parset = create_scalarphase_parset(cellsizetime_p, TEC, clock, group, FFT, uvrange)
runbbs(mslist, skymodel, parset, 'instrument', False, TEC, clusterdesc, dbserver, dbuser, dbname)
#NOTE WORK FROM MODEL_DATA (contains correct phase data from 10SB calibration)
######################################
### MAKE IMAGE 1 ###
logging.info('Make image 1')
imout,mask = make_image(mslist, cluster, '1', 15, 15, nterms, atrous_do, imsize, region, ncores, SCRIPTPATH)
####################
### CALIBRATE WITH BBS PHASE ONLY 2 ###
# create skymodel for BBS
run(SCRIPTPATH+'/casapy2bbs.py -m '+ mask + ' ' +'-t ' + str(nterms)+ ' ' + imout+'.model ' + imout+'.skymodel')
if FFT:
run('casapy --nogui -c '+SCRIPTPATH+'/ft_v2.py ' + msinputlist + ' ' + imout+'.model' \
+ ' ' + str(nterms) + ' '+ str(wplanes))
# phase only calibrate
skymodel = imout+'.skymodel'
parset = create_scalarphase_parset(cellsizetime_p, TEC, clock, group, FFT, uvrange)
runbbs(mslist, skymodel, parset, 'instrument', False, TEC,clusterdesc, dbserver, dbuser, dbname) #NOTE WORK FROM MODEL_DATA (contains correct phase data from 10SB calibration)
######################################
### MAKE IMAGE 2 ###
logging.info('Make image 2')
imout,mask = make_image(mslist, cluster, '2', 15, 15, nterms, atrous_do, imsize, region, ncores, SCRIPTPATH)
####################
### CALIBRATE WITH BBS PHASE+AMP 1 ###
run(SCRIPTPATH+'/casapy2bbs.py -m '+ mask + ' ' +'-t ' + str(nterms)+ ' ' + imout+'.model ' + imout+'.skymodel')
if FFT:
run('casapy --nogui -c '+SCRIPTPATH+'/ft_v2.py ' + msinputlist + ' ' + imout+'.model' \
+ ' ' + str(nterms) + ' '+ str(wplanes))
skymodel = imout+'.skymodel'
parset = create_scalarphase_parset_p(cellsizetime_p, TEC, clock, group, FFT, uvrange)
# solve +apply phases
runbbs(mslist, skymodel, parset, 'instrument_phase0', False, TEC, clusterdesc, dbserver, dbuser, dbname)
# solve amps
parmdb = 'instrument_amps0'
parset = create_amponly_parset(cellsizetime_a, FFT, uvrange)
runbbs(mslist, skymodel, parset, parmdb, False, False, clusterdesc, dbserver, dbuser, dbname)
for ms in mslist:
# remove outliers from the solutions
if phasors:
run('python '+SCRIPTPATH+'/smoothcal_rx42.py ' + ms + ' ' + ms+'/'+parmdb + ' ' + ms+'/'+parmdb+'_smoothed'+' > '+ms+'_'+parmdb+'_smoothed.log')
else:
run('python '+SCRIPTPATH+'/smoothcal_a2256_nophasors.py ' + ms + ' ' + ms+'/'+parmdb + ' ' + ms+'/'+parmdb+'_smoothed'+' > '+ms+'_'+parmdb+'_smoothed.log')
# apply amps
if smooth:
runbbs(mslist, skymodel,SCRIPTPATH+'/apply_amplitudeonly.parset', parmdb+'_smoothed', True, False, clusterdesc, dbserver, dbuser, dbname)
else:
runbbs(mslist, skymodel,SCRIPTPATH+'/apply_amplitudeonly.parset', parmdb, True, False, clusterdesc, dbserver, dbuser, dbname)
### MAKE IMAGE 3 ###
logging.info('Make image 3')
imout,mask = make_image(mslist, cluster, '3', 10, 10, nterms, atrous_do, imsize, region, ncores, SCRIPTPATH)
#### CALIBRATE BBS PHASE+AMP 2 ###
# make model
run(SCRIPTPATH+'/casapy2bbs.py -m '+ mask + ' ' +'-t ' + str(nterms)+ ' ' + imout+'.model ' + imout+'.skymodel')
if FFT:
run('casapy --nogui -c '+SCRIPTPATH+'/ft_v2.py ' + msinputlist + ' ' + imout+'.model' \
+ ' ' + str(nterms) + ' '+ str(wplanes))
#parmdb keep from previous step
skymodel = imout+'.skymodel'
# reset the phases from instrument_amps0 to zero to prevent large phase corrections from incorrect AP solve
if phasezero:
inputparmdb = parmdb +'_smoothed'
outputparmdb = parmdb +'_smoothed_phasezero'
for ms in mslist:
run('python '+SCRIPTPATH+'/setphasezero.py ' + ms + ' ' + ms+'/'+inputparmdb +' ' + ms+'/'+outputparmdb)
else:
outputparmdb = parmdb +'_smoothed'
# phase only cal
skymodel = imout+'.skymodel'
parset = create_scalarphase_parset_p(cellsizetime_p, TEC, clock, group, FFT, uvrange)
runbbs(mslist, skymodel, parset, 'instrument_phase1', False, TEC, clusterdesc, dbserver, dbuser, dbname)
# solve amps
parmdb = 'instrument_amps1'
parset = create_amponly_parset(cellsizetime_a, FFT, uvrange)
runbbs(mslist, skymodel, parset,parmdb, False, False, clusterdesc, dbserver, dbuser, dbname)
for ms in mslist:
# remove outliers from the solutions
if phasors:
run('python '+SCRIPTPATH+'/smoothcal_rx42.py ' + ms + ' ' + ms+'/'+parmdb + ' ' + ms+'/'+parmdb+'_smoothed'+' > '+ms+'_'+parmdb+'_smoothed.log')
else:
run('python '+SCRIPTPATH+'/smoothcal_a2256_nophasors.py ' + ms + ' ' + ms+'/'+parmdb + ' ' + ms+'/'+parmdb+'_smoothed'+' > '+ms+'_'+parmdb+'_smoothed.log')
# apply amps
if smooth:
runbbs(mslist, skymodel,SCRIPTPATH+'/apply_amplitudeonly.parset',parmdb+'_smoothed', True, False, clusterdesc, dbserver, dbuser, dbname)
else:
runbbs(mslist, skymodel,SCRIPTPATH+'/apply_amplitudeonly.parset',parmdb, True, False, clusterdesc, dbserver, dbuser, dbname)
### MAKE IMAGE 4 ###
logging.info('Make image 4')
imout,mask = make_image(mslist, cluster, '4', 10, 10, nterms, atrous_do, imsize, region, ncores, SCRIPTPATH)
### CREATE FINAL MODEL ###
logging.info('Create final model')
skymodelf= 'im_cluster'+cluster+ '.final.skymodel'
run(SCRIPTPATH+'/casapy2bbs.py -m '+ mask + ' ' +'-t ' + str(nterms)+ ' ' + imout+'.model ' + skymodelf)
if FFT:
run('casapy --nogui -c '+SCRIPTPATH+'/ft_v2.py ' + msinputlist + ' ' + imout+'.model' \
+ ' ' + str(nterms) + ' '+ str(wplanes))
### CREATED MERGED PARMDB SCALARPHASE+AMPS ###
### INCLUDES SPLINE INTERPOLARION OF AMPS ###
if merge_parmdb:
logging.info('Merge parmdb')
if phasors:
dummyparset = SCRIPTPATH+'/scalarphase+amp.parset'
else:
if TEC:
dummyparset = SCRIPTPATH+'/scalarphase+ap+TEC.parset'
else:
dummyparset = SCRIPTPATH+'/scalarphase+ap.parset'
if TEC:
if clock:
dummyparmdb = 'instrument_template_TECclock'
else:
dummyparmdb = 'instrument_template_Gain_TEC_CSphase'
#if not os.path.isdir(dummyparmdb):
#runbbs([mslist[0]], skymodel,dummyparset, dummyparmdb, True, False)
# TO SPEED THINGS UP, hard coded for BOOTES - i.e. the above has already been run
for ms in mslist:
os.system('rm -rf ' + ms +'/' + dummyparmdb)
os.system('cp -r ' + dummyparmdb + ' ' + ms + '/instrument_template')
if smooth:
parmdb_a = 'instrument_amps1_smoothed' # last/best ampplitude(+phase) parmdb
else:
parmdb_a = 'instrument_amps1' # last/best ampplitude(+phase) parmdb
parmdb_p = 'instrument_phase1' # last/best CommonScalarPhase parmdb
parmdbout = 'instrument_merged'
#reset in case of instrument_template_TECclock
dummyparmdb = 'instrument_template'
for ms in mslist:
create_merged_parmdb(ms, ms+'/'+parmdb_a, ms+'/'+parmdb_p, ms+'/'+dummyparmdb,ms+'/'+parmdbout,cellsizetime_a,cellsizetime_p)
def make_image(mslist, cluster, callnumber, threshpix, threshisl, nterms, atrous_do, imsize, region, ncores, SCRIPTPATH):
do_mask = True
niter = 1000 # 7500 causes nasty clean artifacts
mscale = 'False'
if atrous_do:
mscale = 'True'
average = True # average the data a lot to speed up the imaging,
# ONLY average for small FOV, otherwise timesmearing is a problem
#average data
if average:
b=bg(maxp=2)
for ms in (mslist):
ndppp_parset = ms + '_NDPPP.parset'
ndppplog = ndppp_parset.replace('.parset','.log')
os.system('rm -f ' + ndppp_parset)
output = ms + '.tmpavg'
os.system('rm -rf ' + output)
f=open(ndppp_parset, 'w')
f.write('msin = %s\n' % ms)
f.write('msin.datacolumn = CORRECTED_DATA\n')
f.write('msout = %s\n' % output)
f.write('msout.writefullresflag=False\n')
f.write('steps=[avg]\n')
f.write('rficonsole.type=aoflagger\n')
f.write('avg.type = squash\n')
f.write('avg.freqstep = 1\n')
f.write('avg.timestep = 12\n') # is the default
f.close()
ndpppcmd = 'NDPPP ' + ndppp_parset+ ' >'+ndppplog+' 2>&1'
b.run(ndpppcmd)
#os.system (ndpppcmd)
b.wait()
#output=numpy.int(Popen(cmd, shell=True, stdout=PIPE).communicate()[0])
#while output > 0 :
#time.sleep(10)
#output=numpy.int(Popen(cmd, shell=True, stdout=PIPE).communicate()[0])
#pid = (Popen('pgrep NDPPP', shell=True, stdout=PIPE).communicate()[0])
##print pid
#pid_list = pid.split()
#for pidnr in pid_list:
##print 'PID', numpy.int(pidnr)
#os.system('kill -s CONT ' + str(pidnr))
ms = ''
for m in mslist:
if average:
ms = ms + ' ' + m + '.tmpavg'
else:
ms = ms + ' ' + m
imout = 'im'+ callnumber +'_cluster'+cluster+'nm'
logging.debug(ms + ' ' + imout + ' ' + 'None' + ' ' + '1mJy' + ' ' + str(niter) + ' ' + str(nterms))
if do_mask:
if cluster == 'a2256': ## special case for a2256
niter = niter*15 # clean very deep here
run('casapy --nogui --logfile casapy-'+imout+'.log -c '+SCRIPTPATH+'/casapy_cleanv4.py ' + ms + ' ' + imout + ' ' + 'None' +\
' ' + '1mJy' + ' ' + str(niter) + ' ' + str(nterms) + ' ' + str(imsize) + ' ' + mscale)
# make mask
if nterms > 1:
do_makecleanmask(imout +'.image.tt0',threshpix,threshisl,atrous_do,ncores=ncores)
# run('python '+SCRIPTPATH+'/makecleanmask.py --threshpix '+str(threshpix)+\
# ' --threshisl '+str(threshisl) +' --atrous_do '+ str(atrous_do) +' ' +imout +'.image.tt0')
else:
do_makecleanmask(imout +'.image',threshpix,threshisl,atrous_do,ncores=ncores)
# run('python '+SCRIPTPATH+'/makecleanmask.py --threshpix '+str(threshpix)+\
# ' --threshisl '+str(threshisl) +' --atrous_do '+ str(atrous_do) + ' ' + imout +'.image')
# clean image with manual mask
mask = imout+'.cleanmask'
niter = 1000
imout = 'im'+ callnumber +'_cluster'+cluster
if region != 'empty' : ## special cases
niter = niter*3
run('casapy --nogui --logfile casapy-'+imout+'.log -c '+SCRIPTPATH+'/casapy_cleanv4.py '+ ms + ' ' + imout + ' ' + mask+','+region + \
' ' + '1mJy' + ' ' + str(niter) + ' ' + str(nterms) + ' ' + str(imsize) + ' ' + mscale)
else:
run('casapy --nogui --logfile casapy-'+imout+'.log -c '+SCRIPTPATH+'/casapy_cleanv4.py '+ ms + ' ' + imout + ' ' + mask + \
' ' + '1mJy' + ' ' + str(niter) + ' ' + str(nterms) + ' ' + str(imsize) + ' ' + mscale)
# convert to FITS
if nterms > 1:
run('image2fits in=' + imout +'.image.tt0' + ' ' + 'out='+ imout + '.fits')
else:
run('image2fits in=' + imout +'.image' + ' ' + 'out='+ imout + '.fits')
if not os.path.exists(imout+'.fits'):
raise Exception('Imaging Error: '+imout+'.fits does not exist' )
if average:
logging.debug('rm -rf {}'.format(ms))
os.system('rm -rf ' + ms)
return imout,imout+'.mask'
def do_verify_subtract(mslist, res_val, source, numchanperms=20):
''' main function for verify_subtract '''
msavglist = []
for ms_id, ms in enumerate(mslist):
msavglist.append(ms.split('.')[0] + '.' + source + '.ms.avgcheck')
username = pwd.getpwuid(os.getuid())[0]
###########################################################################
# NDPPP phase shift, less averaging (NEW: run 2 in parallel)
b=bg(maxp=2)
for ms_id, ms in enumerate(mslist):
parset = create_phaseshift_parset_field(ms, msavglist[ms_id], numchanperms=numchanperms)
#ncmd='NDPPP ' + parset+' &>'+ms.split('.')[0] +'.ndppp_avgphaseshift_check.log'
ncmd='NDPPP ' + parset+' >'+ms.split('.')[0] +'.ndppp_avgphaseshift_check.log' + ' 2>&1'
print 'Running',ncmd
b.run(ncmd)
# Check if all NDPPP processes are finished
b.wait()
###########################################################################
imsize = 2048
###########################################################################
# IMAGE IN PARALLEL
b=bg(maxp=8)
for ms in msavglist:
imout = 'im'+ '_residual_' + source + '_' + ms.split('.')[0]
b.run('casapy --nogui -c '+SCRIPTPATH+'/casapy_cleanv4_checksubtract.py ' +\
ms + ' ' + imout + ' ' + str(imsize))
time.sleep(20)
# Check if all NDPPP processes are finished
b.wait()
#conver the images to FITS format
for ms in msavglist:
imout = 'im'+ '_residual_' + source + '_' + ms.split('.')[0]
run('image2fits in=' + imout +'.image' + ' ' + 'out='+ imout + '.fits')
stopcal = False
for ms in msavglist:
# find the source which was done before the current one
g=sorted(glob.glob('im_residual_*_' + ms.split('.')[0] + '.image'),key=os.path.getmtime,reverse=True)
if len(g) > 1:
pre_sourcename = g[1]
#print 'Previous image was', pre_sourcename
else:
pre_sourcename=None
image = 'im'+ '_residual_' + source + '_' + ms.split('.')[0] + '.image'
img = pyrap.images.image(image)
pixels = numpy.copy(img.getdata())
maxval = numpy.max(pixels)
minval = numpy.min(pixels)
maxvalpre = 1e9
minvalpre = -maxvalpre
if pre_sourcename is not None:
imgpre = pyrap.images.image(pre_sourcename)
pixelspre = numpy.copy(imgpre.getdata())
maxvalpre = numpy.max(pixelspre)
minvalpre = numpy.min(pixelspre)
logging.info('verify_subtract: For %s max, min residual were %f, %f' % (image,maxval,minval))
logging.debug("{} {} {}".format(maxval, minval, image))
if pre_sourcename is not None:
logging.debug("{} {} {}".format(maxvalpre, minvalpre, pre_sourcename))
if (maxval > res_val) or ((maxval*0.92) > maxvalpre) :
stopcal = True
logging.warning('RESIDUAL TOO LARGE, STOPPING {} {}'.format(maxval, res_val))
logging.warning('RESIDUAL TOO LARGE, STOPPING, previous max in image {}'.format(maxvalpre))
return stopcal
def do_selfcal(mslist,
cluster,
atrous_do,
imsize,
nterms,
cellsizetime_a,
cellsizetime_p,
TECi,
clocki,
HRi,
region,
clusterdesc,
dbserver,
dbuser,
dbname,
SCRIPTPATH,
ncores=8,
config=None):
TEC = False
FFT = False
clock = False
HR = False # high dynamic range
if TECi == "True":
TEC = True
if clocki == "True":
clock = True
if HRi == "HD":
HR = True
if imsize <= 2048:
wplanes = 1
FFT = True # FFT image into MODEL_DATA
if imsize >= 512:
wplanes = 64
if imsize > 799:
wplanes = 96
if imsize > 1023:
wplanes = 128
if imsize > 1599:
wplanes = 180
if imsize > 1800:
wplanes = 196
if imsize > 2049:
wplanes = 256
#if imsize > 3000:
# wplanes = 448
#if imsize > 4095:
# wplanes = 512
logging.info('mslist {}'.format(mslist))
logging.info('source {}'.format(cluster))
logging.info('atrous_do {}'.format(atrous_do))
logging.info('imsize {} '.format(imsize))
logging.info('TEC is {} and clock is {}'.format(TEC, clock))
msinputlist = ''
for m in mslist:
msinputlist = msinputlist + ' ' + m
#if len(mslist) == 29:
#group = "9,10,10"
#elif len(mslist) == 28:
#group = "7,7,7,7"
#elif len(mslist) == 20:
##group = "10,10"
#group = "7,7,6"
#elif len(mslist) == 16:
#group = "8,8"
#else:
#group = str(len(mslist))
group = get_group(mslist)
logging.info('GROUP {}'.format(group))
uvrange = '80'
#uvrange = '400'
merge_parmdb = True
phasors = False # if true only solve for amps on long timescales
smooth = False # seems that smooth does not help the selfcal (various reasons for that)
# 1. boundaries of flagged vs non-flagged data are sharp (should not be smoothed)
# 2. there sre some strong ampl various at low elevations
smooth = True # sometimes almost 0.0 amplitude, causes ripples
phasezero = True # reset phases from ap calibration
## Loop parameters
rms_old = 1.e9 # bad values to start with
dynamicrange_old = 1. # low value to start with so we get into the while loop
im_count = 4
number_forced_selfcalcycles = 8
factor = 1.0125 # demand 1.25% improvement
max_selfcalcycles = 16
if config is not None:
number_forced_selfcalcycles = config.get("selfcal_forced_cycles", 8)
factor = config.get("selfcal_factor", 1.0125)
max_selfcalcycles = config.get("selfcal_max_cycles", 16)
empty_mask_cycle = config.get("selfcal_empty_cycle", 5)
logging.info(
'Selfcal loop params: forced {}; empty mask {}; max {}; factor {}'.
format(number_forced_selfcalcycles, empty_mask_cycle,
max_selfcalcycles, factor))
#####################
#####################
#### MAKE IMAGE 0 ###
logging.info('Make image 0')
imout, mask = make_image(mslist, cluster, '0', 10, 6, nterms, atrous_do,
imsize, region, ncores, SCRIPTPATH)
#####################
### CALIBRATE WITH BBS PHASE ONLY 1 ###
# create skymodel for BBS
run(SCRIPTPATH + '/casapy2bbs.py -m ' + mask + ' ' + '-t ' + str(nterms) +
' ' + imout + '.model ' + imout + '.skymodel')
if FFT:
os.system('casapy --nogui -c '+SCRIPTPATH+'/ft_v2.py ' + msinputlist + ' ' + imout+'.model' \
+ ' ' + str(nterms) + ' '+ str(wplanes))
# phase only calibrate
skymodel = imout + '.skymodel'
parset = create_scalarphase_parset(cellsizetime_p, TEC, clock, group,
False, uvrange)
runbbs(mslist, '3C244.1.skymodel', parset, 'instrument', False, TEC,
clusterdesc, dbserver, dbuser, dbname)
#NOTE WORK FROM MODEL_DATA (contains correct phase data from 10SB calibration)
######################################
### MAKE IMAGE 1 ###
logging.info('Make image 1')
imout, mask = make_image(mslist, cluster, '1', 15, 15, nterms, atrous_do,
imsize, region, ncores, SCRIPTPATH)
####################
### CALIBRATE WITH BBS PHASE ONLY 2 ###
# create skymodel for BBS
run(SCRIPTPATH + '/casapy2bbs.py -m ' + mask + ' ' + '-t ' + str(nterms) +
' ' + imout + '.model ' + imout + '.skymodel')
if FFT:
run('casapy --nogui -c '+SCRIPTPATH+'/ft_v2.py ' + msinputlist + ' ' + imout+'.model' \
+ ' ' + str(nterms) + ' '+ str(wplanes))
# phase only calibrate
skymodel = imout + '.skymodel'
parset = create_scalarphase_parset(cellsizetime_p, TEC, clock, group, FFT,
uvrange)
runbbs(
mslist, skymodel, parset, 'instrument', False, TEC, clusterdesc,
dbserver, dbuser, dbname
) #NOTE WORK FROM MODEL_DATA (contains correct phase data from 10SB calibration)
######################################
### MAKE IMAGE 2 ###
logging.info('Make image 2')
imout, mask = make_image(mslist, cluster, '2', 15, 15, nterms, atrous_do,
imsize, region, ncores, SCRIPTPATH)
####################
### CALIBRATE WITH BBS PHASE+AMP 1 ###
run(SCRIPTPATH + '/casapy2bbs.py -m ' + mask + ' ' + '-t ' + str(nterms) +
' ' + imout + '.model ' + imout + '.skymodel')
if FFT:
run('casapy --nogui -c '+SCRIPTPATH+'/ft_v2.py ' + msinputlist + ' ' + imout+'.model' \
+ ' ' + str(nterms) + ' '+ str(wplanes))
skymodel = imout + '.skymodel'
parset = create_scalarphase_parset_p(cellsizetime_p, TEC, clock, group,
FFT, uvrange)
# solve +apply phases
runbbs(mslist, skymodel, parset, 'instrument_phase0', False, TEC,
clusterdesc, dbserver, dbuser, dbname)
# solve amps
parmdb = 'instrument_amps0'
parset = create_amponly_parset(cellsizetime_a, FFT, uvrange)
runbbs(mslist, skymodel, parset, parmdb, False, False, clusterdesc,
dbserver, dbuser, dbname)
for ms in mslist:
# remove outliers from the solutions
if phasors:
run('python ' + SCRIPTPATH + '/smoothcal_rx42.py ' + ms + ' ' +
ms + '/' + parmdb + ' ' + ms + '/' + parmdb + '_smoothed' +
' > ' + ms + '_' + parmdb + '_smoothed.log')
else:
run('python ' + SCRIPTPATH + '/smoothcal_a2256_nophasors.py ' +
ms + ' ' + ms + '/' + parmdb + ' ' + ms + '/' + parmdb +
'_smoothed' + ' > ' + ms + '_' + parmdb + '_smoothed.log')
# apply amps
if smooth:
runbbs(mslist, skymodel, SCRIPTPATH + '/apply_amplitudeonly.parset',
parmdb + '_smoothed', True, False, clusterdesc, dbserver,
dbuser, dbname)
else:
runbbs(mslist, skymodel, SCRIPTPATH + '/apply_amplitudeonly.parset',
parmdb, True, False, clusterdesc, dbserver, dbuser, dbname)
### MAKE IMAGE 3 ###
logging.info('Make image 3')
imout, mask = make_image(mslist, cluster, '3', 10, 10, nterms, atrous_do,
imsize, region, ncores, SCRIPTPATH)
####################
# LOOP
####################
rms, dynamicrange = find_imagenoise(imout + '.image')
while (((dynamicrange / factor) > dynamicrange_old)
or ((rms * factor) < rms_old)):
logging.info('Starting selfcal loop {}'.format(im_count))
#### CALIBRATE BBS PHASE+AMP 2 (LOOP) ###
# make model
run(SCRIPTPATH + '/casapy2bbs.py -m ' + mask + ' ' + '-t ' +
str(nterms) + ' ' + imout + '.model ' + imout + '.skymodel')
if FFT:
run('casapy --nogui -c '+SCRIPTPATH+'/ft_v2.py ' + msinputlist + ' ' + imout+'.model' \
+ ' ' + str(nterms) + ' '+ str(wplanes))
#parmdb keep from previous step
skymodel = imout + '.skymodel'
# reset the phases from instrument_amps0 to zero to prevent large phase corrections from incorrect AP solve
# FIXME: Is this used?
if phasezero:
inputparmdb = parmdb + '_smoothed'
outputparmdb = parmdb + '_smoothed_phasezero'
for ms in mslist:
run('python ' + SCRIPTPATH + '/setphasezero.py ' + ms + ' ' +
ms + '/' + inputparmdb + ' ' + ms + '/' + outputparmdb)
else:
outputparmdb = parmdb + '_smoothed'
# phase only cal
skymodel = imout + '.skymodel'
parset = create_scalarphase_parset_p(cellsizetime_p, TEC, clock, group,
FFT, uvrange)
runbbs(mslist, skymodel, parset, 'instrument_phase1', False, TEC,
clusterdesc, dbserver, dbuser, dbname)
# solve amps
parmdb = 'instrument_amps1'
parset = create_amponly_parset(cellsizetime_a, FFT, uvrange)
runbbs(mslist, skymodel, parset, parmdb, False, False, clusterdesc,
dbserver, dbuser, dbname)
for ms in mslist:
# remove outliers from the solutions
if phasors:
run('python ' + SCRIPTPATH + '/smoothcal_rx42.py ' + ms + ' ' +
ms + '/' + parmdb + ' ' + ms + '/' + parmdb + '_smoothed' +
' > ' + ms + '_' + parmdb + '_smoothed.log')
else:
run('python ' + SCRIPTPATH + '/smoothcal_a2256_nophasors.py ' +
ms + ' ' + ms + '/' + parmdb + ' ' + ms + '/' + parmdb +
'_smoothed' + ' > ' + ms + '_' + parmdb + '_smoothed.log')
# apply amps
if smooth:
runbbs(mslist, skymodel,
SCRIPTPATH + '/apply_amplitudeonly.parset',
parmdb + '_smoothed', True, False, clusterdesc, dbserver,
dbuser, dbname)
else:
runbbs(mslist, skymodel,
SCRIPTPATH + '/apply_amplitudeonly.parset', parmdb, True,
False, clusterdesc, dbserver, dbuser, dbname)
### MAKE IMAGE #N ###
logging.info('Make image {}'.format(im_count))
# Do not use the initial mask in the final cycles
if im_count >= empty_mask_cycle:
region_im = "empty"
else:
region_im = region
imout, mask = make_image(mslist, cluster, str(im_count), 10, 10,
nterms, atrous_do, imsize, region_im, ncores,
SCRIPTPATH)
## Prepare the next iteration
im_count += 1
# save previous values to compare with
rms_old = rms
dynamicrange_old = dynamicrange
if nterms < 2:
rms, dynamicrange = find_imagenoise(imout + '.image')
else:
rms, dynamicrange = find_imagenoise(imout + '.image.tt0')
logging.info('IMAGE STATISTICS {}, {}'.format(rms, dynamicrange))
if im_count < number_forced_selfcalcycles:
rms_old = 1.e9 # bad values to start with
dynamicrange_old = 1.
logging.debug(
"Count below the number of forced selfcal cycles. Force new loop."
)
if im_count >= max_selfcalcycles:
logging.info(
"Maximum number of cycles ({}) reached. Leaving selfcal loop.".
format(max_selfcalcycles))
break
### CREATE FINAL MODEL ###
logging.info('Create final model')
skymodelf = 'im_cluster' + cluster + '.final.skymodel'
run(SCRIPTPATH + '/casapy2bbs.py -m ' + mask + ' ' + '-t ' + str(nterms) +
' ' + imout + '.model ' + skymodelf)
if FFT:
run('casapy --nogui -c '+SCRIPTPATH+'/ft_v2.py ' + msinputlist + ' ' + imout+'.model' \
+ ' ' + str(nterms) + ' '+ str(wplanes))
### CREATED MERGED PARMDB SCALARPHASE+AMPS ###
### INCLUDES SPLINE INTERPOLARION OF AMPS ###
if merge_parmdb:
logging.info('Merge parmdb')
if phasors:
dummyparset = SCRIPTPATH + '/scalarphase+amp.parset'
else:
if TEC:
dummyparset = SCRIPTPATH + '/scalarphase+ap+TEC.parset'
else:
dummyparset = SCRIPTPATH + '/scalarphase+ap.parset'
if TEC:
if clock:
dummyparmdb = 'instrument_template_TECclock'
else:
dummyparmdb = 'instrument_template_Gain_TEC_CSphase'
#if not os.path.isdir(dummyparmdb):
#runbbs([mslist[0]], skymodel,dummyparset, dummyparmdb, True, False)
# TO SPEED THINGS UP, hard coded for BOOTES - i.e. the above has already been run
for ms in mslist:
os.system('rm -rf ' + ms + '/' + dummyparmdb)
os.system('cp -r ' + dummyparmdb + ' ' + ms +
'/instrument_template')
if smooth:
parmdb_a = 'instrument_amps1_smoothed' # last/best ampplitude(+phase) parmdb
else:
parmdb_a = 'instrument_amps1' # last/best ampplitude(+phase) parmdb
parmdb_p = 'instrument_phase1' # last/best CommonScalarPhase parmdb
parmdbout = 'instrument_merged'
#reset in case of instrument_template_TECclock
dummyparmdb = 'instrument_template'
for ms in mslist:
create_merged_parmdb(ms, ms + '/' + parmdb_a, ms + '/' + parmdb_p,
ms + '/' + dummyparmdb, ms + '/' + parmdbout,
cellsizetime_a, cellsizetime_p)
def make_image_wsclean_nomask(mslist, cluster, callnumber, threshpix,
threshisl, nterms, atrous_do, imsize, inputmask,
mscale, region, cellsize, uvrange, wsclean,
WSCleanRobust):
if imsize is None:
imsize = image_size_from_mask(inputmask)
niter = numpy.int(20000 * (numpy.sqrt(numpy.float(len(mslist)))))
cellsizeim = str(cellsize) + 'arcsec'
depth = 1e-3 * 0.7 / (numpy.sqrt(numpy.float(len(mslist))))
print 'Cleaning to a noise level of', depth, 'Jy: niter is', niter
cleandepth2 = str(depth) #+ 'mJy'
wideband = False
if len(mslist) > 5:
wideband = True
ms = ''
for m in mslist:
ms = ms + ' ' + m
imout = 'im' + callnumber + '_cluster' + cluster
os.system('rm -rf ' + imout + '-*')
outms = 'field-' + cluster + '.ms'
parsetname = 'concatforwsclean-' + cluster + '.parset'
msinstr = ""
for ms_id, ms in enumerate(mslist):
msinstr = msinstr + "'" + ms + "'"
if ms_id < len(mslist) - 1:
msinstr = msinstr + ", "
os.system('rm -rf ' + parsetname)
f = open(parsetname, 'w')
f.write('msin = [%s]\n' % msinstr)
f.write('msin.datacolumn = DATA\n')
f.write('msin.missingdata=True\n')
f.write('msin.orderms=False\n')
f.write('msout=%s\n' % outms)
f.write('steps=[]\n')
f.close()
os.system('rm -rf ' + outms)
run('NDPPP ' + parsetname)
if wideband:
channelsout = 1 # there is a factor of 5 averaging
cmd1 = wsclean + ' -reorder -name ' + imout + ' -size ' + str(
imsize) + ' ' + str(imsize) + ' '
cmd2 = '-scale ' + cellsizeim + ' -weight briggs ' + str(
WSCleanRobust) + ' -niter ' + str(
niter) + '-cleanborder 0 -threshold ' + cleandepth2 + ' '
cmd3 = '-minuv-l '+ str(uvrange) + ' -casamask ' + inputmask + ' '\
+' -mgain 0.6 -fitbeam -datacolumn DATA -no-update-model-required -joinchannels -channelsout ' +\
str(channelsout) + ' ' + outms
else:
cmd1 = wsclean + ' -reorder -name ' + imout + ' -size ' + str(
imsize) + ' ' + str(imsize) + ' '
cmd2 = '-scale ' + cellsizeim + ' -weight briggs ' + str(
WSCleanRobust) + ' -niter ' + str(
niter) + ' -cleanborder 0 -threshold ' + cleandepth2 + ' '
cmd3 = '-minuv-l ' + str(
uvrange
) + ' -casamask ' + inputmask + ' -mgain 0.6 -fitbeam -datacolumn DATA -no-update-model-required ' + outms
print cmd1 + cmd2 + cmd3
run(cmd1 + cmd2 + cmd3)
finalim = pyrap.images.image(imout + '-image.fits')
finalim.saveas(imout + '.image')
return imout, None, imsize
## NOTE: addback now needs to be done as subtract in reverse... with FT and allbands concat
## STEP 1: check ##
if StartAtStep in ['preSC']:
# if we didn't keep the allbands.concat.shifted
if not os.path.exists('allbands.concat.shifted_' + source + '.ms'):
parset = create_phaseshift_parset_full(
'allbands.concat.ms',
'allbands.concat.shifted_' + source + '.ms', facetdir,
'DATA')
ndppplog = parset.replace('.parset', '.log')
#ndpppcmd = 'NDPPP ' + parset + ' > '+ ndppplog + ' 2>&1'
#ndppprc = os.system(ndpppcmd)
ndpppcmd = 'NDPPP ' + parset + ' > ' + ndppplog + ' 2>&1'
run(ndpppcmd)
### STEP 3: prep for facet ##
if StartAtStep in ['preSC', 'preFACET']:
# image model to add back
imsizef = image_size_from_mask(output_template_im)
imout = 'im' + 'field0' + '_cluster' + source
if not os.path.exists(imout + '-model.fits'):
raise Exception(imout + '-model.fits is missing')
logging.info('running ft: ' + imout)
# DO THE FFT
do_fieldFFT('allbands.concat.shifted_' + source + '.ms', imout,