def test_param3_no_source_default(self):
"""
Existing parameter with other sources and default in key
Use default over function default when appropriate
"""
self.assertEqual(get_config("param3", self.config1, default=0), 4)
self.assertEqual(get_config("param3", self.config1, source="s1", default=0), 4)
self.assertEqual(get_config("param3", self.config1, source="s2", default=0), 5)
def test_param4(self):
"""
Test non existing parameter
"""
self.assertEqual(get_config("param4", self.config1, default=0), 0)
with self.assertRaises(KeyError):
get_config("param4", self.config1, source="s2")
with self.assertRaises(KeyError):
get_config("param4", self.config1)
def test_real(self):
"""
Test real case
"""
self.assertEqual(get_config("model_facet", self.config2, default=""),
'/data/models/output_2015-12/facet2/Facet_1.sky')
def test_param3_no_source(self):
"""
Existing parameter with other sources and default in key
Use default
"""
self.assertEqual(get_config("param3", self.config1), 4)
def test_param3_no_source2(self):
"""
Existing parameter with other sources and default in key
Use default, non-existent source
"""
self.assertEqual(get_config("param3", self.config1, source="s1"), 4)
def test_param2_no_source_default(self):
"""
Existing parameter with other sources and default in function
"""
self.assertEqual(get_config("param2", self.config1, source="s2", default=0), 0)
def test_param3(self):
"""
Existing parameter with other sources and default in key
Use exsisting source
"""
self.assertEqual(get_config("param3", self.config1, source="s2"), 5)
def test_param2_no_source(self):
"""
Existing parameter with other sources and no default
"""
with self.assertRaises(KeyError):
get_config("param2", self.config1, source="s2")
def test_param2(self):
"""
Exising parameter with source
"""
self.assertEqual(get_config("param2", self.config1, source="s1"), 2)
self.assertEqual(get_config("param2", self.config1, source="s3"), 3)
def test_param1(self):
"""
Normal existing parameter
"""
self.assertEqual(get_config("param1", self.config1), 1)
def do_selfcal(mslist, cluster, atrous_do, imsize, nterms, cellsizetime_a, cellsizetime_p,
TECi, clocki, HRi, region, clusterdesc, dbserver, dbuser, dbname, SCRIPTPATH,
ncores=8, config=None, model=None):
TEC = False
FFT = False
clock= False
HR = False # high dynamic range
if TECi == "True":
TEC = True
if clocki == "True":
clock = True
if HRi == "HD":
HR = True
if imsize <=2048:
wplanes = 1
FFT = True # FFT image into MODEL_DATA
if imsize >= 512:
wplanes = 64
if imsize > 799:
wplanes = 96
if imsize > 1023:
wplanes = 128
if imsize > 1599:
wplanes = 180
if imsize > 1800:
wplanes = 196
if imsize > 2049:
wplanes = 256
#if imsize > 3000:
# wplanes = 448
#if imsize > 4095:
# wplanes = 512
logging.info('mslist {}'.format(mslist))
logging.info('source {}'.format(cluster))
logging.info('atrous_do {}'.format(atrous_do))
logging.info('imsize {} '.format(imsize))
logging.info('TEC is {} and clock is {}'.format(TEC, clock))
msinputlist = ''
for m in mslist:
msinputlist = msinputlist + ' ' + m
#if len(mslist) == 29:
#group = "9,10,10"
#elif len(mslist) == 28:
#group = "7,7,7,7"
#elif len(mslist) == 20:
##group = "10,10"
#group = "7,7,6"
#elif len(mslist) == 16:
#group = "8,8"
#else:
#group = str(len(mslist))
group = get_group(mslist)
logging.info('GROUP {}'.format(group))
uvrange = '80'
#uvrange = '400'
merge_parmdb = True
phasors = False # if true only solve for amps on long timescales
smooth = False # seems that smooth does not help the selfcal (various reasons for that)
# 1. boundaries of flagged vs non-flagged data are sharp (should not be smoothed)
# 2. there sre some strong ampl various at low elevations
smooth = True # sometimes almost 0.0 amplitude, causes ripples
phasezero = True # reset phases from ap calibration
## Loop parameters
rms_old = 1.e9 # bad values to start with
dynamicrange_old = 1. # low value to start with so we get into the while loop
im_count = 4
number_forced_selfcalcycles = 8
factor = 1.0125 # demand 1.25% improvement
max_selfcalcycles = 16
model_facet = ""
if config is not None:
number_forced_selfcalcycles = get_config("selfcal_forced_cycles", config, default=8)
factor = get_config("selfcal_factor", config, default=1.0125)
max_selfcalcycles = get_config("selfcal_max_cycles", config, default=16)
empty_mask_cycle = get_config("selfcal_empty_cycle", config, default=5)
model_facet = get_config("model_facet", config, default="")
logging.info('Selfcal loop params: forced {}; empty mask {}; max {}; factor {}; model {}; model_facet {}'.format(
number_forced_selfcalcycles, empty_mask_cycle, max_selfcalcycles, factor, model, model_facet))
#####################
#####################
#### MAKE IMAGE 0 ###
logging.info('Make image 0')
imout,mask = make_image(mslist, cluster, '0', 10, 6, nterms, atrous_do, imsize, region, ncores, SCRIPTPATH)
#####################
### CALIBRATE WITH BBS PHASE ONLY 1 ###
# create skymodel for BBS
run(SCRIPTPATH+'/casapy2bbs.py -m '+ mask + ' ' +'-t ' + str(nterms)+ ' ' + imout+'.model ' + imout+'.skymodel')
if FFT:
os.system('casapy --nogui -c '+SCRIPTPATH+'/ft_v2.py ' + msinputlist + ' ' + imout+'.model' \
+ ' ' + str(nterms) + ' '+ str(wplanes))
# phase only calibrate
if model_facet:
skymodel = model_facet
logging.info('Using config model '+model_facet+" skip model from image 0")
else:
skymodel = imout+'.skymodel'
parset = create_scalarphase_parset(cellsizetime_p, TEC, clock, group, FFT, uvrange)
logging.debug("Run BBS (pre-image 1) with skymodel: {}".format(skymodel))
runbbs(mslist, skymodel, parset, 'instrument', False, TEC, clusterdesc, dbserver, dbuser, dbname)
#NOTE WORK FROM MODEL_DATA (contains correct phase data from 10SB calibration)
######################################
### MAKE IMAGE 1 ###
logging.info('Make image 1')
imout,mask = make_image(mslist, cluster, '1', 15, 15, nterms, atrous_do, imsize, region, ncores, SCRIPTPATH)
####################
### CALIBRATE WITH BBS PHASE ONLY 2 ###
# create skymodel for BBS
logging.debug("".format())
run(SCRIPTPATH+'/casapy2bbs.py -m '+ mask + ' ' +'-t ' + str(nterms)+ ' ' + imout+'.model ' + imout+'.skymodel')
if FFT:
run('casapy --nogui -c '+SCRIPTPATH+'/ft_v2.py ' + msinputlist + ' ' + imout+'.model' \
+ ' ' + str(nterms) + ' '+ str(wplanes))
# phase only calibrate
skymodel = imout+'.skymodel'
parset = create_scalarphase_parset(cellsizetime_p, TEC, clock, group, FFT, uvrange)
logging.debug("Run BBS (pre-image 2) with skymodel: {}".format(skymodel))
runbbs(mslist, skymodel, parset, 'instrument', False, TEC,clusterdesc, dbserver, dbuser, dbname) #NOTE WORK FROM MODEL_DATA (contains correct phase data from 10SB calibration)
######################################
### MAKE IMAGE 2 ###
logging.info('Make image 2')
imout,mask = make_image(mslist, cluster, '2', 15, 15, nterms, atrous_do, imsize, region, ncores, SCRIPTPATH)
####################
### CALIBRATE WITH BBS PHASE+AMP 1 ###
run(SCRIPTPATH+'/casapy2bbs.py -m '+ mask + ' ' +'-t ' + str(nterms)+ ' ' + imout+'.model ' + imout+'.skymodel')
if FFT:
run('casapy --nogui -c '+SCRIPTPATH+'/ft_v2.py ' + msinputlist + ' ' + imout+'.model' \
+ ' ' + str(nterms) + ' '+ str(wplanes))
skymodel = imout+'.skymodel'
parset = create_scalarphase_parset_p(cellsizetime_p, TEC, clock, group, FFT, uvrange)
# solve +apply phases
logging.debug("Run BBS (pre-image 3) with skymodel: {}".format(skymodel))
runbbs(mslist, skymodel, parset, 'instrument_phase0', False, TEC, clusterdesc, dbserver, dbuser, dbname)
# solve amps
parmdb = 'instrument_amps0'
parset = create_amponly_parset(cellsizetime_a, FFT, uvrange)
runbbs(mslist, skymodel, parset, parmdb, False, False, clusterdesc, dbserver, dbuser, dbname)
for ms in mslist:
# remove outliers from the solutions
if phasors:
run('python '+SCRIPTPATH+'/smoothcal_rx42.py ' + ms + ' ' + ms+'/'+parmdb + ' ' + ms+'/'+parmdb+'_smoothed'+' > '+ms+'_'+parmdb+'_smoothed.log')
else:
run('python '+SCRIPTPATH+'/smoothcal_a2256_nophasors.py ' + ms + ' ' + ms+'/'+parmdb + ' ' + ms+'/'+parmdb+'_smoothed'+' > '+ms+'_'+parmdb+'_smoothed.log')
# apply amps
if smooth:
runbbs(mslist, skymodel,SCRIPTPATH+'/apply_amplitudeonly.parset', parmdb+'_smoothed', True, False, clusterdesc, dbserver, dbuser, dbname)
else:
runbbs(mslist, skymodel,SCRIPTPATH+'/apply_amplitudeonly.parset', parmdb, True, False, clusterdesc, dbserver, dbuser, dbname)
### MAKE IMAGE 3 ###
logging.info('Make image 3')
imout,mask = make_image(mslist, cluster, '3', 10, 10, nterms, atrous_do, imsize, region, ncores, SCRIPTPATH)
####################
# LOOP
####################
rms, dynamicrange = find_imagenoise(imout + '.image')
while (((dynamicrange/factor) > dynamicrange_old) or ((rms*factor) < rms_old)):
logging.info('Starting selfcal loop {}'.format(im_count))
#### CALIBRATE BBS PHASE+AMP 2 (LOOP) ###
# make model
run(SCRIPTPATH+'/casapy2bbs.py -m '+ mask + ' ' +'-t ' + str(nterms)+ ' ' + imout+'.model ' + imout+'.skymodel')
if FFT:
run('casapy --nogui -c '+SCRIPTPATH+'/ft_v2.py ' + msinputlist + ' ' + imout+'.model' \
+ ' ' + str(nterms) + ' '+ str(wplanes))
#parmdb keep from previous step
skymodel = imout+'.skymodel'
# reset the phases from instrument_amps0 to zero to prevent large phase corrections from incorrect AP solve
# FIXME: Is this used?
if phasezero:
inputparmdb = parmdb +'_smoothed'
outputparmdb = parmdb +'_smoothed_phasezero'
for ms in mslist:
run('python '+SCRIPTPATH+'/setphasezero.py ' + ms + ' ' + ms+'/'+inputparmdb +' ' + ms+'/'+outputparmdb)
else:
outputparmdb = parmdb +'_smoothed'
# phase only cal
skymodel = imout+'.skymodel'
parset = create_scalarphase_parset_p(cellsizetime_p, TEC, clock, group, FFT, uvrange)
logging.debug("Run BBS (pre-image {}) with skymodel: {}".format(im_count, skymodel))
runbbs(mslist, skymodel, parset, 'instrument_phase1', False, TEC, clusterdesc, dbserver, dbuser, dbname)
# solve amps
parmdb = 'instrument_amps1'
parset = create_amponly_parset(cellsizetime_a, FFT, uvrange)
runbbs(mslist, skymodel, parset,parmdb, False, False, clusterdesc, dbserver, dbuser, dbname)
for ms in mslist:
# remove outliers from the solutions
if phasors:
run('python '+SCRIPTPATH+'/smoothcal_rx42.py ' + ms + ' ' + ms+'/'+parmdb + ' ' + ms+'/'+parmdb+'_smoothed'+' > '+ms+'_'+parmdb+'_smoothed.log')
else:
run('python '+SCRIPTPATH+'/smoothcal_a2256_nophasors.py ' + ms + ' ' + ms+'/'+parmdb + ' ' + ms+'/'+parmdb+'_smoothed'+' > '+ms+'_'+parmdb+'_smoothed.log')
# apply amps
if smooth:
runbbs(mslist, skymodel,SCRIPTPATH+'/apply_amplitudeonly.parset',parmdb+'_smoothed', True, False, clusterdesc, dbserver, dbuser, dbname)
else:
runbbs(mslist, skymodel,SCRIPTPATH+'/apply_amplitudeonly.parset',parmdb, True, False, clusterdesc, dbserver, dbuser, dbname)
### MAKE IMAGE #N ###
logging.info('Make image {}'.format(im_count))
# Do not use the initial mask in the final cycles
if im_count >= empty_mask_cycle:
region_im = "empty"
else:
region_im = region
imout,mask = make_image(mslist, cluster, str(im_count), 10, 10, nterms, atrous_do, imsize, region_im, ncores, SCRIPTPATH)
## Prepare the next iteration
im_count += 1
# save previous values to compare with
rms_old = rms
dynamicrange_old = dynamicrange
if nterms < 2:
rms, dynamicrange = find_imagenoise(imout + '.image')
else:
rms, dynamicrange = find_imagenoise(imout + '.image.tt0')
logging.info('IMAGE STATISTICS {}, {}'.format(rms, dynamicrange))
if im_count < number_forced_selfcalcycles:
rms_old = 1.e9 # bad values to start with
dynamicrange_old = 1.
logging.debug("Count below the number of forced selfcal cycles. Force new loop.")
if im_count >= max_selfcalcycles:
logging.info("Maximum number of cycles ({}) reached. Leaving selfcal loop.".format(max_selfcalcycles))
break
### CREATE FINAL MODEL ###
logging.info('Create final model')
skymodelf= 'im_cluster'+cluster+ '.final.skymodel'
run(SCRIPTPATH+'/casapy2bbs.py -m '+ mask + ' ' +'-t ' + str(nterms)+ ' ' + imout+'.model ' + skymodelf)
if FFT:
run('casapy --nogui -c '+SCRIPTPATH+'/ft_v2.py ' + msinputlist + ' ' + imout+'.model' \
+ ' ' + str(nterms) + ' '+ str(wplanes))
### CREATED MERGED PARMDB SCALARPHASE+AMPS ###
### INCLUDES SPLINE INTERPOLARION OF AMPS ###
if merge_parmdb:
logging.info('Merge parmdb')
if phasors:
dummyparset = SCRIPTPATH+'/scalarphase+amp.parset'
else:
if TEC:
dummyparset = SCRIPTPATH+'/scalarphase+ap+TEC.parset'
else:
dummyparset = SCRIPTPATH+'/scalarphase+ap.parset'
if TEC:
if clock:
dummyparmdb = 'instrument_template_TECclock'
else:
dummyparmdb = 'instrument_template_Gain_TEC_CSphase'
#if not os.path.isdir(dummyparmdb):
#runbbs([mslist[0]], skymodel,dummyparset, dummyparmdb, True, False)
# TO SPEED THINGS UP, hard coded for BOOTES - i.e. the above has already been run
for ms in mslist:
os.system('rm -rf ' + ms +'/' + dummyparmdb)
os.system('cp -r ' + dummyparmdb + ' ' + ms + '/instrument_template')
if smooth:
parmdb_a = 'instrument_amps1_smoothed' # last/best ampplitude(+phase) parmdb
else:
parmdb_a = 'instrument_amps1' # last/best ampplitude(+phase) parmdb
parmdb_p = 'instrument_phase1' # last/best CommonScalarPhase parmdb
parmdbout = 'instrument_merged'
#reset in case of instrument_template_TECclock
dummyparmdb = 'instrument_template'
for ms in mslist:
create_merged_parmdb(ms, ms+'/'+parmdb_a, ms+'/'+parmdb_p, ms+'/'+dummyparmdb,ms+'/'+parmdbout,cellsizetime_a,cellsizetime_p)
def do_selfcal(mslist,
cluster,
atrous_do,
imsize,
nterms,
cellsizetime_a,
cellsizetime_p,
TECi,
clocki,
HRi,
region,
clusterdesc,
dbserver,
dbuser,
dbname,
SCRIPTPATH,
ncores=8,
config=None,
model=None):
TEC = False
FFT = False
clock = False
HR = False # high dynamic range
if TECi == "True":
TEC = True
if clocki == "True":
clock = True
if HRi == "HD":
HR = True
if imsize <= 2048:
wplanes = 1
FFT = True # FFT image into MODEL_DATA
if imsize >= 512:
wplanes = 64
if imsize > 799:
wplanes = 96
if imsize > 1023:
wplanes = 128
if imsize > 1599:
wplanes = 180
if imsize > 1800:
wplanes = 196
if imsize > 2049:
wplanes = 256
#if imsize > 3000:
# wplanes = 448
#if imsize > 4095:
# wplanes = 512
logging.info('mslist {}'.format(mslist))
logging.info('source {}'.format(cluster))
logging.info('atrous_do {}'.format(atrous_do))
logging.info('imsize {} '.format(imsize))
logging.info('TEC is {} and clock is {}'.format(TEC, clock))
msinputlist = ''
for m in mslist:
msinputlist = msinputlist + ' ' + m
#if len(mslist) == 29:
#group = "9,10,10"
#elif len(mslist) == 28:
#group = "7,7,7,7"
#elif len(mslist) == 20:
##group = "10,10"
#group = "7,7,6"
#elif len(mslist) == 16:
#group = "8,8"
#else:
#group = str(len(mslist))
group = get_group(mslist)
logging.info('GROUP {}'.format(group))
uvrange = '80'
#uvrange = '400'
merge_parmdb = True
phasors = False # if true only solve for amps on long timescales
smooth = False # seems that smooth does not help the selfcal (various reasons for that)
# 1. boundaries of flagged vs non-flagged data are sharp (should not be smoothed)
# 2. there sre some strong ampl various at low elevations
smooth = True # sometimes almost 0.0 amplitude, causes ripples
phasezero = True # reset phases from ap calibration
## Loop parameters
rms_old = 1.e9 # bad values to start with
dynamicrange_old = 1. # low value to start with so we get into the while loop
im_count = 4
number_forced_selfcalcycles = 8
factor = 1.0125 # demand 1.25% improvement
max_selfcalcycles = 16
model_facet = ""
if config is not None:
number_forced_selfcalcycles = get_config("selfcal_forced_cycles",
config,
default=8)
factor = get_config("selfcal_factor", config, default=1.0125)
max_selfcalcycles = get_config("selfcal_max_cycles",
config,
default=16)
empty_mask_cycle = get_config("selfcal_empty_cycle", config, default=5)
model_facet = get_config("model_facet", config, default="")
logging.info(
'Selfcal loop params: forced {}; empty mask {}; max {}; factor {}; model {}; model_facet {}'
.format(number_forced_selfcalcycles, empty_mask_cycle,
max_selfcalcycles, factor, model, model_facet))
#####################
#####################
#### MAKE IMAGE 0 ###
logging.info('Make image 0')
imout, mask = make_image(mslist, cluster, '0', 10, 6, nterms, atrous_do,
imsize, region, ncores, SCRIPTPATH)
#####################
### CALIBRATE WITH BBS PHASE ONLY 1 ###
# create skymodel for BBS
run(SCRIPTPATH + '/casapy2bbs.py -m ' + mask + ' ' + '-t ' + str(nterms) +
' ' + imout + '.model ' + imout + '.skymodel')
if FFT:
os.system('casapy --nogui -c '+SCRIPTPATH+'/ft_v2.py ' + msinputlist + ' ' + imout+'.model' \
+ ' ' + str(nterms) + ' '+ str(wplanes))
# phase only calibrate
if model_facet:
skymodel = model_facet
logging.info('Using config model ' + model_facet +
" skip model from image 0")
else:
skymodel = imout + '.skymodel'
parset = create_scalarphase_parset(cellsizetime_p, TEC, clock, group, FFT,
uvrange)
logging.debug("Run BBS (pre-image 1) with skymodel: {}".format(skymodel))
runbbs(mslist, skymodel, parset, 'instrument', False, TEC, clusterdesc,
dbserver, dbuser, dbname)
#NOTE WORK FROM MODEL_DATA (contains correct phase data from 10SB calibration)
######################################
### MAKE IMAGE 1 ###
logging.info('Make image 1')
imout, mask = make_image(mslist, cluster, '1', 15, 15, nterms, atrous_do,
imsize, region, ncores, SCRIPTPATH)
####################
### CALIBRATE WITH BBS PHASE ONLY 2 ###
# create skymodel for BBS
logging.debug("".format())
run(SCRIPTPATH + '/casapy2bbs.py -m ' + mask + ' ' + '-t ' + str(nterms) +
' ' + imout + '.model ' + imout + '.skymodel')
if FFT:
run('casapy --nogui -c '+SCRIPTPATH+'/ft_v2.py ' + msinputlist + ' ' + imout+'.model' \
+ ' ' + str(nterms) + ' '+ str(wplanes))
# phase only calibrate
skymodel = imout + '.skymodel'
parset = create_scalarphase_parset(cellsizetime_p, TEC, clock, group, FFT,
uvrange)
logging.debug("Run BBS (pre-image 2) with skymodel: {}".format(skymodel))
runbbs(
mslist, skymodel, parset, 'instrument', False, TEC, clusterdesc,
dbserver, dbuser, dbname
) #NOTE WORK FROM MODEL_DATA (contains correct phase data from 10SB calibration)
######################################
### MAKE IMAGE 2 ###
logging.info('Make image 2')
imout, mask = make_image(mslist, cluster, '2', 15, 15, nterms, atrous_do,
imsize, region, ncores, SCRIPTPATH)
####################
### CALIBRATE WITH BBS PHASE+AMP 1 ###
run(SCRIPTPATH + '/casapy2bbs.py -m ' + mask + ' ' + '-t ' + str(nterms) +
' ' + imout + '.model ' + imout + '.skymodel')
if FFT:
run('casapy --nogui -c '+SCRIPTPATH+'/ft_v2.py ' + msinputlist + ' ' + imout+'.model' \
+ ' ' + str(nterms) + ' '+ str(wplanes))
skymodel = imout + '.skymodel'
parset = create_scalarphase_parset_p(cellsizetime_p, TEC, clock, group,
FFT, uvrange)
# solve +apply phases
logging.debug("Run BBS (pre-image 3) with skymodel: {}".format(skymodel))
runbbs(mslist, skymodel, parset, 'instrument_phase0', False, TEC,
clusterdesc, dbserver, dbuser, dbname)
# solve amps
parmdb = 'instrument_amps0'
parset = create_amponly_parset(cellsizetime_a, FFT, uvrange)
runbbs(mslist, skymodel, parset, parmdb, False, False, clusterdesc,
dbserver, dbuser, dbname)
for ms in mslist:
# remove outliers from the solutions
if phasors:
run('python ' + SCRIPTPATH + '/smoothcal_rx42.py ' + ms + ' ' +
ms + '/' + parmdb + ' ' + ms + '/' + parmdb + '_smoothed' +
' > ' + ms + '_' + parmdb + '_smoothed.log')
else:
run('python ' + SCRIPTPATH + '/smoothcal_a2256_nophasors.py ' +
ms + ' ' + ms + '/' + parmdb + ' ' + ms + '/' + parmdb +
'_smoothed' + ' > ' + ms + '_' + parmdb + '_smoothed.log')
# apply amps
if smooth:
runbbs(mslist, skymodel, SCRIPTPATH + '/apply_amplitudeonly.parset',
parmdb + '_smoothed', True, False, clusterdesc, dbserver,
dbuser, dbname)
else:
runbbs(mslist, skymodel, SCRIPTPATH + '/apply_amplitudeonly.parset',
parmdb, True, False, clusterdesc, dbserver, dbuser, dbname)
### MAKE IMAGE 3 ###
logging.info('Make image 3')
imout, mask = make_image(mslist, cluster, '3', 10, 10, nterms, atrous_do,
imsize, region, ncores, SCRIPTPATH)
####################
# LOOP
####################
rms, dynamicrange = find_imagenoise(imout + '.image')
while (((dynamicrange / factor) > dynamicrange_old)
or ((rms * factor) < rms_old)):
logging.info('Starting selfcal loop {}'.format(im_count))
#### CALIBRATE BBS PHASE+AMP 2 (LOOP) ###
# make model
run(SCRIPTPATH + '/casapy2bbs.py -m ' + mask + ' ' + '-t ' +
str(nterms) + ' ' + imout + '.model ' + imout + '.skymodel')
if FFT:
run('casapy --nogui -c '+SCRIPTPATH+'/ft_v2.py ' + msinputlist + ' ' + imout+'.model' \
+ ' ' + str(nterms) + ' '+ str(wplanes))
#parmdb keep from previous step
skymodel = imout + '.skymodel'
# reset the phases from instrument_amps0 to zero to prevent large phase corrections from incorrect AP solve
# FIXME: Is this used?
if phasezero:
inputparmdb = parmdb + '_smoothed'
outputparmdb = parmdb + '_smoothed_phasezero'
for ms in mslist:
run('python ' + SCRIPTPATH + '/setphasezero.py ' + ms + ' ' +
ms + '/' + inputparmdb + ' ' + ms + '/' + outputparmdb)
else:
outputparmdb = parmdb + '_smoothed'
# phase only cal
skymodel = imout + '.skymodel'
parset = create_scalarphase_parset_p(cellsizetime_p, TEC, clock, group,
FFT, uvrange)
logging.debug("Run BBS (pre-image {}) with skymodel: {}".format(
im_count, skymodel))
runbbs(mslist, skymodel, parset, 'instrument_phase1', False, TEC,
clusterdesc, dbserver, dbuser, dbname)
# solve amps
parmdb = 'instrument_amps1'
parset = create_amponly_parset(cellsizetime_a, FFT, uvrange)
runbbs(mslist, skymodel, parset, parmdb, False, False, clusterdesc,
dbserver, dbuser, dbname)
for ms in mslist:
# remove outliers from the solutions
if phasors:
run('python ' + SCRIPTPATH + '/smoothcal_rx42.py ' + ms + ' ' +
ms + '/' + parmdb + ' ' + ms + '/' + parmdb + '_smoothed' +
' > ' + ms + '_' + parmdb + '_smoothed.log')
else:
run('python ' + SCRIPTPATH + '/smoothcal_a2256_nophasors.py ' +
ms + ' ' + ms + '/' + parmdb + ' ' + ms + '/' + parmdb +
'_smoothed' + ' > ' + ms + '_' + parmdb + '_smoothed.log')
# apply amps
if smooth:
runbbs(mslist, skymodel,
SCRIPTPATH + '/apply_amplitudeonly.parset',
parmdb + '_smoothed', True, False, clusterdesc, dbserver,
dbuser, dbname)
else:
runbbs(mslist, skymodel,
SCRIPTPATH + '/apply_amplitudeonly.parset', parmdb, True,
False, clusterdesc, dbserver, dbuser, dbname)
### MAKE IMAGE #N ###
logging.info('Make image {}'.format(im_count))
# Do not use the initial mask in the final cycles
if im_count >= empty_mask_cycle:
region_im = "empty"
else:
region_im = region
imout, mask = make_image(mslist, cluster, str(im_count), 10, 10,
nterms, atrous_do, imsize, region_im, ncores,
SCRIPTPATH)
## Prepare the next iteration
im_count += 1
# save previous values to compare with
rms_old = rms
dynamicrange_old = dynamicrange
if nterms < 2:
rms, dynamicrange = find_imagenoise(imout + '.image')
else:
rms, dynamicrange = find_imagenoise(imout + '.image.tt0')
logging.info('IMAGE STATISTICS {}, {}'.format(rms, dynamicrange))
if im_count < number_forced_selfcalcycles:
rms_old = 1.e9 # bad values to start with
dynamicrange_old = 1.
logging.debug(
"Count below the number of forced selfcal cycles. Force new loop."
)
if im_count >= max_selfcalcycles:
logging.info(
"Maximum number of cycles ({}) reached. Leaving selfcal loop.".
format(max_selfcalcycles))
break
### CREATE FINAL MODEL ###
logging.info('Create final model')
skymodelf = 'im_cluster' + cluster + '.final.skymodel'
run(SCRIPTPATH + '/casapy2bbs.py -m ' + mask + ' ' + '-t ' + str(nterms) +
' ' + imout + '.model ' + skymodelf)
if FFT:
run('casapy --nogui -c '+SCRIPTPATH+'/ft_v2.py ' + msinputlist + ' ' + imout+'.model' \
+ ' ' + str(nterms) + ' '+ str(wplanes))
### CREATED MERGED PARMDB SCALARPHASE+AMPS ###
### INCLUDES SPLINE INTERPOLARION OF AMPS ###
if merge_parmdb:
logging.info('Merge parmdb')
if phasors:
dummyparset = SCRIPTPATH + '/scalarphase+amp.parset'
else:
if TEC:
dummyparset = SCRIPTPATH + '/scalarphase+ap+TEC.parset'
else:
dummyparset = SCRIPTPATH + '/scalarphase+ap.parset'
if TEC:
if clock:
dummyparmdb = 'instrument_template_TECclock'
else:
dummyparmdb = 'instrument_template_Gain_TEC_CSphase'
#if not os.path.isdir(dummyparmdb):
#runbbs([mslist[0]], skymodel,dummyparset, dummyparmdb, True, False)
# TO SPEED THINGS UP, hard coded for BOOTES - i.e. the above has already been run
for ms in mslist:
os.system('rm -rf ' + ms + '/' + dummyparmdb)
os.system('cp -r ' + dummyparmdb + ' ' + ms +
'/instrument_template')
if smooth:
parmdb_a = 'instrument_amps1_smoothed' # last/best ampplitude(+phase) parmdb
else:
parmdb_a = 'instrument_amps1' # last/best ampplitude(+phase) parmdb
parmdb_p = 'instrument_phase1' # last/best CommonScalarPhase parmdb
parmdbout = 'instrument_merged'
#reset in case of instrument_template_TECclock
dummyparmdb = 'instrument_template'
for ms in mslist:
create_merged_parmdb(ms, ms + '/' + parmdb_a, ms + '/' + parmdb_p,
ms + '/' + dummyparmdb, ms + '/' + parmdbout,
cellsizetime_a, cellsizetime_p)