def gaincalXY(vis=None, caltable=None, pols='XXYY', msfileXY=None, gaintableXY=None, **kwargs):
from gaincal_cli import gaincal_cli as gaincal
if pols == 'XXYY':
pols = 'XX,YY'
pols_ = pols.split(',')
rm_msfileXY = False
if msfileXY is None:
rm_msfileXY = True
msfileXY = {}
for pol in pols_:
msfileXY[pol] = '.'.join([vis, pol])
if os.path.exists(msfileXY[pol]):
os.system('rm -rf {}'.format(msfileXY[pol]))
splitX(vis=vis, outputvis=msfileXY[pol], correlation=pol, datacolumn='data', datacolumn2='MODEL_DATA')
if gaintableXY is not None:
if 'gaintable' in kwargs.keys():
kwargs.pop('gaintable')
caltbXY = []
for pol in pols_:
caltb_ = '.'.join([caltable, pol])
if gaintableXY is not None:
kwargs['gaintable'] = gaintableXY[pol]
gaincal(vis=msfileXY[pol], caltable=caltb_, **kwargs)
caltbXY.append(caltb_)
concat_slftb(caltbXY, caltable)
if rm_msfileXY:
for k, v in msfileXY.iteritems():
os.system('rm -rf {}'.format(v))
return
def mygaincal(vis, name_regex=name_regex, caltable=None, **kwargs):
if isinstance(vis, list) or isinstance(vis, tuple):
for onems in vis:
matches = name_regex.search(onems)
assert matches.groups() is not None
mon, day, hr = matches.groups()
caltable_ = "{0}_{1}_T{2}_{3}".format(mon, day, hr, caltable)
if not os.path.exists(caltable_):
gaincal(vis=onems, caltable=caltable_, **kwargs)
else:
casalog.post("FAILURE: bad vis input {0}".format(vis),
origin='mygaincal',
priority='SEVERE')
raise ValueError
wprojplanes=64,
niter=10000,
threshold='3mJy',
scales=[0,3,9],
robust=0.5,
savemodel='modelcolumn',
mask=mask,
)
caltable = '18A-229_Q_concatenated_cal_iter1_wterms.cal'
if not os.path.exists(caltable):
gaincal(vis=cont_vis,
caltable=caltable,
field='Sgr B2 N Q,Sgr B2 NM Q,Sgr B2 MS Q',
calmode='p',
refant='',
solint='60s',
minsnr=5,
#uvrange='0~2000klambda',
#minblperant=3,
)
# do a purely diagnostic ampcal
gaincal(vis=cont_vis,
caltable='18A-229_Q_concatenated_cal_iter1_ampcal_diagnostic_wterms.cal',
gaintype='G',
combine='spw,scan,field',
solint='inf',
calmode='a',
solnorm=True)
def mk_qlook_image(trange,
doimport=False,
docalib=False,
ncpu=10,
twidth=12,
stokes=None,
antenna='0~12',
lowcutoff_freq=3.7,
imagedir=None,
spws=['1~5', '6~10', '11~15', '16~25'],
toTb=True,
overwrite=True,
doslfcal=False,
verbose=False):
'''
trange: can be 1) a single Time() object: use the entire day
2) a range of Time(), e.g., Time(['2017-08-01 00:00','2017-08-01 23:00'])
3) a single or a list of UDBms file(s)
4) None -- use current date Time.now()
'''
antenna0 = antenna
if type(trange) == Time:
mslist = trange2ms(trange=trange, doimport=doimport)
vis = mslist['ms']
tsts = [l.to_datetime() for l in mslist['tstlist']]
if type(trange) == str:
try:
date = Time(trange)
mslist = trange2ms(trange=trange, doimport=doimport)
vis = mslist['ms']
tsts = [l.to_datetime() for l in mslist['tstlist']]
except:
vis = [trange]
tsts = []
for v in vis:
tb.open(v + '/OBSERVATION')
tsts.append(
Time(tb.getcell('TIME_RANGE')[0] / 24 / 3600,
format='mjd').datetime)
tb.close()
subdir = [tst.strftime("%Y/%m/%d/") for tst in tsts]
for idx, f in enumerate(vis):
if f[-1] == '/':
vis[idx] = f[:-1]
if not stokes:
stokes = 'XX'
if not imagedir:
imagedir = './'
imres = {
'Succeeded': [],
'BeginTime': [],
'EndTime': [],
'ImageName': [],
'Spw': [],
'Vis': [],
'Synoptic': {
'Succeeded': [],
'BeginTime': [],
'EndTime': [],
'ImageName': [],
'Spw': [],
'Vis': []
}
}
for n, msfile in enumerate(vis):
msfilebs = os.path.basename(msfile)
imdir = imagedir + subdir[n]
if not os.path.exists(imdir):
os.makedirs(imdir)
if doslfcal:
slfcalms = './' + msfilebs + '.xx'
split(msfile,
outputvis=slfcalms,
datacolumn='corrected',
correlation='XX')
cfreqs = getspwfreq(msfile)
for spw in spws:
antenna = antenna0
if spw == '':
continue
spwran = [s.zfill(2) for s in spw.split('~')]
freqran = [cfreqs[int(s)] for s in spw.split('~')]
cfreq = np.mean(freqran)
bmsz = max(150. / cfreq, 20.)
uvrange = '<10klambda'
if doslfcal:
slfcal_img = './' + msfilebs + '.slf.spw' + spw.replace(
'~', '-') + '.slfimg'
slfcal_tb = './' + msfilebs + '.slf.spw' + spw.replace(
'~', '-') + '.slftb'
try:
clean(vis=slfcalms,
antenna=antenna,
imagename=slfcal_img,
spw=spw,
mode='mfs',
timerange='',
imagermode='csclean',
psfmode='clark',
imsize=[512, 512],
cell=['5arcsec'],
niter=100,
gain=0.05,
stokes='I',
weighting='natural',
restoringbeam=[str(bmsz) + 'arcsec'],
pbcor=False,
interactive=False,
usescratch=True)
except:
print('error in cleaning spw: ' + spw)
break
gaincal(vis=slfcalms,
refant='0',
antenna=antenna,
caltable=slfcal_tb,
spw=spw,
uvrange='',
gaintable=[],
selectdata=True,
timerange='',
solint='600s',
gaintype='G',
calmode='p',
combine='',
minblperant=3,
minsnr=2,
append=False)
if not os.path.exists(slfcal_tb):
print('No solution found in spw: ' + spw)
break
else:
clearcal(slfcalms)
delmod(slfcalms)
applycal(vis=slfcalms,
gaintable=[slfcal_tb],
spw=spw,
selectdata=True,
antenna=antenna,
interp='nearest',
flagbackup=False,
applymode='calonly',
calwt=False)
msfile = slfcalms
imsize = 512
cell = ['5arcsec']
if len(spwran) == 2:
spwstr = spwran[0] + '~' + spwran[1]
else:
spwstr = spwran[0]
restoringbeam = ['{0:.1f}arcsec'.format(bmsz)]
imagesuffix = '.spw' + spwstr.replace('~', '-')
if cfreq > 10.:
antenna = antenna + ';!0&1;!0&2' # deselect the shortest baselines
# else:
# antenna = antenna + ';!0&1' # deselect the shortest baselines
res = ptclean3(vis=msfile,
imageprefix=imdir,
imagesuffix=imagesuffix,
twidth=twidth,
uvrange=uvrange,
spw=spw,
ncpu=ncpu,
niter=1000,
gain=0.05,
antenna=antenna,
imsize=imsize,
cell=cell,
stokes=stokes,
doreg=True,
usephacenter=False,
overwrite=overwrite,
toTb=toTb,
restoringbeam=restoringbeam,
specmode="mfs",
deconvolver="hogbom",
datacolumn='data',
pbcor=True)
if res:
imres['Succeeded'] += res['Succeeded']
imres['BeginTime'] += res['BeginTime']
imres['EndTime'] += res['EndTime']
imres['ImageName'] += res['ImageName']
imres['Spw'] += [spwstr] * len(res['ImageName'])
imres['Vis'] += [msfile] * len(res['ImageName'])
else:
continue
if len(vis) == 1:
# produce the band-by-band whole-day images
ms.open(msfile)
ms.selectinit()
timfreq = ms.getdata(['time', 'axis_info'], ifraxis=True)
tim = timfreq['time']
ms.close()
cfreqs = getspwfreq(msfile)
imdir = imagedir + subdir[0]
if not os.path.exists(imdir):
os.makedirs(imdir)
for spw in spws:
antenna = antenna0
if spw == '':
spw = '{:d}~{:d}'.format(
next(x[0] for x in enumerate(cfreqs)
if x[1] > lowcutoff_freq),
len(cfreqs) - 1)
spwran = [s.zfill(2) for s in spw.split('~')]
freqran = [cfreqs[int(s)] for s in spw.split('~')]
cfreq = np.mean(freqran)
bmsz = max(150. / cfreq, 20.)
uvrange = ''
imsize = 512
cell = ['5arcsec']
if len(spwran) == 2:
spwstr = spwran[0] + '~' + spwran[1]
else:
spwstr = spwran[0]
restoringbeam = ['{0:.1f}arcsec'.format(bmsz)]
imagesuffix = '.synoptic.spw' + spwstr.replace('~', '-')
antenna = antenna + ';!0&1' # deselect the shortest baselines
res = ptclean3(vis=msfile,
imageprefix=imdir,
imagesuffix=imagesuffix,
twidth=len(tim),
uvrange=uvrange,
spw=spw,
ncpu=1,
niter=0,
gain=0.05,
antenna=antenna,
imsize=imsize,
cell=cell,
stokes=stokes,
doreg=True,
usephacenter=False,
overwrite=overwrite,
toTb=toTb,
restoringbeam=restoringbeam,
specmode="mfs",
deconvolver="hogbom",
datacolumn='data',
pbcor=True)
if res:
imres['Synoptic']['Succeeded'] += res['Succeeded']
imres['Synoptic']['BeginTime'] += res['BeginTime']
imres['Synoptic']['EndTime'] += res['EndTime']
imres['Synoptic']['ImageName'] += res['ImageName']
imres['Synoptic']['Spw'] += [spwstr] * len(res['ImageName'])
imres['Synoptic']['Vis'] += [msfile] * len(res['ImageName'])
else:
continue
# save it for debugging purposes
np.savez('imres.npz', imres=imres)
return imres
for selfcaliter in selfcalpars.keys():
logprint("Gaincal iteration {0}".format(selfcaliter),
origin='contim_selfcal')
# iteration #1 of phase-only self-calibration
caltype = 'amp' if 'a' in selfcalpars[selfcaliter][
'calmode'] else 'phase'
caltable = '{0}_{1}_{2}{3}_{4}.cal'.format(
basename, arrayname, caltype, selfcaliter,
selfcalpars[selfcaliter]['solint'])
if not os.path.exists(caltable):
#check_model_is_populated(selfcal_ms)
gaincal(vis=selfcal_ms,
caltable=caltable,
gaintable=cals,
**selfcalpars[selfcaliter])
else:
logprint("Skipping existing caltable {0}".format(caltable),
origin='contim_selfcal')
cals.append(caltable)
# set up the imaging parameters for this round, allowing for a flexible definition
# with either, e.g. {'niter': 1000} or {'niter': {1:1000, 2:100000, 3:999999}} etc
impars_thisiter = copy.copy(impars)
if 'maskname' in impars_thisiter:
if selfcaliter in impars_thisiter['maskname']:
maskname = impars_thisiter['maskname'][selfcaliter]
else:
logprint("Self cal iteration {0} has no associated mask. "
def feature_slfcal(vis, niter=200, spws=['0~1', '2~5', '6~10', '11~20', '21~30', '31~49'], slfcaltbdir='./',
bright=None, pols='XX'):
''' Uses images from disk-selfcaled data as model for further self-calibration of outer antennas.
This is only a good idea if there are bright active regions that provide strong signal on the
londer baselines.
'''
trange = ant_trange(vis)
if bright is None:
bright = [True] * len(spws)
# Insert model into ms and do "inf" gaincal, appending to table each subsequent time
if os.path.exists('images_init'):
os.system('rm -rf images_init')
os.system('mv images images_init')
clearcal(vis, addmodel=True)
fd_images(vis, cleanup=False, niter=niter, spws=spws, bright=bright) # Does shallow clean for selfcal purposes
tdate = mstl.get_trange(vis)[0].datetime.strftime('%Y%m%d')
caltb = os.path.join(slfcaltbdir, tdate + '_d1.pha')
if os.path.exists(caltb):
os.system('rm -rf {}*'.format(caltb))
for s, sp in enumerate(spws):
if bright[s]:
spwstr = '-'.join(['{:02d}'.format(int(sp_)) for sp_ in sp.split('~')])
imname = "images/briggs" + spwstr + '.model'
if sp == '31~49':
# The high-band image is only made to band 43, so adjust the name
imname = 'images/briggs31-43.model'
ft(vis=vis, spw=sp, model=imname, usescratch=True, incremental=True)
if pols == 'XXYY':
mstl.modeltransfer(vis, spw=sp)
if pols == 'XXYY':
mstl.gaincalXY(vis=vis, caltable=caltb, pols=pols, selectdata=True, timerange=trange, uvrange='>1.5Klambda',
combine="scan", antenna='0~12&0~12', refant='0', refantmode="strict", solint='inf', gaintype='G',
minsnr=1.0, calmode='p', append=False)
else:
gaincal(vis=vis, caltable=caltb, selectdata=True, timerange=trange, uvrange='>1.5Klambda',
combine="scan", antenna='0~12&0~12', refant='0', refantmode="strict", solint='inf', gaintype='G',
minsnr=1.0, calmode='p', append=False)
# Apply the corrections to the data and split to a new ms
applycal(vis=vis, selectdata=True, antenna="0~12", gaintable=caltb, interp="linear", calwt=False,
applymode="calonly")
vis1 = 'dslf1_' + vis
if os.path.exists(vis1):
os.system('rm -rf {}'.format(vis1))
split(vis, outputvis=vis1, datacolumn="corrected")
caltb = os.path.join(slfcaltbdir, tdate + '_d2.pha')
if os.path.exists(caltb):
os.system('rm -rf {}*'.format(caltb))
# Move the existing images directory so that a new one will be created
if os.path.exists('images_ftcal_rnd1'):
os.system('rm -rf images_ftcal_rnd1')
# shutil.move('images', 'old_images2')
os.system('mv images images_ftcal_rnd1')
# Make new model images for another round of selfcal
fd_images(vis1, cleanup=False, niter=niter, spws=spws, bright=bright)
for s, sp in enumerate(spws):
if bright[s]:
spwstr = '-'.join(['{:02d}'.format(int(sp_)) for sp_ in sp.split('~')])
imname = "images/briggs" + spwstr + '.model'
if sp == '31~49':
# The high-band image is only made to band 43, so adjust the name
imname = 'images/briggs31-43.model'
ft(vis=vis1, spw=sp, model=imname, usescratch=True)
if pols == 'XXYY':
mstl.modeltransfer(vis1, spw=sp)
if pols == 'XXYY':
mstl.gaincalXY(vis=vis1, caltable=caltb, pols=pols, selectdata=True, timerange=trange, uvrange='>1.5Klambda',
combine="scan", antenna='0~12&0~12', refant='0', solint='10min', refantmode="strict",
gaintype='G', minsnr=1.0, calmode='p', append=False)
else:
gaincal(vis=vis1, caltable=caltb, selectdata=True, timerange=trange, uvrange='>1.5Klambda',
combine="scan", antenna='0~12&0~12', refant='0', solint='10min', refantmode="strict", gaintype='G',
minsnr=1.0, calmode='p', append=False)
# Apply the corrections to the data and split to a new ms
applycal(vis=vis1, selectdata=True, antenna="0~12", gaintable=caltb, interp="linear", calwt=False,
applymode="calonly")
vis2 = 'dslf2_' + vis
if os.path.exists(vis2):
os.system('rm -rf {}'.format(vis2))
mstl.splitX(vis1, outputvis=vis2, datacolumn="corrected", datacolumn2="model_data")
if os.path.exists('images_ftcal_rnd2'):
os.system('rm -rf images_ftcal_rnd2')
os.system('mv images images_ftcal_rnd2')
return vis2
def disk_slfcal(vis, slfcaltbdir='./', active=False, clearcache=False, pols='XX'):
''' Starting with the name of a calibrated ms (vis, which must have 'UDByyyymmdd' in the name)
add a model disk based on the solar disk size for that date and perform multiple selfcal
adjustments (two phase and one amplitude), and write out a final selfcaled database with
the disk subtracted. Returns the name of the final database.
'''
trange = ant_trange(vis)
if vis.endswith('/'):
vis = vis[:-1]
# Use vis name to determine date, and hence number of bands
spw2band = np.array([0, 1] + range(4, 52))
defaultfreq = 1.1 + 0.325 * (spw2band + 0.5)
# Calculate the center frequency of each spectral window
if mstl.get_trange(vis)[0].mjd > 58536:
# After 2019 Feb 22, the band numbers changed to 1-52, and spw from 0-49
nbands = 52
freq = defaultfreq
else:
# Before 2019 Feb 22, the band numbers were 1-34, and spw from 0-30
nbands = 34
freq = 1.419 + np.arange(nbands) / 2.
caltbs = []
slashdate = trange[:10]
# Verify that the vis is not in the current working directory
'''
if os.getcwd() == os.path.dirname(vis):
print('Cannot copy vis file onto itself.')
print('Please change to a different working directory')
return None
# Copy original ms to local directory
if os.path.exists(os.path.basename(vis)):
shutil.rmtree(os.path.basename(vis))
print('Copy {} to working directory {}.'.format(vis, os.getcwd()))
shutil.copytree(vis, os.path.basename(vis))
vis = os.path.basename(vis)
'''
if not active:
clearcal(vis)
flagmanager(vis, mode='save', versionname='with-RFI-or-BURSTS')
## automaticaly flag any high amplitudes from flares or RFI
flagdata(vis=vis, mode="tfcrop", spw='', action='apply', display='',
timecutoff=3.0, freqcutoff=2.0, maxnpieces=2, flagbackup=False)
flagmanager(vis, mode='save', versionname='without-RFI-or-BURSTS')
dsize, fdens = calc_diskmodel(slashdate, nbands, freq, defaultfreq)
diskxmlfile = vis + '.SOLDISK.xml'
# Insert the disk model (msfile is the same as vis, and will be used as the "original" vis file name)
msfile, diskim = insertdiskmodel(vis, dsize=dsize, fdens=fdens, xmlfile=diskxmlfile, active=active)
if pols == 'XXYY':
caltbs_ = {'XX': [], 'YY': []}
pols_ = ['XX', 'YY']
msfileXY = {}
for pol in pols_:
msfileXY[pol] = '.'.join([msfile, pol])
if os.path.exists(msfileXY[pol]):
os.system('rm -rf {}'.format(msfileXY[pol]))
mstl.splitX(vis=msfile, outputvis=msfileXY[pol], correlation=pol, datacolumn='data',
datacolumn2='MODEL_DATA')
tdate = mstl.get_trange(msfile)[0].datetime.strftime('%Y%m%d')
caltb = os.path.join(slfcaltbdir, tdate + '_1.pha')
if os.path.exists(caltb):
os.system('rm -rf {}*'.format(caltb))
if pols == 'XXYY':
mstl.gaincalXY(vis=msfile, caltable=caltb, pols=pols, msfileXY=msfileXY, selectdata=True, uvrange="",
antenna="0~12&0~12", solint="inf",
combine="scan", refant="0", refantmode="strict", minsnr=1.0, gaintype="G", calmode="p",
append=False)
for pol in pols_:
caltb_ = '.'.join([caltb, pol])
caltbs_[pol].append(caltb_)
else:
gaincal(vis=msfile, caltable=caltb, selectdata=True, uvrange="", antenna="0~12&0~12", solint="inf",
combine="scan", refant="0", refantmode="strict", minsnr=1.0, gaintype="G", calmode="p", append=False)
caltbs.append(caltb)
caltb = os.path.join(slfcaltbdir, tdate + '_2.pha')
if os.path.exists(caltb):
os.system('rm -rf {}*'.format(caltb))
# Second round of phase selfcal on the disk using solution interval "1min"
if pols == 'XXYY':
mstl.gaincalXY(vis=msfile, caltable=caltb, pols=pols, msfileXY=msfileXY, gaintableXY=caltbs_,
selectdata=True, uvrange="", antenna="0~12&0~12",
solint="10min",
combine="scan", interp="linear",
refant="0", refantmode="strict", minsnr=1.0, gaintype="G", calmode="p", append=False)
for pol in pols_:
caltb_ = '.'.join([caltb, pol])
caltbs_[pol].append(caltb_)
else:
gaincal(vis=msfile, caltable=caltb, selectdata=True, uvrange="", antenna="0~12&0~12", solint="10min",
combine="scan", gaintable=caltbs, interp="linear",
refant="0", refantmode="strict", minsnr=1.0, gaintype="G", calmode="p", append=False)
caltbs.append(caltb)
caltb = os.path.join(slfcaltbdir, tdate + '_3.amp')
if os.path.exists(caltb):
os.system('rm -rf {}*'.format(caltb))
# Final round of amplitude selfcal with 1-h solution interval (restrict to 16-24 UT)
if pols == 'XXYY':
mstl.gaincalXY(vis=msfile, caltable=caltb, pols=pols, msfileXY=msfileXY, gaintableXY=caltbs_,
selectdata=True, uvrange="", antenna="0~12&0~12",
timerange=trange, interp="linear",
solint="60min", combine="scan", refant="10", refantmode="flex", minsnr=1.0, gaintype="G",
calmode="a",
append=False)
for pol in pols_:
caltb_ = '.'.join([caltb, pol])
caltbs_[pol].append(caltb_)
else:
gaincal(vis=msfile, caltable=caltb, selectdata=True, uvrange="", antenna="0~12&0~12",
timerange=trange, gaintable=caltbs, interp="linear",
solint="60min", combine="scan", refant="10", refantmode="flex", minsnr=1.0, gaintype="G", calmode="a",
append=False)
mstl.flagcaltboutliers(caltb, limit=[0.125, 8.0])
# mstl.flagcaltboutliers(caltb, limit=[0.5, 2.0])
caltbs.append(caltb)
# Split out corrected data and model and do uvsub
vis2 = 'slf3_' + msfile
if os.path.exists(vis2):
os.system('rm -rf {}'.format(vis2))
if os.path.exists(vis2 + '.flagversions'):
os.system('rm -rf {}'.format(vis2 + '.flagversions'))
# flagmanager(msfile, mode='restore', versionname='with-RFI-or-BURSTS')
clearcal(msfile)
applycal(vis=msfile, selectdata=True, antenna="0~12", gaintable=caltbs, interp="linear", calwt=False,
applymode="calonly")
split(msfile, outputvis=vis2, datacolumn="corrected")
for sp, dkim in tqdm(enumerate(diskim), desc='Inserting disk model', ascii=True):
ft(vis=vis2, spw=str(sp), field='', model=str(dkim), nterms=1,
reffreq="", complist="", incremental=False, usescratch=True)
# mstl.modeltransfer(msfile, spw='{}'.format(sp))
uvsub(vis=vis2, reverse=False)
# Final split to
final = 'final_' + msfile
if os.path.exists(final):
os.system('rm -rf {}'.format(final))
if os.path.exists(final + '.flagversions'):
os.system('rm -rf {}'.format(final + '.flagversions'))
split(vis2, outputvis=final, datacolumn='corrected')
os.system('mv {} {}'.format(msfile + '.flagversions', final + '.flagversions'))
# Remove the interim ms files
if clearcache:
if os.path.exists(msfile):
os.system('rm -rf {}'.format(msfile))
if os.path.exists(msfile + '.flagversions'):
os.system('rm -rf {}'.format(msfile + '.flagversions'))
if os.path.exists(vis2):
os.system('rm -rf {}'.format(vis2))
if os.path.exists(vis2 + '.flagversions'):
os.system('rm -rf {}'.format(vis2 + '.flagversions'))
# Return the name of the selfcaled ms
return final, diskxmlfile
caltable = '{0}_sgrb2_selfcal_phase_30ssolint.cal'.format(name)
if not os.path.exists(caltable):
myprint("Caltable {0} does not exist, making it".format(caltable))
ft(vis=cont_ms,
field='Sgr B2 N Q,Sgr B2 NM Q,Sgr B2 MS Q',
spw='',
model=model)
gaincal(
vis=cont_ms,
caltable=caltable,
field='Sgr B2 N Q,Sgr B2 NM Q,Sgr B2 MS Q',
calmode='p',
refant='',
solint='30s',
#uvrange='0~2000klambda',
minblperant=3,
)
# these tables are JUST FOR DIAGNOSTICS: don't use them
ampcaltable = '{0}_sgrb2_selfcal_amp_INFsolint_dontuse.cal'.format(
name)
gaincal(
vis=cont_ms,
caltable=ampcaltable,
field='Sgr B2 N Q,Sgr B2 NM Q,Sgr B2 MS Q',
calmode='a',
refant='',
solint='inf',
specmode='cube',
weighting='briggs',
pblimit=0.2,
interactive=False,
outframe='LSRK',
datacolumn='data',
robust=0.5,
)
makefits(imagename)
caltable1 = 'NH322_selfcal_iter1_phase30s.cal'
gaincal(
vis=merged_ms,
caltable=caltable1,
field=field,
calmode='p',
refant='',
solint='30s',
minblperant=3,
spw=selfcal_spw,
)
assert os.path.exists(caltable1)
applycal(flagbackup=False,
gainfield=[],
interp=[],
gaintable=[caltable1],
calwt=[False],
vis=merged_ms,
applymode='calonly',
antenna='*&*',
spwmap=[],
modelim = fits.open(imagename + ".model.tt0.fits")
mx = modelim[0].data.max()
logprint("max value in model: {0}".format(mx))
assert mx > 0
caltable = '{2}_{1}_{0}.cal'.format(field_nospace, iternum, caltype)
rmtables([caltable])
if 'phase' in caltype or 'amp' in caltype:
gaincal(
vis=selfcal_vis,
caltable=caltable,
solint=solint,
combine=combine,
gaintype='G',
# use all fields field=field,
calmode=calmode,
gaintable=caltables,
minsnr=1.5,
spwmap=[[0] * nspws if calinfo[ii]['combine'] == 'spw' else []
for ii in range(len(caltables))],
interp='linear,linear',
solnorm=True)
if 'amp' in caltype:
# avoid extreme outliers: assume anything going more than 2x in either direction is wrong
flagdata(caltable,
mode='clip',
clipminmax=[0.5, 2.0],
datacolumn='CPARAM')
elif 'bandpass' in caltype:
bandpass(
vis=selfcal_vis,
def mk_qlook_image(trange, doimport=False, docalib=False, ncpu=10, twidth=12, stokes=None, antenna='0~12',
#imagedir=None, spws=['1~3','4~6','7~9','10~13','14~18','19~28'],verbose=False):
imagedir=None, spws=['1~5','6~10','11~15','16~25'], toTb=True, overwrite=True,
doslfcal=False, verbose=False):
'''
trange: can be 1) a single Time() object: use the entire day
2) a range of Time(), e.g., Time(['2017-08-01 00:00','2017-08-01 23:00'])
3) a single or a list of UDBms file(s)
4) None -- use current date Time.now()
'''
if type(trange) == Time:
mslist = trange2ms(trange=trange, doimport=doimport)
vis = mslist['ms']
tsts = [l.to_datetime() for l in mslist['tstlist']]
subdir = [tst.strftime("%Y/%m/%d/") for tst in tsts]
if type(trange) == str:
try:
date = Time(trange)
mslist = trange2ms(trange=trange, doimport=doimport)
vis = mslist['ms']
except:
vis = [trange]
subdir = ['/']
for idx, f in enumerate(vis):
if f[-1] == '/':
vis[idx] = f[:-1]
if not stokes:
stokes = 'XX'
if not imagedir:
imagedir='./'
imres = {'Succeeded': [], 'BeginTime': [], 'EndTime': [], 'ImageName': [], 'Spw': [], 'Vis': []}
for n, msfile in enumerate(vis):
msfilebs=os.path.basename(msfile)
imdir = imagedir + subdir[n]
if not os.path.exists(imdir):
os.makedirs(imdir)
if doslfcal:
slfcalms = './'+msfilebs+'.xx'
split(msfile,outputvis=slfcalms,datacolumn='corrected',correlation='XX')
for spw in spws:
spwran = [s.zfill(2) for s in spw.split('~')]
freqran = [int(s)*0.5+2.9 for s in spw.split('~')]
cfreq=np.mean(freqran)
bmsz=max(150./cfreq,20.)
uvrange='<10klambda'
if doslfcal:
slfcal_img = './'+msfilebs+'.slf.spw'+spw.replace('~','-')+'.slfimg'
slfcal_tb = './'+msfilebs+'.slf.spw'+spw.replace('~','-')+'.slftb'
try:
clean(vis=slfcalms,
antenna=antenna,
imagename=slfcal_img,
spw=spw,
mode='mfs',
timerange='',
imagermode='csclean',
psfmode='clark',
imsize=[512,512],
cell=['5arcsec'],
niter=100,
gain=0.05,
stokes='I',
weighting='natural',
restoringbeam=[str(bmsz)+'arcsec'],
pbcor=False,
interactive=False,
usescratch=True)
except:
print 'error in cleaning spw: '+spw
break
gaincal(vis=slfcalms, refant='0',antenna=antenna,caltable=slfcal_tb,spw=spw, uvrange='',\
gaintable=[],selectdata=True,timerange='',solint='600s',gaintype='G',calmode='p',\
combine='',minblperant=3,minsnr=2,append=False)
if not os.path.exists(slfcal_tb):
print 'No solution found in spw: '+spw
break
else:
clearcal(slfcalms)
delmod(slfcalms)
applycal(vis=slfcalms,gaintable=[slfcal_tb],spw=spw,selectdata=True,\
antenna=antenna,interp='nearest',flagbackup=False,applymode='calonly',calwt=False)
msfile=slfcalms
if cfreq < 10.:
imsize=512
cell=['5arcsec']
else:
imsize=1024
cell=['2.5arcsec']
if len(spwran) == 2:
spwstr = spwran[0]+'~'+spwran[1]
else:
spwstr = spwran[0]
restoringbeam=['{0:.1f}arcsec'.format(bmsz)]
imagesuffix='.spw'+spwstr.replace('~','-')
if cfreq > 10.:
antenna=antenna+';!0&1;!0&2' #deselect the shortest baselines
res=ptclean(vis=msfile, imageprefix=imdir, imagesuffix=imagesuffix, twidth=twidth, uvrange=uvrange,
spw=spw, ncpu=ncpu, niter=1000, gain=0.05, antenna=antenna,imsize=imsize, cell=cell,
stokes=stokes, doreg=True, usephacenter=False, overwrite=overwrite, toTb=toTb, restoringbeam=restoringbeam,
uvtaper=True,outertaper=['30arcsec'])
if res:
imres['Succeeded'] += res['Succeeded']
imres['BeginTime'] += res['BeginTime']
imres['EndTime'] += res['EndTime']
imres['ImageName'] += res['ImageName']
imres['Spw'] += [spwstr]*len(res['ImageName'])
imres['Vis'] += [msfile]*len(res['ImageName'])
else:
return None
#save it for debugging purposes
np.savez('imres.npz',imres=imres)
return imres
def disk_slfcal(vis, slfcaltbdir='./'):
''' Starting with the name of a calibrated ms (vis, which must have 'UDByyyymmdd' in the name)
add a model disk based on the solar disk size for that date and perform multiple selfcal
adjustments (two phase and one amplitude), and write out a final selfcaled database with
the disk subtracted. Returns the name of the final database.
'''
trange = ant_trange(vis)
# Use vis name to determine date, and hence number of bands
spw2band = np.array([0, 1] + range(4, 52))
defaultfreq = 1.1 + 0.325 * (spw2band + 0.5)
# Calculate the center frequency of each spectral window
if mstl.get_trange(vis)[0].mjd > 58536:
# After 2019 Feb 22, the band numbers changed to 1-52, and spw from 0-49
nbands = 52
freq = defaultfreq
else:
# Before 2019 Feb 22, the band numbers were 1-34, and spw from 0-30
nbands = 34
freq = np.hstack([[1.419], 2.0 + 0.5 * (np.arange(32)) + (0.5 - 0.081)])
slashdate = trange[:10]
# Verify that the vis is not in the current working directory
if os.getcwd() == os.path.dirname(vis):
print('Cannot copy vis file onto itself.')
print('Please change to a different working directory')
return None
# Copy original ms to local directory
if os.path.exists(os.path.basename(vis)):
shutil.rmtree(os.path.basename(vis))
print('Copy {} to working directory {}.'.format(vis, os.getcwd()))
shutil.copytree(vis, os.path.basename(vis))
vis = os.path.basename(vis)
clearcal(vis)
## automaticaly flag impossibly high amplitudes
flagdata(vis=vis, mode="tfcrop", spw='', correlation='ABS_XX', action='apply', display='',
timecutoff=3.0, freqcutoff=2.0, maxnpieces=2, flagbackup=True)
# Default disk size measured for 2019/09/03
# todo add monthly fitting procedure for the disk size and flux density
defaultsize = np.array([990.6, 989.4, 988.2, 987.1, 986.0, 984.9, 983.8, 982.7, 981.7, 980.7,
979.7, 978.8, 977.8, 976.9, 976.0, 975.2, 974.3, 973.5, 972.7, 972.0,
971.2, 970.5, 969.8, 969.1, 968.5, 967.8, 967.2, 966.7, 966.1, 965.6,
965.1, 964.6, 964.1, 963.7, 963.3, 962.9, 962.5, 962.1, 961.8, 961.5,
961.3, 961.0, 960.8, 960.6, 960.4, 960.2, 960.1, 960.0, 959.9, 959.8])
# Get current solar distance and modify the default size accordingly
fac = eph.get_sunearth_distance('2019/09/03') / eph.get_sunearth_distance(slashdate)
newsize = defaultsize * fac.to_value()
if nbands == 34:
# Interpolate size to 31 spectal windows
newsize = np.polyval(np.polyfit(defaultfreq, newsize, 5), freq)
dsize = np.array([str(i)[:5] + 'arcsec' for i in newsize], dtype='S12')
# These are nominal flux densities * 2, determined on 2019/09/03
defaultfdens = np.array([891282, 954570, 1173229, 1245433, 1373730, 1506802,
1613253, 1702751, 1800721, 1946756, 2096020, 2243951,
2367362, 2525968, 2699795, 2861604, 3054829, 3220450,
3404182, 3602625, 3794312, 3962926, 4164667, 4360683,
4575677, 4767210, 4972824, 5211717, 5444632, 5648266,
5926634, 6144249, 6339863, 6598018, 6802707, 7016012,
7258929, 7454951, 7742816, 7948976, 8203206, 8411834,
8656720, 8908130, 9087766, 9410760, 9571365, 9827078,
10023598, 8896671])
fdens = defaultfdens
if nbands == 34:
# Interpolate size to 31 spectal windows
fdens = np.polyval(np.polyfit(defaultfreq, fdens, 5), freq)
diskxmlfile = vis + '.SOLDISK.xml'
# Insert the disk model (msfile is the same as vis, and will be used as the "original" vis file name)
msfile = insertdiskmodel(vis, dsize=dsize, fdens=fdens, xmlfile=diskxmlfile)
tdate = mstl.get_trange(vis)[0].datetime.strftime('%Y%m%d')
caltb = os.path.join(slfcaltbdir, tdate + '_1.pha')
if os.path.exists(caltb):
os.system('rm -rf {}'.format(caltb))
# Phase selfcal on the disk using solution interval "infinite"
gaincal(vis=msfile, caltable=caltb, selectdata=True, uvrange="<3.0Klambda", antenna="0~12&0~12", solint="inf",
combine="scan",
refant="0", refantmode="flex", minsnr=1.0, gaintype="G", calmode="p", append=False)
applycal(vis=msfile, selectdata=True, antenna="0~12", gaintable=caltb, interp="nearest", calwt=False,
applymode="calonly")
# Split corrected data and model to a new ms for round 2 of phase selfcal
vis1 = 'slf_' + msfile
if os.path.exists(vis1):
os.system('rm -rf {}'.format(vis1))
mstl.splitX(msfile, outputvis=vis1, datacolumn="corrected", datacolumn2="model_data")
caltb = os.path.join(slfcaltbdir, tdate + '_2.pha')
if os.path.exists(caltb):
os.system('rm -rf {}'.format(caltb))
# Second round of phase selfcal on the disk using solution interval "1min"
gaincal(vis=vis1, caltable=caltb, selectdata=True, uvrange="<3.0Klambda", antenna="0~12&0~12", solint="1min",
combine="scan",
refant="0", refantmode="flex", minsnr=1.0, gaintype="G", calmode="p", append=False)
applycal(vis=vis1, selectdata=True, antenna="0~12", gaintable=caltb, interp="nearest", calwt=False,
applymode="calonly")
# Split corrected data and model to a new ms
vis2 = 'slf2_' + msfile
if os.path.exists(vis2):
os.system('rm -rf {}'.format(vis2))
mstl.splitX(vis, outputvis=vis2, datacolumn="corrected", datacolumn2="model_data")
caltb = os.path.join(slfcaltbdir, tdate + '_3.amp')
if os.path.exists(caltb):
os.system('rm -rf {}'.format(caltb))
# Final round of amplitude selfcal with 1-h solution interval (restrict to 16-24 UT)
gaincal(vis=vis2, caltable=caltb, selectdata=True, uvrange=">0.1Klambda", antenna="0~12&0~12",
timerange=trange,
solint="60min", combine="scan", refant="10", refantmode="flex", minsnr=1.0, gaintype="G", calmode="a",
append=False)
applycal(vis=vis2, selectdata=True, antenna="0~12", gaintable=caltb, interp="nearest", calwt=False,
applymode="calonly")
# Split out corrected data and model and do uvsub
vis3 = 'slf3_' + msfile
if os.path.exists(vis3):
os.system('rm -rf {}'.format(vis3))
mstl.splitX(vis, outputvis=vis3, datacolumn="corrected", datacolumn2="model_data")
uvsub(vis=vis3, reverse=False)
# Final split to
final = 'final_' + msfile
if os.path.exists(final):
os.system('rm -rf {}'.format(final))
split(vis3, outputvis=final, datacolumn='corrected')
# Remove the interim ms files
shutil.rmtree(vis)
shutil.rmtree(vis1)
shutil.rmtree(vis2)
shutil.rmtree(vis3)
# Return the name of the selfcaled ms
return final, diskxmlfile
if not os.path.exists(caltable_nocombine):
gaintables = [x for x in pipeline_tables if os.path.exists(x)]
gainfield = ['' for x in pipeline_tables if os.path.exists(x)]
interp = [
'linear,nearestflag' if 'BPcal' in x else ''
for x in pipeline_tables if os.path.exists(x)
]
gaincal(
vis=ms,
caltable=caltable_nocombine,
field='Sgr B2 NM Q,Sgr B2 MS Q',
calmode='p',
refant='',
solint='30s',
uvrange='0~2000klambda',
minblperant=3,
gaintable=gaintables,
gainfield=gainfield,
interp=interp,
)
if False:
for spw, bb in spw_bb:
caltable = 'sgrb2m_selfcal_phase_refFLEX_withpipe_{0}_{1}_30ssolint_shortbaselines.cal'.format(
bb, name)
if not os.path.exists(caltable):
gaintables = [x for x in pipeline_tables if os.path.exists(x)]
name=imagename,
niter=10000,
threshold='3mJy',
scales=[0,3,9],
robust=0.5,
savemodel='modelcolumn',
mask=mask,
)
caltable = '18A-229_{msname}_Q_cal_iter1.cal'.format(msname=msname)
if not os.path.exists(caltable):
gaincal(vis=cont_vis,
caltable=caltable,
field='Sgr B2 NM Q,Sgr B2 MS Q',
calmode='p',
refant='',
solint='60s',
minsnr=5,
#uvrange='0~2000klambda',
#minblperant=3,
)
# do a purely diagnostic ampcal
gaincal(vis=cont_vis,
caltable='18A-229_{msname}_Q_cal_iter1_ampcal_diagnostic.cal'.format(msname=msname),
gaintype='G',
combine='spw,scan,field',
solint='120s',
calmode='a',
solnorm=True)
