def insertdiskmodel(vis, sizescale=1.0, fdens=None, dsize=None, xmlfile='SOLDISK.xml', writediskinfoonly=False,
active=False, overwrite=True):
# Apply size scale adjustment (default is no adjustment)
for i in range(len(dsize)):
num, unit = dsize[i].split('arc')
dsize[i] = str(float(num) * sizescale)[:6] + 'arc' + unit
msfile = vis
ms.open(msfile)
spwinfo = ms.getspectralwindowinfo()
nspw = len(spwinfo.keys())
ms.close()
diskimdir = 'diskim/'
if not os.path.exists(diskimdir):
os.makedirs(diskimdir)
frq = []
spws = range(nspw)
for sp in spws:
spw = spwinfo[str(sp)]
frq.append('{:.4f}GHz'.format((spw['RefFreq'] + spw['TotalWidth'] / 2.0) / 1e9))
frq = np.array(frq)
writediskxml(dsize, fdens, frq, xmlfile=xmlfile)
if not writediskinfoonly:
tb.open(msfile + '/FIELD')
phadir = tb.getcol('PHASE_DIR').flatten()
tb.close()
ra = phadir[0]
dec = phadir[1]
direction = 'J2000 ' + str(ra) + 'rad ' + str(dec) + 'rad'
diskim = []
for sp in tqdm(spws, desc='Generating {} disk models'.format(nspw), ascii=True):
diskim.append(
mk_diskmodel(outname=diskimdir + 'disk{:02d}_'.format(sp), bdwidth=spwinfo[str(sp)],
direction=direction,
reffreq=frq[sp],
flux=fdens[sp], eqradius=dsize[sp], polradius=dsize[sp], overwrite=overwrite))
if not active:
delmod(msfile, otf=True, scr=True)
for sp in tqdm(spws, desc='Inserting disk model', ascii=True):
ft(vis=msfile, spw=str(sp), field='', model=str(diskim[sp]), nterms=1,
reffreq="", complist="", incremental=True, usescratch=True)
return msfile, diskim
def feature_slfcal(vis, niter=200, spws=['0~1', '2~5', '6~10', '11~20', '21~30', '31~49'], slfcaltbdir='./',
bright=None, pols='XX'):
''' Uses images from disk-selfcaled data as model for further self-calibration of outer antennas.
This is only a good idea if there are bright active regions that provide strong signal on the
londer baselines.
'''
trange = ant_trange(vis)
if bright is None:
bright = [True] * len(spws)
# Insert model into ms and do "inf" gaincal, appending to table each subsequent time
if os.path.exists('images_init'):
os.system('rm -rf images_init')
os.system('mv images images_init')
clearcal(vis, addmodel=True)
fd_images(vis, cleanup=False, niter=niter, spws=spws, bright=bright) # Does shallow clean for selfcal purposes
tdate = mstl.get_trange(vis)[0].datetime.strftime('%Y%m%d')
caltb = os.path.join(slfcaltbdir, tdate + '_d1.pha')
if os.path.exists(caltb):
os.system('rm -rf {}*'.format(caltb))
for s, sp in enumerate(spws):
if bright[s]:
spwstr = '-'.join(['{:02d}'.format(int(sp_)) for sp_ in sp.split('~')])
imname = "images/briggs" + spwstr + '.model'
if sp == '31~49':
# The high-band image is only made to band 43, so adjust the name
imname = 'images/briggs31-43.model'
ft(vis=vis, spw=sp, model=imname, usescratch=True, incremental=True)
if pols == 'XXYY':
mstl.modeltransfer(vis, spw=sp)
if pols == 'XXYY':
mstl.gaincalXY(vis=vis, caltable=caltb, pols=pols, selectdata=True, timerange=trange, uvrange='>1.5Klambda',
combine="scan", antenna='0~12&0~12', refant='0', refantmode="strict", solint='inf', gaintype='G',
minsnr=1.0, calmode='p', append=False)
else:
gaincal(vis=vis, caltable=caltb, selectdata=True, timerange=trange, uvrange='>1.5Klambda',
combine="scan", antenna='0~12&0~12', refant='0', refantmode="strict", solint='inf', gaintype='G',
minsnr=1.0, calmode='p', append=False)
# Apply the corrections to the data and split to a new ms
applycal(vis=vis, selectdata=True, antenna="0~12", gaintable=caltb, interp="linear", calwt=False,
applymode="calonly")
vis1 = 'dslf1_' + vis
if os.path.exists(vis1):
os.system('rm -rf {}'.format(vis1))
split(vis, outputvis=vis1, datacolumn="corrected")
caltb = os.path.join(slfcaltbdir, tdate + '_d2.pha')
if os.path.exists(caltb):
os.system('rm -rf {}*'.format(caltb))
# Move the existing images directory so that a new one will be created
if os.path.exists('images_ftcal_rnd1'):
os.system('rm -rf images_ftcal_rnd1')
# shutil.move('images', 'old_images2')
os.system('mv images images_ftcal_rnd1')
# Make new model images for another round of selfcal
fd_images(vis1, cleanup=False, niter=niter, spws=spws, bright=bright)
for s, sp in enumerate(spws):
if bright[s]:
spwstr = '-'.join(['{:02d}'.format(int(sp_)) for sp_ in sp.split('~')])
imname = "images/briggs" + spwstr + '.model'
if sp == '31~49':
# The high-band image is only made to band 43, so adjust the name
imname = 'images/briggs31-43.model'
ft(vis=vis1, spw=sp, model=imname, usescratch=True)
if pols == 'XXYY':
mstl.modeltransfer(vis1, spw=sp)
if pols == 'XXYY':
mstl.gaincalXY(vis=vis1, caltable=caltb, pols=pols, selectdata=True, timerange=trange, uvrange='>1.5Klambda',
combine="scan", antenna='0~12&0~12', refant='0', solint='10min', refantmode="strict",
gaintype='G', minsnr=1.0, calmode='p', append=False)
else:
gaincal(vis=vis1, caltable=caltb, selectdata=True, timerange=trange, uvrange='>1.5Klambda',
combine="scan", antenna='0~12&0~12', refant='0', solint='10min', refantmode="strict", gaintype='G',
minsnr=1.0, calmode='p', append=False)
# Apply the corrections to the data and split to a new ms
applycal(vis=vis1, selectdata=True, antenna="0~12", gaintable=caltb, interp="linear", calwt=False,
applymode="calonly")
vis2 = 'dslf2_' + vis
if os.path.exists(vis2):
os.system('rm -rf {}'.format(vis2))
mstl.splitX(vis1, outputvis=vis2, datacolumn="corrected", datacolumn2="model_data")
if os.path.exists('images_ftcal_rnd2'):
os.system('rm -rf images_ftcal_rnd2')
os.system('mv images images_ftcal_rnd2')
return vis2
def disk_slfcal(vis, slfcaltbdir='./', active=False, clearcache=False, pols='XX'):
''' Starting with the name of a calibrated ms (vis, which must have 'UDByyyymmdd' in the name)
add a model disk based on the solar disk size for that date and perform multiple selfcal
adjustments (two phase and one amplitude), and write out a final selfcaled database with
the disk subtracted. Returns the name of the final database.
'''
trange = ant_trange(vis)
if vis.endswith('/'):
vis = vis[:-1]
# Use vis name to determine date, and hence number of bands
spw2band = np.array([0, 1] + range(4, 52))
defaultfreq = 1.1 + 0.325 * (spw2band + 0.5)
# Calculate the center frequency of each spectral window
if mstl.get_trange(vis)[0].mjd > 58536:
# After 2019 Feb 22, the band numbers changed to 1-52, and spw from 0-49
nbands = 52
freq = defaultfreq
else:
# Before 2019 Feb 22, the band numbers were 1-34, and spw from 0-30
nbands = 34
freq = 1.419 + np.arange(nbands) / 2.
caltbs = []
slashdate = trange[:10]
# Verify that the vis is not in the current working directory
'''
if os.getcwd() == os.path.dirname(vis):
print('Cannot copy vis file onto itself.')
print('Please change to a different working directory')
return None
# Copy original ms to local directory
if os.path.exists(os.path.basename(vis)):
shutil.rmtree(os.path.basename(vis))
print('Copy {} to working directory {}.'.format(vis, os.getcwd()))
shutil.copytree(vis, os.path.basename(vis))
vis = os.path.basename(vis)
'''
if not active:
clearcal(vis)
flagmanager(vis, mode='save', versionname='with-RFI-or-BURSTS')
## automaticaly flag any high amplitudes from flares or RFI
flagdata(vis=vis, mode="tfcrop", spw='', action='apply', display='',
timecutoff=3.0, freqcutoff=2.0, maxnpieces=2, flagbackup=False)
flagmanager(vis, mode='save', versionname='without-RFI-or-BURSTS')
dsize, fdens = calc_diskmodel(slashdate, nbands, freq, defaultfreq)
diskxmlfile = vis + '.SOLDISK.xml'
# Insert the disk model (msfile is the same as vis, and will be used as the "original" vis file name)
msfile, diskim = insertdiskmodel(vis, dsize=dsize, fdens=fdens, xmlfile=diskxmlfile, active=active)
if pols == 'XXYY':
caltbs_ = {'XX': [], 'YY': []}
pols_ = ['XX', 'YY']
msfileXY = {}
for pol in pols_:
msfileXY[pol] = '.'.join([msfile, pol])
if os.path.exists(msfileXY[pol]):
os.system('rm -rf {}'.format(msfileXY[pol]))
mstl.splitX(vis=msfile, outputvis=msfileXY[pol], correlation=pol, datacolumn='data',
datacolumn2='MODEL_DATA')
tdate = mstl.get_trange(msfile)[0].datetime.strftime('%Y%m%d')
caltb = os.path.join(slfcaltbdir, tdate + '_1.pha')
if os.path.exists(caltb):
os.system('rm -rf {}*'.format(caltb))
if pols == 'XXYY':
mstl.gaincalXY(vis=msfile, caltable=caltb, pols=pols, msfileXY=msfileXY, selectdata=True, uvrange="",
antenna="0~12&0~12", solint="inf",
combine="scan", refant="0", refantmode="strict", minsnr=1.0, gaintype="G", calmode="p",
append=False)
for pol in pols_:
caltb_ = '.'.join([caltb, pol])
caltbs_[pol].append(caltb_)
else:
gaincal(vis=msfile, caltable=caltb, selectdata=True, uvrange="", antenna="0~12&0~12", solint="inf",
combine="scan", refant="0", refantmode="strict", minsnr=1.0, gaintype="G", calmode="p", append=False)
caltbs.append(caltb)
caltb = os.path.join(slfcaltbdir, tdate + '_2.pha')
if os.path.exists(caltb):
os.system('rm -rf {}*'.format(caltb))
# Second round of phase selfcal on the disk using solution interval "1min"
if pols == 'XXYY':
mstl.gaincalXY(vis=msfile, caltable=caltb, pols=pols, msfileXY=msfileXY, gaintableXY=caltbs_,
selectdata=True, uvrange="", antenna="0~12&0~12",
solint="10min",
combine="scan", interp="linear",
refant="0", refantmode="strict", minsnr=1.0, gaintype="G", calmode="p", append=False)
for pol in pols_:
caltb_ = '.'.join([caltb, pol])
caltbs_[pol].append(caltb_)
else:
gaincal(vis=msfile, caltable=caltb, selectdata=True, uvrange="", antenna="0~12&0~12", solint="10min",
combine="scan", gaintable=caltbs, interp="linear",
refant="0", refantmode="strict", minsnr=1.0, gaintype="G", calmode="p", append=False)
caltbs.append(caltb)
caltb = os.path.join(slfcaltbdir, tdate + '_3.amp')
if os.path.exists(caltb):
os.system('rm -rf {}*'.format(caltb))
# Final round of amplitude selfcal with 1-h solution interval (restrict to 16-24 UT)
if pols == 'XXYY':
mstl.gaincalXY(vis=msfile, caltable=caltb, pols=pols, msfileXY=msfileXY, gaintableXY=caltbs_,
selectdata=True, uvrange="", antenna="0~12&0~12",
timerange=trange, interp="linear",
solint="60min", combine="scan", refant="10", refantmode="flex", minsnr=1.0, gaintype="G",
calmode="a",
append=False)
for pol in pols_:
caltb_ = '.'.join([caltb, pol])
caltbs_[pol].append(caltb_)
else:
gaincal(vis=msfile, caltable=caltb, selectdata=True, uvrange="", antenna="0~12&0~12",
timerange=trange, gaintable=caltbs, interp="linear",
solint="60min", combine="scan", refant="10", refantmode="flex", minsnr=1.0, gaintype="G", calmode="a",
append=False)
mstl.flagcaltboutliers(caltb, limit=[0.125, 8.0])
# mstl.flagcaltboutliers(caltb, limit=[0.5, 2.0])
caltbs.append(caltb)
# Split out corrected data and model and do uvsub
vis2 = 'slf3_' + msfile
if os.path.exists(vis2):
os.system('rm -rf {}'.format(vis2))
if os.path.exists(vis2 + '.flagversions'):
os.system('rm -rf {}'.format(vis2 + '.flagversions'))
# flagmanager(msfile, mode='restore', versionname='with-RFI-or-BURSTS')
clearcal(msfile)
applycal(vis=msfile, selectdata=True, antenna="0~12", gaintable=caltbs, interp="linear", calwt=False,
applymode="calonly")
split(msfile, outputvis=vis2, datacolumn="corrected")
for sp, dkim in tqdm(enumerate(diskim), desc='Inserting disk model', ascii=True):
ft(vis=vis2, spw=str(sp), field='', model=str(dkim), nterms=1,
reffreq="", complist="", incremental=False, usescratch=True)
# mstl.modeltransfer(msfile, spw='{}'.format(sp))
uvsub(vis=vis2, reverse=False)
# Final split to
final = 'final_' + msfile
if os.path.exists(final):
os.system('rm -rf {}'.format(final))
if os.path.exists(final + '.flagversions'):
os.system('rm -rf {}'.format(final + '.flagversions'))
split(vis2, outputvis=final, datacolumn='corrected')
os.system('mv {} {}'.format(msfile + '.flagversions', final + '.flagversions'))
# Remove the interim ms files
if clearcache:
if os.path.exists(msfile):
os.system('rm -rf {}'.format(msfile))
if os.path.exists(msfile + '.flagversions'):
os.system('rm -rf {}'.format(msfile + '.flagversions'))
if os.path.exists(vis2):
os.system('rm -rf {}'.format(vis2))
if os.path.exists(vis2 + '.flagversions'):
os.system('rm -rf {}'.format(vis2 + '.flagversions'))
# Return the name of the selfcaled ms
return final, diskxmlfile
# Definitely DO NOT use this for amplitude self-cal!!
model = '../reduction_scripts/DePree_NM_regridNM_jypix.image'
if os.path.exists(model):
myprint("Model {0} exists, using it for selfcal of {1}".format(
model, cont_ms))
# if it doesn't exist, we need to run
# continuum_imaging_make_selfcal_model
caltable = '{0}_sgrb2_selfcal_phase_30ssolint.cal'.format(name)
if not os.path.exists(caltable):
myprint("Caltable {0} does not exist, making it".format(caltable))
ft(vis=cont_ms,
field='Sgr B2 N Q,Sgr B2 NM Q,Sgr B2 MS Q',
spw='',
model=model)
gaincal(
vis=cont_ms,
caltable=caltable,
field='Sgr B2 N Q,Sgr B2 NM Q,Sgr B2 MS Q',
calmode='p',
refant='',
solint='30s',
#uvrange='0~2000klambda',
minblperant=3,
)
# these tables are JUST FOR DIAGNOSTICS: don't use them
ampcaltable = '{0}_sgrb2_selfcal_amp_INFsolint_dontuse.cal'.format(
def insertdiskmodel(vis, sizescale=1.0, fdens=None, dsize=None, overwrite=True, xmlfile='SOLDISK.xml'):
if fdens is None:
# Default flux density for solar minimum
fdens = np.array([891282, 954570, 1173229, 1245433, 1373730, 1506802,
1613253, 1702751, 1800721, 1946756, 2096020, 2243951,
2367362, 2525968, 2699795, 2861604, 3054829, 3220450,
3404182, 3602625, 3794312, 3962926, 4164667, 4360683,
4575677, 4767210, 4972824, 5211717, 5444632, 5648266,
5926634, 6144249, 6339863, 6598018, 6802707, 7016012,
7258929, 7454951, 7742816, 7948976, 8203206, 8411834,
8656720, 8908130, 9087766, 9410760, 9571365, 9827078,
10023598, 8896671])
if dsize is None:
# Default solar disk radius for solar minimum
dsize = np.array(['1228.0arcsec', '1194.0arcsec', '1165.0arcsec', '1139.0arcsec', '1117.0arcsec',
'1097.0arcsec', '1080.0arcsec', '1065.0arcsec', '1053.0arcsec', '1042.0arcsec',
'1033.0arcsec', '1025.0arcsec', '1018.0arcsec', '1012.0arcsec', '1008.0arcsec',
'1003.0arcsec', '1000.0arcsec', '997.0arcsec', '994.0arcsec', '992.0arcsec',
'990.0arcsec', '988.0arcsec', '986.0arcsec', '985.0arcsec', '983.0arcsec', '982.0arcsec',
'980.0arcsec', '979.0arcsec', '978.0arcsec', '976.0arcsec', '975.0arcsec', '974.0arcsec',
'972.0arcsec', '971.0arcsec', '970.0arcsec', '969.0arcsec', '968.0arcsec', '967.0arcsec',
'966.0arcsec', '965.0arcsec', '964.0arcsec', '964.0arcsec', '963.0arcsec', '962.0arcsec',
'962.0arcsec', '961.0arcsec', '960.0arcsec', '959.0arcsec', '957.0arcsec', '956.0arcsec'])
# Apply size scale adjustment (default is no adjustment)
for i in range(len(dsize)):
num, unit = dsize[i].split('arc')
dsize[i] = str(float(num) * sizescale)[:6] + 'arc' + unit
msfile = vis
diskim = []
ms.open(msfile)
spwinfo = ms.getspectralwindowinfo()
nspw = len(spwinfo.keys())
ms.close()
diskimdir = 'diskim/'
if not os.path.exists(diskimdir):
os.makedirs(diskimdir)
frq = []
for sp in range(nspw):
spw = spwinfo[str(sp)]
frq.append('{:.4f}GHz'.format((spw['RefFreq'] + spw['TotalWidth'] / 2.0) / 1e9))
frq = np.array(frq)
writediskxml(dsize, fdens, frq, xmlfile=xmlfile)
tb.open(msfile + '/FIELD')
phadir = tb.getcol('PHASE_DIR').flatten()
tb.close()
ra = phadir[0]
dec = phadir[1]
direction = 'J2000 ' + str(ra) + 'rad ' + str(dec) + 'rad'
for sp in tqdm(range(nspw), desc='Generating {} disk models'.format(nspw), ascii=True):
diskim.append(
diskmodel(outname=diskimdir + 'disk{:02d}_'.format(sp), bdwidth=spwinfo[str(sp)], direction=direction,
reffreq=frq[sp],
flux=fdens[sp], eqradius=dsize[sp], polradius=dsize[sp], overwrite=overwrite))
delmod(msfile, otf=True, scr=True)
mstl.clearflagrow(msfile, mode='clear')
for sp in tqdm(range(nspw), desc='Inserting disk model', ascii=True):
ft(vis=msfile, spw=str(sp), field='', model=str(diskim[sp]), nterms=1,
reffreq="", complist="", incremental=False, usescratch=True)
uvsub(vis=msfile)
# tb.open(os.path.join(msfile, 'DATA_DESCRIPTION'), nomodify=False)
#
# tabdesc = {'DISK_SIZE': {'comment': 'Size of solar disk',
# 'dataManagerGroup': 'StandardStMan',
# 'dataManagerType': 'StandardStMan',
# 'keywords': {},
# 'maxlen': 0,
# 'option': 0,
# 'valueType': 'double'},
# 'DISK_FLUX': {'comment': 'Flux [Jy] of solar disk',
# 'dataManagerGroup': 'StandardStMan',
# 'dataManagerType': 'StandardStMan',
# 'keywords': {},
# 'maxlen': 0,
# 'option': 0,
# 'valueType': 'double'},
# '_define_hypercolumn_': {},
# '_keywords_': {},
# '_private_keywords_': {}}
#
# tb.addcols(tabdesc)
# tb.close()
# tabdesc = {'DISK_SIZE': {'comment': 'Size of solar disk',
# 'dataManagerGroup': 'StandardStMan',
# 'dataManagerType': 'StandardStMan',
# 'keywords': {},
# 'maxlen': 0,
# 'option': 0,
# 'valueType': 'double'},
# 'DISK_FLUX': {'comment': 'Flux [Jy] of solar disk',
# 'dataManagerGroup': 'StandardStMan',
# 'dataManagerType': 'StandardStMan',
# 'keywords': {},
# 'maxlen': 0,
# 'option': 0,
# 'valueType': 'double'},
# '_define_hypercolumn_': {},
# '_keywords_': {},
# '_private_keywords_': {}}
# tb.create(os.path.join(msfile, 'SOLDISK'),tabdesc)
# tb.addrows(50) # Add the rows _before_ filling the columns.
# tb.close()
return msfile
history=["{0}: {1}".format(key, val) for key, val in
impars.items()])
ia.close()
exportfits(imname+".image.tt0", imname+".image.tt0.fits")
exportfits(imname+".image.tt0.pbcor", imname+".image.tt0.pbcor.fits")
else:
# populate the model column
modelname = [contimagename+"_robust{0}.model.tt0".format(robust),
contimagename+"_robust{0}.model.tt1".format(robust)]
logprint("Using ``ft`` to populate the model column",
origin='almaimf_cont_selfcal')
ft(vis=selfcal_ms,
field=field.encode(),
model=modelname,
nterms=2,
usescratch=True
)
logprint("Skipped completed file {0} (dirty),"
" populated model column".format(imname),
origin='almaimf_cont_selfcal')
# make a custom mask using the first-pass clean
# (note: this will be replaced after each iteration if there is a file with
# the appropriate name)
if 'maskname' not in locals():
maskname = make_custom_mask(field, imname+".image.tt0",
os.getenv('ALMAIMF_ROOTDIR'), band,
rootdir=imaging_root,
)
assert ms in Qmses
name = ms[:22]
ms = '../' + ms
flagdata(vis=ms,
mode='unflag',
field=('J1744-3116,Sgr B2 N Q,Sgr B2 NM Q,SgrB2 MS Q,'
'Sgr B2 S Q,Sgr B2 DS1 Q,Sgr B2 DS2 Q,Sgr B2 DS3 Q'))
flagdata(vis=ms, mode='manual', autocorr=True)
ft(vis=ms,
field='Sgr B2 N Q,Sgr B2 NM Q,Sgr B2 MS Q',
spw='',
model='../reduction_scripts/DePree_NM_regridNM.image',
nterms=1)
#splitvis = '{0}.1744-3116.split.ms'.format(name)
## *do not* split out spw to avoid re-mapping later
#split(vis="../"+ms, outputvis=splitvis,
# datacolumn='corrected',
# width=512, # averaged each spw to 1 channel
# field='J1744-3116',
# timebin='5s',)
pipeline_tables = [
ms + '.hifv_priorcals.s5_5.rq.tbl',
ms + '.hifv_priorcals.s5_7.ants.tbl',
