def calibeovsa(vis=None,
caltype=None,
interp=None,
docalib=True,
doflag=True,
flagant=None,
doimage=False,
imagedir=None,
antenna=None,
timerange=None,
spw=None,
stokes=None,
doconcat=False,
msoutdir=None,
keep_orig_ms=True):
'''
:param vis: EOVSA visibility dataset(s) to be calibrated
:param caltype:
:param interp:
:param docalib:
:param qlookimage:
:param flagant:
:param stokes:
:param doconcat:
:return:
'''
if type(vis) == str:
vis = [vis]
for idx, f in enumerate(vis):
if f[-1] == '/':
vis[idx] = f[:-1]
for msfile in vis:
casalog.origin('calibeovsa')
if not caltype:
casalog.post(
"Caltype not provided. Perform reference phase calibration and daily phase calibration."
)
caltype = [
'refpha', 'phacal', 'fluxcal'
] ## use this line after the phacal is applied # caltype = ['refcal']
if not os.path.exists(msfile):
casalog.post("Input visibility does not exist. Aborting...")
continue
if msfile.endswith('/'):
msfile = msfile[:-1]
if not msfile[-3:] in ['.ms', '.MS']:
casalog.post(
"Invalid visibility. Please provide a proper visibility file ending with .ms"
)
# if not caltable:
# caltable=[os.path.basename(vis).replace('.ms','.'+c) for c in caltype]
# get band information
tb.open(msfile + '/SPECTRAL_WINDOW')
nspw = tb.nrows()
bdname = tb.getcol('NAME')
bd_nchan = tb.getcol('NUM_CHAN')
bd = [int(b[4:]) - 1 for b in bdname] # band index from 0 to 33
# nchans = tb.getcol('NUM_CHAN')
# reffreqs = tb.getcol('REF_FREQUENCY')
# cenfreqs = np.zeros((nspw))
tb.close()
tb.open(msfile + '/ANTENNA')
nant = tb.nrows()
antname = tb.getcol('NAME')
antlist = [str(ll) for ll in range(len(antname) - 1)]
antennas = ','.join(antlist)
tb.close()
# get time stamp, use the beginning of the file
tb.open(msfile + '/OBSERVATION')
trs = {'BegTime': [], 'EndTime': []}
for ll in range(tb.nrows()):
tim0, tim1 = Time(tb.getcell('TIME_RANGE', ll) / 24 / 3600,
format='mjd')
trs['BegTime'].append(tim0)
trs['EndTime'].append(tim1)
tb.close()
trs['BegTime'] = Time(trs['BegTime'])
trs['EndTime'] = Time(trs['EndTime'])
btime = np.min(trs['BegTime'])
etime = np.max(trs['EndTime'])
# ms.open(vis)
# summary = ms.summary()
# ms.close()
# btime = Time(summary['BeginTime'], format='mjd')
# etime = Time(summary['EndTime'], format='mjd')
## stop using ms.summary to avoid conflicts with importeovsa
t_mid = Time((btime.mjd + etime.mjd) / 2., format='mjd')
print "This scan observed from {} to {} UTC".format(
btime.iso, etime.iso)
gaintables = []
if ('refpha' in caltype) or ('refamp' in caltype) or ('refcal'
in caltype):
refcal = ra.sql2refcalX(btime)
pha = refcal['pha'] # shape is 15 (nant) x 2 (npol) x 34 (nband)
pha[np.where(refcal['flag'] == 1)] = 0.
amp = refcal['amp']
amp[np.where(refcal['flag'] == 1)] = 1.
t_ref = refcal['timestamp']
# find the start and end time of the local day when refcal is registered
try:
dhr = t_ref.LocalTime.utcoffset().total_seconds() / 60. / 60.
except:
dhr = -7.
bt = Time(np.fix(t_ref.mjd + dhr / 24.) - dhr / 24., format='mjd')
et = Time(bt.mjd + 1., format='mjd')
(yr, mon, day) = (bt.datetime.year, bt.datetime.month,
bt.datetime.day)
dirname = caltbdir + str(yr) + str(mon).zfill(2) + '/'
if not os.path.exists(dirname):
os.mkdir(dirname)
# check if there is any ROACH reboot between the reference calibration found and the current data
t_rbts = db.get_reboot(Time([t_ref, btime]))
if not t_rbts:
casalog.post(
"Reference calibration is derived from observation at " +
t_ref.iso)
print "Reference calibration is derived from observation at " + t_ref.iso
else:
casalog.post(
"Oh crap! Roach reboot detected between the reference calibration time "
+ t_ref.iso + ' and the current observation at ' +
btime.iso)
casalog.post("Aborting...")
print "Oh crap! Roach reboot detected between the reference calibration time " + t_ref.iso + ' and the current observation at ' + btime.iso
print "Aborting..."
para_pha = []
para_amp = []
calpha = np.zeros((nspw, 15, 2))
calamp = np.zeros((nspw, 15, 2))
for s in range(nspw):
for n in range(15):
for p in range(2):
calpha[s, n, p] = pha[n, p, bd[s]]
calamp[s, n, p] = amp[n, p, bd[s]]
para_pha.append(np.degrees(pha[n, p, bd[s]]))
para_amp.append(amp[n, p, bd[s]])
if 'fluxcal' in caltype:
calfac = pc.get_calfac(Time(t_mid.iso.split(' ')[0] + 'T23:59:59'))
t_bp = Time(calfac['timestamp'], format='lv')
if int(t_mid.mjd) == int(t_bp.mjd):
accalfac = calfac['accalfac'] # (ant x pol x freq)
# tpcalfac = calfac['tpcalfac'] # (ant x pol x freq)
caltb_autoamp = dirname + t_bp.isot[:-4].replace(
':', '').replace('-', '') + '.bandpass'
if not os.path.exists(caltb_autoamp):
bandpass(vis=msfile,
caltable=caltb_autoamp,
solint='inf',
refant='eo01',
minblperant=0,
minsnr=0,
bandtype='B',
docallib=False)
tb.open(caltb_autoamp, nomodify=False) # (ant x spw)
bd_chanidx = np.hstack([[0], bd_nchan.cumsum()])
for ll in range(nspw):
antfac = np.sqrt(
accalfac[:, :, bd_chanidx[ll]:bd_chanidx[ll + 1]])
# # antfac *= tpcalfac[:, :,bd_chanidx[ll]:bd_chanidx[ll + 1]]
antfac = np.moveaxis(antfac, 0, 2)
cparam = np.zeros((2, bd_nchan[ll], nant))
cparam[:, :, :-3] = 1.0 / antfac
tb.putcol('CPARAM', cparam + 0j, ll * nant, nant)
paramerr = tb.getcol('PARAMERR', ll * nant, nant)
paramerr = paramerr * 0
tb.putcol('PARAMERR', paramerr, ll * nant, nant)
bpflag = tb.getcol('FLAG', ll * nant, nant)
bpant1 = tb.getcol('ANTENNA1', ll * nant, nant)
bpflagidx, = np.where(bpant1 >= 13)
bpflag[:] = False
bpflag[:, :, bpflagidx] = True
tb.putcol('FLAG', bpflag, ll * nant, nant)
bpsnr = tb.getcol('SNR', ll * nant, nant)
bpsnr[:] = 100.0
bpsnr[:, :, bpflagidx] = 0.0
tb.putcol('SNR', bpsnr, ll * nant, nant)
tb.close()
msg_prompt = "Scaling calibration is derived for {}.".format(
msfile)
casalog.post(msg_prompt)
print msg_prompt
gaintables.append(caltb_autoamp)
else:
msg_prompt = "Caution: No TPCAL is available on {}. No scaling calibration is derived for {}.".format(
t_mid.datetime.strftime('%b %d, %Y'), msfile)
casalog.post(msg_prompt)
print msg_prompt
if ('refpha' in caltype) or ('refcal' in caltype):
# caltb_pha = os.path.basename(vis).replace('.ms', '.refpha')
# check if the calibration table already exists
caltb_pha = dirname + t_ref.isot[:-4].replace(':', '').replace(
'-', '') + '.refpha'
if not os.path.exists(caltb_pha):
gencal(vis=msfile,
caltable=caltb_pha,
caltype='ph',
antenna=antennas,
pol='X,Y',
spw='0~' + str(nspw - 1),
parameter=para_pha)
gaintables.append(caltb_pha)
if ('refamp' in caltype) or ('refcal' in caltype):
# caltb_amp = os.path.basename(vis).replace('.ms', '.refamp')
caltb_amp = dirname + t_ref.isot[:-4].replace(':', '').replace(
'-', '') + '.refamp'
if not os.path.exists(caltb_amp):
gencal(vis=msfile,
caltable=caltb_amp,
caltype='amp',
antenna=antennas,
pol='X,Y',
spw='0~' + str(nspw - 1),
parameter=para_amp)
gaintables.append(caltb_amp)
# calibration for the change of delay center between refcal time and beginning of scan -- hopefully none!
xml, buf = ch.read_calX(4, t=[t_ref, btime], verbose=False)
if buf:
dly_t2 = Time(stf.extract(buf[0], xml['Timestamp']), format='lv')
dlycen_ns2 = stf.extract(buf[0], xml['Delaycen_ns'])[:15]
xml, buf = ch.read_calX(4, t=t_ref)
dly_t1 = Time(stf.extract(buf, xml['Timestamp']), format='lv')
dlycen_ns1 = stf.extract(buf, xml['Delaycen_ns'])[:15]
dlycen_ns_diff = dlycen_ns2 - dlycen_ns1
for n in range(2):
dlycen_ns_diff[:, n] -= dlycen_ns_diff[0, n]
print 'Multi-band delay is derived from delay center difference at {} & {}'.format(
dly_t1.iso, dly_t2.iso)
# print '=====Delays relative to Ant 14====='
# for i, dl in enumerate(dlacen_ns_diff[:, 0] - dlacen_ns_diff[13, 0]):
# ant = antlist[i]
# print 'Ant eo{0:02d}: x {1:.2f} ns & y {2:.2f} ns'.format(int(ant) + 1, dl
# dlacen_ns_diff[i, 1] - dlacen_ns_diff[13, 1])
# caltb_mbd0 = os.path.basename(vis).replace('.ms', '.mbd0')
caltb_dlycen = dirname + dly_t2.isot[:-4].replace(':', '').replace(
'-', '') + '.dlycen'
if not os.path.exists(caltb_dlycen):
gencal(vis=msfile,
caltable=caltb_dlycen,
caltype='mbd',
pol='X,Y',
antenna=antennas,
parameter=dlycen_ns_diff.flatten().tolist())
gaintables.append(caltb_dlycen)
if 'phacal' in caltype:
phacals = np.array(
ra.sql2phacalX([bt, et], neat=True, verbose=False))
if not phacals.any() or len(phacals) == 0:
print "Found no phacal records in SQL database, will skip phase calibration"
else:
# first generate all phacal calibration tables if not already exist
t_phas = Time([phacal['t_pha'] for phacal in phacals])
# sort the array in ascending order by t_pha
sinds = t_phas.mjd.argsort()
t_phas = t_phas[sinds]
phacals = phacals[sinds]
caltbs_phambd = []
for i, phacal in enumerate(phacals):
# filter out phase cals with reference time stamp >30 min away from the provided refcal time
if (phacal['t_ref'].jd -
refcal['timestamp'].jd) > 30. / 1440.:
del phacals[i]
del t_phas[i]
continue
else:
t_pha = phacal['t_pha']
phambd_ns = phacal['pslope']
for n in range(2):
phambd_ns[:, n] -= phambd_ns[0, n]
# set all flagged values to be zero
phambd_ns[np.where(phacal['flag'] == 1)] = 0.
caltb_phambd = dirname + t_pha.isot[:-4].replace(
':', '').replace('-', '') + '.phambd'
caltbs_phambd.append(caltb_phambd)
if not os.path.exists(caltb_phambd):
gencal(vis=msfile,
caltable=caltb_phambd,
caltype='mbd',
pol='X,Y',
antenna=antennas,
parameter=phambd_ns.flatten().tolist())
# now decides which table to apply depending on the interpolation method ("neatest" or "linear")
if interp == 'nearest':
tbind = np.argmin(np.abs(t_phas.mjd - t_mid.mjd))
dt = np.min(np.abs(t_phas.mjd - t_mid.mjd)) * 24.
print "Selected nearest phase calibration table at " + t_phas[
tbind].iso
gaintables.append(caltbs_phambd[tbind])
if interp == 'linear':
# bphacal = ra.sql2phacalX(btime)
# ephacal = ra.sql2phacalX(etime,reverse=True)
bt_ind, = np.where(t_phas.mjd < btime.mjd)
et_ind, = np.where(t_phas.mjd > etime.mjd)
if len(bt_ind) == 0 and len(et_ind) == 0:
print "No phacal found before or after the ms data within the day of observation"
print "Skipping daily phase calibration"
elif len(bt_ind) > 0 and len(et_ind) == 0:
gaintables.append(caltbs_phambd[bt_ind[-1]])
elif len(bt_ind) == 0 and len(et_ind) > 0:
gaintables.append(caltbs_phambd[et_ind[0]])
elif len(bt_ind) > 0 and len(et_ind) > 0:
bphacal = phacals[bt_ind[-1]]
ephacal = phacals[et_ind[0]]
# generate a new table interpolating between two daily phase calibrations
t_pha_mean = Time(np.mean(
[bphacal['t_pha'].mjd, ephacal['t_pha'].mjd]),
format='mjd')
phambd_ns = (bphacal['pslope'] +
ephacal['pslope']) / 2.
for n in range(2):
phambd_ns[:, n] -= phambd_ns[0, n]
# set all flagged values to be zero
phambd_ns[np.where(bphacal['flag'] == 1)] = 0.
phambd_ns[np.where(ephacal['flag'] == 1)] = 0.
caltb_phambd_interp = dirname + t_pha_mean.isot[:-4].replace(
':', '').replace('-', '') + '.phambd'
if not os.path.exists(caltb_phambd_interp):
gencal(vis=msfile,
caltable=caltb_phambd_interp,
caltype='mbd',
pol='X,Y',
antenna=antennas,
parameter=phambd_ns.flatten().tolist())
print "Using phase calibration table interpolated between records at " + bphacal[
't_pha'].iso + ' and ' + ephacal['t_pha'].iso
gaintables.append(caltb_phambd_interp)
if docalib:
clearcal(msfile)
applycal(vis=msfile,
gaintable=gaintables,
applymode='calflag',
calwt=False)
# delete the interpolated phase calibration table
try:
caltb_phambd_interp
except:
pass
else:
if os.path.exists(caltb_phambd_interp):
shutil.rmtree(caltb_phambd_interp)
if doflag:
# flag zeros and NaNs
flagdata(vis=msfile, mode='clip', clipzeros=True)
if flagant:
try:
flagdata(vis=msfile, antenna=flagant)
except:
print "Something wrong with flagant. Abort..."
if doimage:
from matplotlib import pyplot as plt
from suncasa.utils import helioimage2fits as hf
from sunpy import map as smap
if not antenna:
antenna = '0~12'
if not stokes:
stokes = 'XX'
if not timerange:
timerange = ''
if not spw:
spw = '1~3'
if not imagedir:
imagedir = '.'
#(yr, mon, day) = (bt.datetime.year, bt.datetime.month, bt.datetime.day)
#dirname = imagedir + str(yr) + '/' + str(mon).zfill(2) + '/' + str(day).zfill(2) + '/'
#if not os.path.exists(dirname):
# os.makedirs(dirname)
bds = [spw]
nbd = len(bds)
imgs = []
for bd in bds:
if '~' in bd:
bdstr = bd.replace('~', '-')
else:
bdstr = str(bd).zfill(2)
imname = imagedir + '/' + os.path.basename(msfile).replace(
'.ms', '.bd' + bdstr)
print 'Cleaning image: ' + imname
try:
clean(vis=msfile,
imagename=imname,
antenna=antenna,
spw=bd,
timerange=timerange,
imsize=[512],
cell=['5.0arcsec'],
stokes=stokes,
niter=500)
except:
print 'clean not successfull for band ' + str(bd)
else:
imgs.append(imname + '.image')
junks = ['.flux', '.mask', '.model', '.psf', '.residual']
for junk in junks:
if os.path.exists(imname + junk):
shutil.rmtree(imname + junk)
tranges = [btime.iso + '~' + etime.iso] * nbd
fitsfiles = [img.replace('.image', '.fits') for img in imgs]
hf.imreg(vis=msfile,
timerange=tranges,
imagefile=imgs,
fitsfile=fitsfiles,
usephacenter=False)
plt.figure(figsize=(6, 6))
for i, fitsfile in enumerate(fitsfiles):
plt.subplot(1, nbd, i + 1)
eomap = smap.Map(fitsfile)
sz = eomap.data.shape
if len(sz) == 4:
eomap.data = eomap.data.reshape((sz[2], sz[3]))
eomap.plot_settings['cmap'] = plt.get_cmap('jet')
eomap.plot()
eomap.draw_limb()
eomap.draw_grid()
plt.show()
if doconcat:
if len(vis) > 1:
# from suncasa.eovsa import concateovsa as ce
from suncasa.tasks import concateovsa_cli as ce
if msoutdir is None:
msoutdir = './'
concatvis = os.path.basename(vis[0])
concatvis = msoutdir + '/' + concatvis.split('.')[0] + '_concat.ms'
ce.concateovsa(vis,
concatvis,
datacolumn='corrected',
keep_orig_ms=keep_orig_ms,
cols2rm="model,corrected")
return [concatvis]
else:
return vis
def disk_slfcal(vis, slfcaltbdir='./', active=False, clearcache=False, pols='XX'):
''' Starting with the name of a calibrated ms (vis, which must have 'UDByyyymmdd' in the name)
add a model disk based on the solar disk size for that date and perform multiple selfcal
adjustments (two phase and one amplitude), and write out a final selfcaled database with
the disk subtracted. Returns the name of the final database.
'''
trange = ant_trange(vis)
if vis.endswith('/'):
vis = vis[:-1]
# Use vis name to determine date, and hence number of bands
spw2band = np.array([0, 1] + range(4, 52))
defaultfreq = 1.1 + 0.325 * (spw2band + 0.5)
# Calculate the center frequency of each spectral window
if mstl.get_trange(vis)[0].mjd > 58536:
# After 2019 Feb 22, the band numbers changed to 1-52, and spw from 0-49
nbands = 52
freq = defaultfreq
else:
# Before 2019 Feb 22, the band numbers were 1-34, and spw from 0-30
nbands = 34
freq = 1.419 + np.arange(nbands) / 2.
caltbs = []
slashdate = trange[:10]
# Verify that the vis is not in the current working directory
'''
if os.getcwd() == os.path.dirname(vis):
print('Cannot copy vis file onto itself.')
print('Please change to a different working directory')
return None
# Copy original ms to local directory
if os.path.exists(os.path.basename(vis)):
shutil.rmtree(os.path.basename(vis))
print('Copy {} to working directory {}.'.format(vis, os.getcwd()))
shutil.copytree(vis, os.path.basename(vis))
vis = os.path.basename(vis)
'''
if not active:
clearcal(vis)
flagmanager(vis, mode='save', versionname='with-RFI-or-BURSTS')
## automaticaly flag any high amplitudes from flares or RFI
flagdata(vis=vis, mode="tfcrop", spw='', action='apply', display='',
timecutoff=3.0, freqcutoff=2.0, maxnpieces=2, flagbackup=False)
flagmanager(vis, mode='save', versionname='without-RFI-or-BURSTS')
dsize, fdens = calc_diskmodel(slashdate, nbands, freq, defaultfreq)
diskxmlfile = vis + '.SOLDISK.xml'
# Insert the disk model (msfile is the same as vis, and will be used as the "original" vis file name)
msfile, diskim = insertdiskmodel(vis, dsize=dsize, fdens=fdens, xmlfile=diskxmlfile, active=active)
if pols == 'XXYY':
caltbs_ = {'XX': [], 'YY': []}
pols_ = ['XX', 'YY']
msfileXY = {}
for pol in pols_:
msfileXY[pol] = '.'.join([msfile, pol])
if os.path.exists(msfileXY[pol]):
os.system('rm -rf {}'.format(msfileXY[pol]))
mstl.splitX(vis=msfile, outputvis=msfileXY[pol], correlation=pol, datacolumn='data',
datacolumn2='MODEL_DATA')
tdate = mstl.get_trange(msfile)[0].datetime.strftime('%Y%m%d')
caltb = os.path.join(slfcaltbdir, tdate + '_1.pha')
if os.path.exists(caltb):
os.system('rm -rf {}*'.format(caltb))
if pols == 'XXYY':
mstl.gaincalXY(vis=msfile, caltable=caltb, pols=pols, msfileXY=msfileXY, selectdata=True, uvrange="",
antenna="0~12&0~12", solint="inf",
combine="scan", refant="0", refantmode="strict", minsnr=1.0, gaintype="G", calmode="p",
append=False)
for pol in pols_:
caltb_ = '.'.join([caltb, pol])
caltbs_[pol].append(caltb_)
else:
gaincal(vis=msfile, caltable=caltb, selectdata=True, uvrange="", antenna="0~12&0~12", solint="inf",
combine="scan", refant="0", refantmode="strict", minsnr=1.0, gaintype="G", calmode="p", append=False)
caltbs.append(caltb)
caltb = os.path.join(slfcaltbdir, tdate + '_2.pha')
if os.path.exists(caltb):
os.system('rm -rf {}*'.format(caltb))
# Second round of phase selfcal on the disk using solution interval "1min"
if pols == 'XXYY':
mstl.gaincalXY(vis=msfile, caltable=caltb, pols=pols, msfileXY=msfileXY, gaintableXY=caltbs_,
selectdata=True, uvrange="", antenna="0~12&0~12",
solint="10min",
combine="scan", interp="linear",
refant="0", refantmode="strict", minsnr=1.0, gaintype="G", calmode="p", append=False)
for pol in pols_:
caltb_ = '.'.join([caltb, pol])
caltbs_[pol].append(caltb_)
else:
gaincal(vis=msfile, caltable=caltb, selectdata=True, uvrange="", antenna="0~12&0~12", solint="10min",
combine="scan", gaintable=caltbs, interp="linear",
refant="0", refantmode="strict", minsnr=1.0, gaintype="G", calmode="p", append=False)
caltbs.append(caltb)
caltb = os.path.join(slfcaltbdir, tdate + '_3.amp')
if os.path.exists(caltb):
os.system('rm -rf {}*'.format(caltb))
# Final round of amplitude selfcal with 1-h solution interval (restrict to 16-24 UT)
if pols == 'XXYY':
mstl.gaincalXY(vis=msfile, caltable=caltb, pols=pols, msfileXY=msfileXY, gaintableXY=caltbs_,
selectdata=True, uvrange="", antenna="0~12&0~12",
timerange=trange, interp="linear",
solint="60min", combine="scan", refant="10", refantmode="flex", minsnr=1.0, gaintype="G",
calmode="a",
append=False)
for pol in pols_:
caltb_ = '.'.join([caltb, pol])
caltbs_[pol].append(caltb_)
else:
gaincal(vis=msfile, caltable=caltb, selectdata=True, uvrange="", antenna="0~12&0~12",
timerange=trange, gaintable=caltbs, interp="linear",
solint="60min", combine="scan", refant="10", refantmode="flex", minsnr=1.0, gaintype="G", calmode="a",
append=False)
mstl.flagcaltboutliers(caltb, limit=[0.125, 8.0])
# mstl.flagcaltboutliers(caltb, limit=[0.5, 2.0])
caltbs.append(caltb)
# Split out corrected data and model and do uvsub
vis2 = 'slf3_' + msfile
if os.path.exists(vis2):
os.system('rm -rf {}'.format(vis2))
if os.path.exists(vis2 + '.flagversions'):
os.system('rm -rf {}'.format(vis2 + '.flagversions'))
# flagmanager(msfile, mode='restore', versionname='with-RFI-or-BURSTS')
clearcal(msfile)
applycal(vis=msfile, selectdata=True, antenna="0~12", gaintable=caltbs, interp="linear", calwt=False,
applymode="calonly")
split(msfile, outputvis=vis2, datacolumn="corrected")
for sp, dkim in tqdm(enumerate(diskim), desc='Inserting disk model', ascii=True):
ft(vis=vis2, spw=str(sp), field='', model=str(dkim), nterms=1,
reffreq="", complist="", incremental=False, usescratch=True)
# mstl.modeltransfer(msfile, spw='{}'.format(sp))
uvsub(vis=vis2, reverse=False)
# Final split to
final = 'final_' + msfile
if os.path.exists(final):
os.system('rm -rf {}'.format(final))
if os.path.exists(final + '.flagversions'):
os.system('rm -rf {}'.format(final + '.flagversions'))
split(vis2, outputvis=final, datacolumn='corrected')
os.system('mv {} {}'.format(msfile + '.flagversions', final + '.flagversions'))
# Remove the interim ms files
if clearcache:
if os.path.exists(msfile):
os.system('rm -rf {}'.format(msfile))
if os.path.exists(msfile + '.flagversions'):
os.system('rm -rf {}'.format(msfile + '.flagversions'))
if os.path.exists(vis2):
os.system('rm -rf {}'.format(vis2))
if os.path.exists(vis2 + '.flagversions'):
os.system('rm -rf {}'.format(vis2 + '.flagversions'))
# Return the name of the selfcaled ms
return final, diskxmlfile
antenna='',
gaintable=gaintables,
gainfield=gainfield,
spwmap=spwmap,
interp=interp,
calwt=calwt)
cont_ms = ms[:-3] + "_continuum.ms"
if os.path.exists(cont_ms):
os.rename(cont_ms, "_" + cont_ms)
# flag edge channels
flagchans = ",".join(
["{0}:0~5;123~128".format(xx) for xx in Kmses[fullpathms].split(",")])
flagdata(vis=ms, mode='manual', spw=flagchans)
# flag CH3OH maser
#flagdata(vis=ms, mode='manual',
# spw='44054800170~44084179830Hz:44054800170~44084179830Hz')
tb.open(ms)
if 'CORRECTED_DATA' in tb.colnames():
datacolumn = 'corrected'
else:
datacolumn = 'data'
tb.close()
split(
vis=ms,
outputvis=cont_ms,
antenna='',
gaintable=gaintables,
gainfield=gainfield,
spwmap=spwmap,
interp=interp,
calwt=calwt)
cont_ms = ms[:-3] + "_continuum.ms"
if os.path.exists(cont_ms):
os.rename(cont_ms, "_" + cont_ms)
# flag edge channels
flagchans = ",".join(
["{0}:0~5;123~128".format(xx) for xx in Qmses[fullpathms].split(",")])
flagdata(vis=ms, mode='manual', spw=flagchans)
# flag CH3OH maser
flagdata(vis=ms,
mode='manual',
spw='44054800170~44084179830Hz:44054800170~44084179830Hz')
split(
vis=ms,
outputvis=cont_ms,
field=('Sgr B2 N Q,Sgr B2 NM Q,Sgr B2 MS Q,'
'Sgr B2 S Q,Sgr B2 DS1 Q,Sgr B2 DS2 Q,Sgr B2 DS3 Q'),
width=16,
spw=Qmses[fullpathms],
)
)
for ms in mses:
assert ms in Qmses
name = ms[:22]
fullpathms = '../' + ms
cont_ms = fullpathms[:-3] + "_continuum.ms"
if not os.path.exists(cont_ms):
myprint("Flagging & splitting {0}".format(fullpathms))
# ensure that both phase-calibrator and data are not flagged.
flagdata(vis=fullpathms,
mode='unflag',
field=('J1744-3116,Sgr B2 N Q,Sgr B2 NM Q,Sgr B2 MS Q,'
'Sgr B2 S Q,Sgr B2 DS1 Q,Sgr B2 DS2 Q,Sgr B2 DS3 Q'))
flagdata(vis=fullpathms, mode='manual', autocorr=True)
flagdata(vis=fullpathms, mode='quack', quackinterval=10)
# flag edge channels
flagchans = ",".join(
["{0}:0~5;123~128".format(xx) for xx in Qmses[ms].split(",")])
flagdata(vis=fullpathms, mode='manual', spw=flagchans)
# flag CH3OH maser
flagdata(vis=fullpathms,
mode='manual',
spw='44054800170~44084179830Hz:44054800170~44084179830Hz')
split(
from taskinit import casalog
def get_flagonline_base(fn):
comps = fn.split("/")
basedir = comps[1]
ms = comps[2]
msbase = ms.split("_")[0]
return os.path.join('..', 'reduction_scripts', basedir, msbase+".flagonline.txt")
for suffix in ("_continuum_split_for_selfcal", "_continuum", ""):
vis='../18A-229_2018_04_05_T10_59_52.640/18A-229.sb35258391.eb35265194.58213.344589120374{0}.ms'.format(suffix)
flagdata(vis=vis, mode='unflag')
flagdata(vis=vis, mode='quack', quackmode='beg', quackinterval=10.0)
# high and discrepant amplitudes
# conservative: flagdata(vis=vis, mode='manual', antenna='ea24&ea25', spw='23,24', correlation='LL')
# aggressive:
flagdata(vis=vis, mode='manual', antenna='ea24&ea25', spw='', correlation='LL')
# no solutions =(
flagdata(vis=vis, mode='manual', antenna='ea18', spw='', correlation='')
# EA19 is OK on NM, but not MS or N. ??bad pointing??
flagdata(vis=vis, mode='manual', antenna='ea19', spw='', correlation='')
flagdata(vis=vis, mode='manual', antenna='ea14', timerange='09:55:00~10:40:00')
flagfile = get_flagonline_base(vis)
if os.path.exists(flagfile):
flagdata(vis=vis, inpfile=flagfile, mode='list')
else:
raise ValueError("Flagfile {0} does not exist".format(flagfile))
caltable=caltable,
solint=solint,
combine=combine,
gaintype='G',
# use all fields field=field,
calmode=calmode,
gaintable=caltables,
minsnr=1.5,
spwmap=[[0] * nspws if calinfo[ii]['combine'] == 'spw' else []
for ii in range(len(caltables))],
interp='linear,linear',
solnorm=True)
if 'amp' in caltype:
# avoid extreme outliers: assume anything going more than 2x in either direction is wrong
flagdata(caltable,
mode='clip',
clipminmax=[0.5, 2.0],
datacolumn='CPARAM')
elif 'bandpass' in caltype:
bandpass(
vis=selfcal_vis,
caltable=caltable,
solint='{0},16ch'.format(solint),
combine=combine,
# use all fields # field=field,
#refant='ea07',
gaintable=caltables,
minsnr=1.5,
spwmap=[[0] * nspws if calinfo[ii]['combine'] == 'spw' else []
for ii in range(len(caltables))],
interp='linear,linear',
solnorm=True)
def disk_slfcal(vis, slfcaltbdir='./'):
''' Starting with the name of a calibrated ms (vis, which must have 'UDByyyymmdd' in the name)
add a model disk based on the solar disk size for that date and perform multiple selfcal
adjustments (two phase and one amplitude), and write out a final selfcaled database with
the disk subtracted. Returns the name of the final database.
'''
trange = ant_trange(vis)
# Use vis name to determine date, and hence number of bands
spw2band = np.array([0, 1] + range(4, 52))
defaultfreq = 1.1 + 0.325 * (spw2band + 0.5)
# Calculate the center frequency of each spectral window
if mstl.get_trange(vis)[0].mjd > 58536:
# After 2019 Feb 22, the band numbers changed to 1-52, and spw from 0-49
nbands = 52
freq = defaultfreq
else:
# Before 2019 Feb 22, the band numbers were 1-34, and spw from 0-30
nbands = 34
freq = np.hstack([[1.419], 2.0 + 0.5 * (np.arange(32)) + (0.5 - 0.081)])
slashdate = trange[:10]
# Verify that the vis is not in the current working directory
if os.getcwd() == os.path.dirname(vis):
print('Cannot copy vis file onto itself.')
print('Please change to a different working directory')
return None
# Copy original ms to local directory
if os.path.exists(os.path.basename(vis)):
shutil.rmtree(os.path.basename(vis))
print('Copy {} to working directory {}.'.format(vis, os.getcwd()))
shutil.copytree(vis, os.path.basename(vis))
vis = os.path.basename(vis)
clearcal(vis)
## automaticaly flag impossibly high amplitudes
flagdata(vis=vis, mode="tfcrop", spw='', correlation='ABS_XX', action='apply', display='',
timecutoff=3.0, freqcutoff=2.0, maxnpieces=2, flagbackup=True)
# Default disk size measured for 2019/09/03
# todo add monthly fitting procedure for the disk size and flux density
defaultsize = np.array([990.6, 989.4, 988.2, 987.1, 986.0, 984.9, 983.8, 982.7, 981.7, 980.7,
979.7, 978.8, 977.8, 976.9, 976.0, 975.2, 974.3, 973.5, 972.7, 972.0,
971.2, 970.5, 969.8, 969.1, 968.5, 967.8, 967.2, 966.7, 966.1, 965.6,
965.1, 964.6, 964.1, 963.7, 963.3, 962.9, 962.5, 962.1, 961.8, 961.5,
961.3, 961.0, 960.8, 960.6, 960.4, 960.2, 960.1, 960.0, 959.9, 959.8])
# Get current solar distance and modify the default size accordingly
fac = eph.get_sunearth_distance('2019/09/03') / eph.get_sunearth_distance(slashdate)
newsize = defaultsize * fac.to_value()
if nbands == 34:
# Interpolate size to 31 spectal windows
newsize = np.polyval(np.polyfit(defaultfreq, newsize, 5), freq)
dsize = np.array([str(i)[:5] + 'arcsec' for i in newsize], dtype='S12')
# These are nominal flux densities * 2, determined on 2019/09/03
defaultfdens = np.array([891282, 954570, 1173229, 1245433, 1373730, 1506802,
1613253, 1702751, 1800721, 1946756, 2096020, 2243951,
2367362, 2525968, 2699795, 2861604, 3054829, 3220450,
3404182, 3602625, 3794312, 3962926, 4164667, 4360683,
4575677, 4767210, 4972824, 5211717, 5444632, 5648266,
5926634, 6144249, 6339863, 6598018, 6802707, 7016012,
7258929, 7454951, 7742816, 7948976, 8203206, 8411834,
8656720, 8908130, 9087766, 9410760, 9571365, 9827078,
10023598, 8896671])
fdens = defaultfdens
if nbands == 34:
# Interpolate size to 31 spectal windows
fdens = np.polyval(np.polyfit(defaultfreq, fdens, 5), freq)
diskxmlfile = vis + '.SOLDISK.xml'
# Insert the disk model (msfile is the same as vis, and will be used as the "original" vis file name)
msfile = insertdiskmodel(vis, dsize=dsize, fdens=fdens, xmlfile=diskxmlfile)
tdate = mstl.get_trange(vis)[0].datetime.strftime('%Y%m%d')
caltb = os.path.join(slfcaltbdir, tdate + '_1.pha')
if os.path.exists(caltb):
os.system('rm -rf {}'.format(caltb))
# Phase selfcal on the disk using solution interval "infinite"
gaincal(vis=msfile, caltable=caltb, selectdata=True, uvrange="<3.0Klambda", antenna="0~12&0~12", solint="inf",
combine="scan",
refant="0", refantmode="flex", minsnr=1.0, gaintype="G", calmode="p", append=False)
applycal(vis=msfile, selectdata=True, antenna="0~12", gaintable=caltb, interp="nearest", calwt=False,
applymode="calonly")
# Split corrected data and model to a new ms for round 2 of phase selfcal
vis1 = 'slf_' + msfile
if os.path.exists(vis1):
os.system('rm -rf {}'.format(vis1))
mstl.splitX(msfile, outputvis=vis1, datacolumn="corrected", datacolumn2="model_data")
caltb = os.path.join(slfcaltbdir, tdate + '_2.pha')
if os.path.exists(caltb):
os.system('rm -rf {}'.format(caltb))
# Second round of phase selfcal on the disk using solution interval "1min"
gaincal(vis=vis1, caltable=caltb, selectdata=True, uvrange="<3.0Klambda", antenna="0~12&0~12", solint="1min",
combine="scan",
refant="0", refantmode="flex", minsnr=1.0, gaintype="G", calmode="p", append=False)
applycal(vis=vis1, selectdata=True, antenna="0~12", gaintable=caltb, interp="nearest", calwt=False,
applymode="calonly")
# Split corrected data and model to a new ms
vis2 = 'slf2_' + msfile
if os.path.exists(vis2):
os.system('rm -rf {}'.format(vis2))
mstl.splitX(vis, outputvis=vis2, datacolumn="corrected", datacolumn2="model_data")
caltb = os.path.join(slfcaltbdir, tdate + '_3.amp')
if os.path.exists(caltb):
os.system('rm -rf {}'.format(caltb))
# Final round of amplitude selfcal with 1-h solution interval (restrict to 16-24 UT)
gaincal(vis=vis2, caltable=caltb, selectdata=True, uvrange=">0.1Klambda", antenna="0~12&0~12",
timerange=trange,
solint="60min", combine="scan", refant="10", refantmode="flex", minsnr=1.0, gaintype="G", calmode="a",
append=False)
applycal(vis=vis2, selectdata=True, antenna="0~12", gaintable=caltb, interp="nearest", calwt=False,
applymode="calonly")
# Split out corrected data and model and do uvsub
vis3 = 'slf3_' + msfile
if os.path.exists(vis3):
os.system('rm -rf {}'.format(vis3))
mstl.splitX(vis, outputvis=vis3, datacolumn="corrected", datacolumn2="model_data")
uvsub(vis=vis3, reverse=False)
# Final split to
final = 'final_' + msfile
if os.path.exists(final):
os.system('rm -rf {}'.format(final))
split(vis3, outputvis=final, datacolumn='corrected')
# Remove the interim ms files
shutil.rmtree(vis)
shutil.rmtree(vis1)
shutil.rmtree(vis2)
shutil.rmtree(vis3)
# Return the name of the selfcaled ms
return final, diskxmlfile
('2,4,5,6,7,8,9,11,12,13,14,15,16', 'A1C1'),
("17,18,21,22,23,24,25,27,28,29,30,31", 'A2C2'),
("33,34,35,36,38,41,43,44,46,47,48", 'B1D1'),
("49,51,52,53,54,55,56,57,58,59,60,62,63,64", 'B2D2'),
]
for ms in mses:
assert ms in Qmses
name = ms[:22]
ms = '../' + ms
flagdata(vis=ms,
mode='unflag',
field=('J1744-3116,Sgr B2 N Q,Sgr B2 NM Q,SgrB2 MS Q,'
'Sgr B2 S Q,Sgr B2 DS1 Q,Sgr B2 DS2 Q,Sgr B2 DS3 Q'))
flagdata(vis=ms, mode='manual', autocorr=True)
ft(vis=ms,
field='Sgr B2 N Q,Sgr B2 NM Q,Sgr B2 MS Q',
spw='',
model='../reduction_scripts/DePree_NM_regridNM.image',
nterms=1)
#splitvis = '{0}.1744-3116.split.ms'.format(name)
## *do not* split out spw to avoid re-mapping later
#split(vis="../"+ms, outputvis=splitvis,
# datacolumn='corrected',
# width=512, # averaged each spw to 1 channel
# Do this single case because it was being ignored earlier
# (uncomment this line to just do one field)
#selfcal_mses = {'03_06_T12': selfcal_mses['03_06_T12']}
for msname, cont_vis in selfcal_mses.items():
# this is OK because there should be no corrected column
#clearcal(vis=cont_vis, addmodel=True)
myprint("Beginning work on {0}".format(msname))
if msname == '03_06':
if 'T12' in cont_vis:
msname = '03_06_T12'
else:
msname = '03_06_T01'
flagsummary = flagdata(vis=cont_vis, mode='summary')
if flagsummary['total'] == flagsummary['flagged']:
raise ValueError("MS {0} is flagged out".format(cont_vis))
elif flagsummary['flagged'] / flagsummary['total'] > 0.7:
raise ValueError("MS {0} is >70% flagged out".format(cont_vis))
imagename = '18A-229_{msname}_Q_mosaic_selfcal_iter0_dirty'.format(msname=msname)
if not (os.path.exists(imagename+".image.tt0.pbcor") and os.path.exists(imagename+".image.tt0.pbcor.fits")):
# do a full-mosaic clean to enable mask creation
tclean(
vis=cont_vis,
spw='',
field="Sgr B2 N Q,Sgr B2 NM Q,Sgr B2 MS Q,Sgr B2 S Q",
phasecenter='J2000 17h47m19.693 -28d23m11.527',
imsize=[9000,9000],
cell='0.02arcsec',