def concateovsa(msname,
msfiles,
visprefix='./',
doclearcal=True,
keep_orig_ms=False,
cols2rm=["MODEL_DATA", "CORRECTED_DATA"]):
concatvis = visprefix + msname
msfiles_ = []
for idx, ll in enumerate(msfiles):
if str(ll).endswith('/'):
msfiles[idx] = str(ll)[:-1]
if doclearcal:
print 'Warning: Corrected column in the input ms file will be cleared!!!'
for ll in msfiles:
clearcal(vis=str(ll), addmodel=True)
else:
try:
tmpdir = visprefix + '/tmp_ms/'
if not os.path.exists(tmpdir):
os.makedirs(tmpdir)
for ll in msfiles:
msfile_ = tmpdir + os.path.basename(str(ll))
msfiles_.append(msfile_)
split(vis=str(ll), outputvis=msfile_, datacolumn='corrected')
clearcal(vis=msfile_, addmodel=True)
except:
print 'Warning: Corrected column not found in the input ms file.'
msfiles_ = msfiles
if msfiles_:
concat(vis=msfiles_, concatvis=concatvis, timesort=True)
else:
concat(vis=msfiles, concatvis=concatvis, timesort=True)
# Change all observation ids to be the same (zero)
tb.open(concatvis + '/OBSERVATION', nomodify=False)
nobs = tb.nrows()
tim0 = tb.getcell('TIME_RANGE', 0)[0]
tim1 = tb.getcell('TIME_RANGE', nobs - 1)[1]
tb.removerows([i + 1 for i in range(nobs - 1)])
tb.putcell('TIME_RANGE', 0, [tim0, tim1])
tb.close()
tb.open(concatvis, nomodify=False)
obsid = tb.getcol('OBSERVATION_ID')
newobsid = np.zeros(len(obsid), dtype='int')
tb.putcol('OBSERVATION_ID', newobsid)
colnames = tb.colnames()
for l in range(len(cols2rm)):
if cols2rm[l] in colnames:
try:
tb.removecols(cols2rm[l])
except:
pass
tb.close()
if msfiles_ != [] and msfiles_ != msfiles:
for ll in msfiles_:
os.system('rm -rf {}'.format(ll))
if not keep_orig_ms:
for ll in msfiles:
os.system('rm -rf {}'.format(ll))
def splitX(vis, datacolumn2='MODEL_DATA', **kwargs):
import os
from clearcal_cli import clearcal_cli as clearcal
from split_cli import split_cli as split
'''
:param msfile:
:param datacolumn:
:param datacolumn2:
:return:
'''
kwargs2 = kwargs.copy()
datacolumn2 = datacolumn2.upper()
outmsfile = kwargs['outputvis']
if outmsfile.endswith('/'):
outmsfile = outmsfile[:-1]
if os.path.exists(outmsfile):
os.system('rm -rf {}'.format(outmsfile))
if os.path.exists('{}.flagversions'.format(outmsfile)):
os.system('rm -rf {}.flagversions'.format(outmsfile))
split(vis, **kwargs)
for k in ['datacolumn', 'outputvis']:
if k in kwargs2:
kwargs2.pop(k)
kwargs2['outputvis'] = 'tmpms.ms'
kwargs2['datacolumn'] = datacolumn2.replace('_DATA', '')
if os.path.exists('tmpms.ms'):
os.system('rm -rf tmpms.ms')
split(vis, **kwargs2)
tb.open('tmpms.ms')
nrows = tb.nrows()
data = []
for row in tqdm(range(nrows),
desc='getting {} column'.format(datacolumn2),
ascii=True):
data.append(tb.getcell('DATA', row))
tb.close()
clearcal(outmsfile, addmodel=True)
tb.open(outmsfile, nomodify=False)
for row in tqdm(range(nrows),
desc='writing {} column'.format(datacolumn2),
ascii=True):
tb.putcell(datacolumn2, row, data[row])
tb.close()
os.system('rm -rf tmpms.ms')
return outmsfile
casalog.post(str(x), origin='singlefield')
mses = list(Qmses.keys())
fullpath_mses = [
'../' + ms[:-3] + "_continuum_split_for_selfcal.ms" for ms in mses
if ms in Qmses
]
cont_vis = []
for ms in fullpath_mses:
splitagain = ms[:-3] + "_wtermmerge.ms"
myprint("{0} -> {1}".format(ms, splitagain))
if not os.path.exists(splitagain):
assert split(vis=ms, outputvis=splitagain, datacolumn='corrected')
cont_vis.append(splitagain)
# imagename = '18A-229_Q_mosaic_selfcal_wtermmerge_iter0_dirty'
# if not os.path.exists(imagename+".image.tt0.pbcor"):
# # do a full-mosaic clean to enable mask creation
# tclean(
# vis=cont_vis,
# spw='',
# field="Sgr B2 N Q,Sgr B2 NM Q,Sgr B2 MS Q,Sgr B2 S Q",
# phasecenter='J2000 17h47m19.693 -28d23m11.527',
# imsize=[9000,9000],
# cell='0.02arcsec',
# imagename=imagename,
# niter=0,
# threshold='1000mJy',
def feature_slfcal(vis, niter=200, spws=['0~1', '2~5', '6~10', '11~20', '21~30', '31~49'], slfcaltbdir='./',
bright=None, pols='XX'):
''' Uses images from disk-selfcaled data as model for further self-calibration of outer antennas.
This is only a good idea if there are bright active regions that provide strong signal on the
londer baselines.
'''
trange = ant_trange(vis)
if bright is None:
bright = [True] * len(spws)
# Insert model into ms and do "inf" gaincal, appending to table each subsequent time
if os.path.exists('images_init'):
os.system('rm -rf images_init')
os.system('mv images images_init')
clearcal(vis, addmodel=True)
fd_images(vis, cleanup=False, niter=niter, spws=spws, bright=bright) # Does shallow clean for selfcal purposes
tdate = mstl.get_trange(vis)[0].datetime.strftime('%Y%m%d')
caltb = os.path.join(slfcaltbdir, tdate + '_d1.pha')
if os.path.exists(caltb):
os.system('rm -rf {}*'.format(caltb))
for s, sp in enumerate(spws):
if bright[s]:
spwstr = '-'.join(['{:02d}'.format(int(sp_)) for sp_ in sp.split('~')])
imname = "images/briggs" + spwstr + '.model'
if sp == '31~49':
# The high-band image is only made to band 43, so adjust the name
imname = 'images/briggs31-43.model'
ft(vis=vis, spw=sp, model=imname, usescratch=True, incremental=True)
if pols == 'XXYY':
mstl.modeltransfer(vis, spw=sp)
if pols == 'XXYY':
mstl.gaincalXY(vis=vis, caltable=caltb, pols=pols, selectdata=True, timerange=trange, uvrange='>1.5Klambda',
combine="scan", antenna='0~12&0~12', refant='0', refantmode="strict", solint='inf', gaintype='G',
minsnr=1.0, calmode='p', append=False)
else:
gaincal(vis=vis, caltable=caltb, selectdata=True, timerange=trange, uvrange='>1.5Klambda',
combine="scan", antenna='0~12&0~12', refant='0', refantmode="strict", solint='inf', gaintype='G',
minsnr=1.0, calmode='p', append=False)
# Apply the corrections to the data and split to a new ms
applycal(vis=vis, selectdata=True, antenna="0~12", gaintable=caltb, interp="linear", calwt=False,
applymode="calonly")
vis1 = 'dslf1_' + vis
if os.path.exists(vis1):
os.system('rm -rf {}'.format(vis1))
split(vis, outputvis=vis1, datacolumn="corrected")
caltb = os.path.join(slfcaltbdir, tdate + '_d2.pha')
if os.path.exists(caltb):
os.system('rm -rf {}*'.format(caltb))
# Move the existing images directory so that a new one will be created
if os.path.exists('images_ftcal_rnd1'):
os.system('rm -rf images_ftcal_rnd1')
# shutil.move('images', 'old_images2')
os.system('mv images images_ftcal_rnd1')
# Make new model images for another round of selfcal
fd_images(vis1, cleanup=False, niter=niter, spws=spws, bright=bright)
for s, sp in enumerate(spws):
if bright[s]:
spwstr = '-'.join(['{:02d}'.format(int(sp_)) for sp_ in sp.split('~')])
imname = "images/briggs" + spwstr + '.model'
if sp == '31~49':
# The high-band image is only made to band 43, so adjust the name
imname = 'images/briggs31-43.model'
ft(vis=vis1, spw=sp, model=imname, usescratch=True)
if pols == 'XXYY':
mstl.modeltransfer(vis1, spw=sp)
if pols == 'XXYY':
mstl.gaincalXY(vis=vis1, caltable=caltb, pols=pols, selectdata=True, timerange=trange, uvrange='>1.5Klambda',
combine="scan", antenna='0~12&0~12', refant='0', solint='10min', refantmode="strict",
gaintype='G', minsnr=1.0, calmode='p', append=False)
else:
gaincal(vis=vis1, caltable=caltb, selectdata=True, timerange=trange, uvrange='>1.5Klambda',
combine="scan", antenna='0~12&0~12', refant='0', solint='10min', refantmode="strict", gaintype='G',
minsnr=1.0, calmode='p', append=False)
# Apply the corrections to the data and split to a new ms
applycal(vis=vis1, selectdata=True, antenna="0~12", gaintable=caltb, interp="linear", calwt=False,
applymode="calonly")
vis2 = 'dslf2_' + vis
if os.path.exists(vis2):
os.system('rm -rf {}'.format(vis2))
mstl.splitX(vis1, outputvis=vis2, datacolumn="corrected", datacolumn2="model_data")
if os.path.exists('images_ftcal_rnd2'):
os.system('rm -rf images_ftcal_rnd2')
os.system('mv images images_ftcal_rnd2')
return vis2
def disk_slfcal(vis, slfcaltbdir='./', active=False, clearcache=False, pols='XX'):
''' Starting with the name of a calibrated ms (vis, which must have 'UDByyyymmdd' in the name)
add a model disk based on the solar disk size for that date and perform multiple selfcal
adjustments (two phase and one amplitude), and write out a final selfcaled database with
the disk subtracted. Returns the name of the final database.
'''
trange = ant_trange(vis)
if vis.endswith('/'):
vis = vis[:-1]
# Use vis name to determine date, and hence number of bands
spw2band = np.array([0, 1] + range(4, 52))
defaultfreq = 1.1 + 0.325 * (spw2band + 0.5)
# Calculate the center frequency of each spectral window
if mstl.get_trange(vis)[0].mjd > 58536:
# After 2019 Feb 22, the band numbers changed to 1-52, and spw from 0-49
nbands = 52
freq = defaultfreq
else:
# Before 2019 Feb 22, the band numbers were 1-34, and spw from 0-30
nbands = 34
freq = 1.419 + np.arange(nbands) / 2.
caltbs = []
slashdate = trange[:10]
# Verify that the vis is not in the current working directory
'''
if os.getcwd() == os.path.dirname(vis):
print('Cannot copy vis file onto itself.')
print('Please change to a different working directory')
return None
# Copy original ms to local directory
if os.path.exists(os.path.basename(vis)):
shutil.rmtree(os.path.basename(vis))
print('Copy {} to working directory {}.'.format(vis, os.getcwd()))
shutil.copytree(vis, os.path.basename(vis))
vis = os.path.basename(vis)
'''
if not active:
clearcal(vis)
flagmanager(vis, mode='save', versionname='with-RFI-or-BURSTS')
## automaticaly flag any high amplitudes from flares or RFI
flagdata(vis=vis, mode="tfcrop", spw='', action='apply', display='',
timecutoff=3.0, freqcutoff=2.0, maxnpieces=2, flagbackup=False)
flagmanager(vis, mode='save', versionname='without-RFI-or-BURSTS')
dsize, fdens = calc_diskmodel(slashdate, nbands, freq, defaultfreq)
diskxmlfile = vis + '.SOLDISK.xml'
# Insert the disk model (msfile is the same as vis, and will be used as the "original" vis file name)
msfile, diskim = insertdiskmodel(vis, dsize=dsize, fdens=fdens, xmlfile=diskxmlfile, active=active)
if pols == 'XXYY':
caltbs_ = {'XX': [], 'YY': []}
pols_ = ['XX', 'YY']
msfileXY = {}
for pol in pols_:
msfileXY[pol] = '.'.join([msfile, pol])
if os.path.exists(msfileXY[pol]):
os.system('rm -rf {}'.format(msfileXY[pol]))
mstl.splitX(vis=msfile, outputvis=msfileXY[pol], correlation=pol, datacolumn='data',
datacolumn2='MODEL_DATA')
tdate = mstl.get_trange(msfile)[0].datetime.strftime('%Y%m%d')
caltb = os.path.join(slfcaltbdir, tdate + '_1.pha')
if os.path.exists(caltb):
os.system('rm -rf {}*'.format(caltb))
if pols == 'XXYY':
mstl.gaincalXY(vis=msfile, caltable=caltb, pols=pols, msfileXY=msfileXY, selectdata=True, uvrange="",
antenna="0~12&0~12", solint="inf",
combine="scan", refant="0", refantmode="strict", minsnr=1.0, gaintype="G", calmode="p",
append=False)
for pol in pols_:
caltb_ = '.'.join([caltb, pol])
caltbs_[pol].append(caltb_)
else:
gaincal(vis=msfile, caltable=caltb, selectdata=True, uvrange="", antenna="0~12&0~12", solint="inf",
combine="scan", refant="0", refantmode="strict", minsnr=1.0, gaintype="G", calmode="p", append=False)
caltbs.append(caltb)
caltb = os.path.join(slfcaltbdir, tdate + '_2.pha')
if os.path.exists(caltb):
os.system('rm -rf {}*'.format(caltb))
# Second round of phase selfcal on the disk using solution interval "1min"
if pols == 'XXYY':
mstl.gaincalXY(vis=msfile, caltable=caltb, pols=pols, msfileXY=msfileXY, gaintableXY=caltbs_,
selectdata=True, uvrange="", antenna="0~12&0~12",
solint="10min",
combine="scan", interp="linear",
refant="0", refantmode="strict", minsnr=1.0, gaintype="G", calmode="p", append=False)
for pol in pols_:
caltb_ = '.'.join([caltb, pol])
caltbs_[pol].append(caltb_)
else:
gaincal(vis=msfile, caltable=caltb, selectdata=True, uvrange="", antenna="0~12&0~12", solint="10min",
combine="scan", gaintable=caltbs, interp="linear",
refant="0", refantmode="strict", minsnr=1.0, gaintype="G", calmode="p", append=False)
caltbs.append(caltb)
caltb = os.path.join(slfcaltbdir, tdate + '_3.amp')
if os.path.exists(caltb):
os.system('rm -rf {}*'.format(caltb))
# Final round of amplitude selfcal with 1-h solution interval (restrict to 16-24 UT)
if pols == 'XXYY':
mstl.gaincalXY(vis=msfile, caltable=caltb, pols=pols, msfileXY=msfileXY, gaintableXY=caltbs_,
selectdata=True, uvrange="", antenna="0~12&0~12",
timerange=trange, interp="linear",
solint="60min", combine="scan", refant="10", refantmode="flex", minsnr=1.0, gaintype="G",
calmode="a",
append=False)
for pol in pols_:
caltb_ = '.'.join([caltb, pol])
caltbs_[pol].append(caltb_)
else:
gaincal(vis=msfile, caltable=caltb, selectdata=True, uvrange="", antenna="0~12&0~12",
timerange=trange, gaintable=caltbs, interp="linear",
solint="60min", combine="scan", refant="10", refantmode="flex", minsnr=1.0, gaintype="G", calmode="a",
append=False)
mstl.flagcaltboutliers(caltb, limit=[0.125, 8.0])
# mstl.flagcaltboutliers(caltb, limit=[0.5, 2.0])
caltbs.append(caltb)
# Split out corrected data and model and do uvsub
vis2 = 'slf3_' + msfile
if os.path.exists(vis2):
os.system('rm -rf {}'.format(vis2))
if os.path.exists(vis2 + '.flagversions'):
os.system('rm -rf {}'.format(vis2 + '.flagversions'))
# flagmanager(msfile, mode='restore', versionname='with-RFI-or-BURSTS')
clearcal(msfile)
applycal(vis=msfile, selectdata=True, antenna="0~12", gaintable=caltbs, interp="linear", calwt=False,
applymode="calonly")
split(msfile, outputvis=vis2, datacolumn="corrected")
for sp, dkim in tqdm(enumerate(diskim), desc='Inserting disk model', ascii=True):
ft(vis=vis2, spw=str(sp), field='', model=str(dkim), nterms=1,
reffreq="", complist="", incremental=False, usescratch=True)
# mstl.modeltransfer(msfile, spw='{}'.format(sp))
uvsub(vis=vis2, reverse=False)
# Final split to
final = 'final_' + msfile
if os.path.exists(final):
os.system('rm -rf {}'.format(final))
if os.path.exists(final + '.flagversions'):
os.system('rm -rf {}'.format(final + '.flagversions'))
split(vis2, outputvis=final, datacolumn='corrected')
os.system('mv {} {}'.format(msfile + '.flagversions', final + '.flagversions'))
# Remove the interim ms files
if clearcache:
if os.path.exists(msfile):
os.system('rm -rf {}'.format(msfile))
if os.path.exists(msfile + '.flagversions'):
os.system('rm -rf {}'.format(msfile + '.flagversions'))
if os.path.exists(vis2):
os.system('rm -rf {}'.format(vis2))
if os.path.exists(vis2 + '.flagversions'):
os.system('rm -rf {}'.format(vis2 + '.flagversions'))
# Return the name of the selfcaled ms
return final, diskxmlfile
def disk_slfcal(vis, slfcaltbdir='./'):
''' Starting with the name of a calibrated ms (vis, which must have 'UDByyyymmdd' in the name)
add a model disk based on the solar disk size for that date and perform multiple selfcal
adjustments (two phase and one amplitude), and write out a final selfcaled database with
the disk subtracted. Returns the name of the final database.
'''
trange = ant_trange(vis)
# Use vis name to determine date, and hence number of bands
spw2band = np.array([0, 1] + range(4, 52))
defaultfreq = 1.1 + 0.325 * (spw2band + 0.5)
# Calculate the center frequency of each spectral window
if mstl.get_trange(vis)[0].mjd > 58536:
# After 2019 Feb 22, the band numbers changed to 1-52, and spw from 0-49
nbands = 52
freq = defaultfreq
else:
# Before 2019 Feb 22, the band numbers were 1-34, and spw from 0-30
nbands = 34
freq = np.hstack([[1.419], 2.0 + 0.5 * (np.arange(32)) + (0.5 - 0.081)])
slashdate = trange[:10]
# Verify that the vis is not in the current working directory
if os.getcwd() == os.path.dirname(vis):
print('Cannot copy vis file onto itself.')
print('Please change to a different working directory')
return None
# Copy original ms to local directory
if os.path.exists(os.path.basename(vis)):
shutil.rmtree(os.path.basename(vis))
print('Copy {} to working directory {}.'.format(vis, os.getcwd()))
shutil.copytree(vis, os.path.basename(vis))
vis = os.path.basename(vis)
clearcal(vis)
## automaticaly flag impossibly high amplitudes
flagdata(vis=vis, mode="tfcrop", spw='', correlation='ABS_XX', action='apply', display='',
timecutoff=3.0, freqcutoff=2.0, maxnpieces=2, flagbackup=True)
# Default disk size measured for 2019/09/03
# todo add monthly fitting procedure for the disk size and flux density
defaultsize = np.array([990.6, 989.4, 988.2, 987.1, 986.0, 984.9, 983.8, 982.7, 981.7, 980.7,
979.7, 978.8, 977.8, 976.9, 976.0, 975.2, 974.3, 973.5, 972.7, 972.0,
971.2, 970.5, 969.8, 969.1, 968.5, 967.8, 967.2, 966.7, 966.1, 965.6,
965.1, 964.6, 964.1, 963.7, 963.3, 962.9, 962.5, 962.1, 961.8, 961.5,
961.3, 961.0, 960.8, 960.6, 960.4, 960.2, 960.1, 960.0, 959.9, 959.8])
# Get current solar distance and modify the default size accordingly
fac = eph.get_sunearth_distance('2019/09/03') / eph.get_sunearth_distance(slashdate)
newsize = defaultsize * fac.to_value()
if nbands == 34:
# Interpolate size to 31 spectal windows
newsize = np.polyval(np.polyfit(defaultfreq, newsize, 5), freq)
dsize = np.array([str(i)[:5] + 'arcsec' for i in newsize], dtype='S12')
# These are nominal flux densities * 2, determined on 2019/09/03
defaultfdens = np.array([891282, 954570, 1173229, 1245433, 1373730, 1506802,
1613253, 1702751, 1800721, 1946756, 2096020, 2243951,
2367362, 2525968, 2699795, 2861604, 3054829, 3220450,
3404182, 3602625, 3794312, 3962926, 4164667, 4360683,
4575677, 4767210, 4972824, 5211717, 5444632, 5648266,
5926634, 6144249, 6339863, 6598018, 6802707, 7016012,
7258929, 7454951, 7742816, 7948976, 8203206, 8411834,
8656720, 8908130, 9087766, 9410760, 9571365, 9827078,
10023598, 8896671])
fdens = defaultfdens
if nbands == 34:
# Interpolate size to 31 spectal windows
fdens = np.polyval(np.polyfit(defaultfreq, fdens, 5), freq)
diskxmlfile = vis + '.SOLDISK.xml'
# Insert the disk model (msfile is the same as vis, and will be used as the "original" vis file name)
msfile = insertdiskmodel(vis, dsize=dsize, fdens=fdens, xmlfile=diskxmlfile)
tdate = mstl.get_trange(vis)[0].datetime.strftime('%Y%m%d')
caltb = os.path.join(slfcaltbdir, tdate + '_1.pha')
if os.path.exists(caltb):
os.system('rm -rf {}'.format(caltb))
# Phase selfcal on the disk using solution interval "infinite"
gaincal(vis=msfile, caltable=caltb, selectdata=True, uvrange="<3.0Klambda", antenna="0~12&0~12", solint="inf",
combine="scan",
refant="0", refantmode="flex", minsnr=1.0, gaintype="G", calmode="p", append=False)
applycal(vis=msfile, selectdata=True, antenna="0~12", gaintable=caltb, interp="nearest", calwt=False,
applymode="calonly")
# Split corrected data and model to a new ms for round 2 of phase selfcal
vis1 = 'slf_' + msfile
if os.path.exists(vis1):
os.system('rm -rf {}'.format(vis1))
mstl.splitX(msfile, outputvis=vis1, datacolumn="corrected", datacolumn2="model_data")
caltb = os.path.join(slfcaltbdir, tdate + '_2.pha')
if os.path.exists(caltb):
os.system('rm -rf {}'.format(caltb))
# Second round of phase selfcal on the disk using solution interval "1min"
gaincal(vis=vis1, caltable=caltb, selectdata=True, uvrange="<3.0Klambda", antenna="0~12&0~12", solint="1min",
combine="scan",
refant="0", refantmode="flex", minsnr=1.0, gaintype="G", calmode="p", append=False)
applycal(vis=vis1, selectdata=True, antenna="0~12", gaintable=caltb, interp="nearest", calwt=False,
applymode="calonly")
# Split corrected data and model to a new ms
vis2 = 'slf2_' + msfile
if os.path.exists(vis2):
os.system('rm -rf {}'.format(vis2))
mstl.splitX(vis, outputvis=vis2, datacolumn="corrected", datacolumn2="model_data")
caltb = os.path.join(slfcaltbdir, tdate + '_3.amp')
if os.path.exists(caltb):
os.system('rm -rf {}'.format(caltb))
# Final round of amplitude selfcal with 1-h solution interval (restrict to 16-24 UT)
gaincal(vis=vis2, caltable=caltb, selectdata=True, uvrange=">0.1Klambda", antenna="0~12&0~12",
timerange=trange,
solint="60min", combine="scan", refant="10", refantmode="flex", minsnr=1.0, gaintype="G", calmode="a",
append=False)
applycal(vis=vis2, selectdata=True, antenna="0~12", gaintable=caltb, interp="nearest", calwt=False,
applymode="calonly")
# Split out corrected data and model and do uvsub
vis3 = 'slf3_' + msfile
if os.path.exists(vis3):
os.system('rm -rf {}'.format(vis3))
mstl.splitX(vis, outputvis=vis3, datacolumn="corrected", datacolumn2="model_data")
uvsub(vis=vis3, reverse=False)
# Final split to
final = 'final_' + msfile
if os.path.exists(final):
os.system('rm -rf {}'.format(final))
split(vis3, outputvis=final, datacolumn='corrected')
# Remove the interim ms files
shutil.rmtree(vis)
shutil.rmtree(vis1)
shutil.rmtree(vis2)
shutil.rmtree(vis3)
# Return the name of the selfcaled ms
return final, diskxmlfile
def mk_qlook_image(trange,
doimport=False,
docalib=False,
ncpu=10,
twidth=12,
stokes=None,
antenna='0~12',
lowcutoff_freq=3.7,
imagedir=None,
spws=['1~5', '6~10', '11~15', '16~25'],
toTb=True,
overwrite=True,
doslfcal=False,
verbose=False):
'''
trange: can be 1) a single Time() object: use the entire day
2) a range of Time(), e.g., Time(['2017-08-01 00:00','2017-08-01 23:00'])
3) a single or a list of UDBms file(s)
4) None -- use current date Time.now()
'''
antenna0 = antenna
if type(trange) == Time:
mslist = trange2ms(trange=trange, doimport=doimport)
vis = mslist['ms']
tsts = [l.to_datetime() for l in mslist['tstlist']]
if type(trange) == str:
try:
date = Time(trange)
mslist = trange2ms(trange=trange, doimport=doimport)
vis = mslist['ms']
tsts = [l.to_datetime() for l in mslist['tstlist']]
except:
vis = [trange]
tsts = []
for v in vis:
tb.open(v + '/OBSERVATION')
tsts.append(
Time(tb.getcell('TIME_RANGE')[0] / 24 / 3600,
format='mjd').datetime)
tb.close()
subdir = [tst.strftime("%Y/%m/%d/") for tst in tsts]
for idx, f in enumerate(vis):
if f[-1] == '/':
vis[idx] = f[:-1]
if not stokes:
stokes = 'XX'
if not imagedir:
imagedir = './'
imres = {
'Succeeded': [],
'BeginTime': [],
'EndTime': [],
'ImageName': [],
'Spw': [],
'Vis': [],
'Synoptic': {
'Succeeded': [],
'BeginTime': [],
'EndTime': [],
'ImageName': [],
'Spw': [],
'Vis': []
}
}
for n, msfile in enumerate(vis):
msfilebs = os.path.basename(msfile)
imdir = imagedir + subdir[n]
if not os.path.exists(imdir):
os.makedirs(imdir)
if doslfcal:
slfcalms = './' + msfilebs + '.xx'
split(msfile,
outputvis=slfcalms,
datacolumn='corrected',
correlation='XX')
cfreqs = getspwfreq(msfile)
for spw in spws:
antenna = antenna0
if spw == '':
continue
spwran = [s.zfill(2) for s in spw.split('~')]
freqran = [cfreqs[int(s)] for s in spw.split('~')]
cfreq = np.mean(freqran)
bmsz = max(150. / cfreq, 20.)
uvrange = '<10klambda'
if doslfcal:
slfcal_img = './' + msfilebs + '.slf.spw' + spw.replace(
'~', '-') + '.slfimg'
slfcal_tb = './' + msfilebs + '.slf.spw' + spw.replace(
'~', '-') + '.slftb'
try:
clean(vis=slfcalms,
antenna=antenna,
imagename=slfcal_img,
spw=spw,
mode='mfs',
timerange='',
imagermode='csclean',
psfmode='clark',
imsize=[512, 512],
cell=['5arcsec'],
niter=100,
gain=0.05,
stokes='I',
weighting='natural',
restoringbeam=[str(bmsz) + 'arcsec'],
pbcor=False,
interactive=False,
usescratch=True)
except:
print('error in cleaning spw: ' + spw)
break
gaincal(vis=slfcalms,
refant='0',
antenna=antenna,
caltable=slfcal_tb,
spw=spw,
uvrange='',
gaintable=[],
selectdata=True,
timerange='',
solint='600s',
gaintype='G',
calmode='p',
combine='',
minblperant=3,
minsnr=2,
append=False)
if not os.path.exists(slfcal_tb):
print('No solution found in spw: ' + spw)
break
else:
clearcal(slfcalms)
delmod(slfcalms)
applycal(vis=slfcalms,
gaintable=[slfcal_tb],
spw=spw,
selectdata=True,
antenna=antenna,
interp='nearest',
flagbackup=False,
applymode='calonly',
calwt=False)
msfile = slfcalms
imsize = 512
cell = ['5arcsec']
if len(spwran) == 2:
spwstr = spwran[0] + '~' + spwran[1]
else:
spwstr = spwran[0]
restoringbeam = ['{0:.1f}arcsec'.format(bmsz)]
imagesuffix = '.spw' + spwstr.replace('~', '-')
if cfreq > 10.:
antenna = antenna + ';!0&1;!0&2' # deselect the shortest baselines
# else:
# antenna = antenna + ';!0&1' # deselect the shortest baselines
res = ptclean3(vis=msfile,
imageprefix=imdir,
imagesuffix=imagesuffix,
twidth=twidth,
uvrange=uvrange,
spw=spw,
ncpu=ncpu,
niter=1000,
gain=0.05,
antenna=antenna,
imsize=imsize,
cell=cell,
stokes=stokes,
doreg=True,
usephacenter=False,
overwrite=overwrite,
toTb=toTb,
restoringbeam=restoringbeam,
specmode="mfs",
deconvolver="hogbom",
datacolumn='data',
pbcor=True)
if res:
imres['Succeeded'] += res['Succeeded']
imres['BeginTime'] += res['BeginTime']
imres['EndTime'] += res['EndTime']
imres['ImageName'] += res['ImageName']
imres['Spw'] += [spwstr] * len(res['ImageName'])
imres['Vis'] += [msfile] * len(res['ImageName'])
else:
continue
if len(vis) == 1:
# produce the band-by-band whole-day images
ms.open(msfile)
ms.selectinit()
timfreq = ms.getdata(['time', 'axis_info'], ifraxis=True)
tim = timfreq['time']
ms.close()
cfreqs = getspwfreq(msfile)
imdir = imagedir + subdir[0]
if not os.path.exists(imdir):
os.makedirs(imdir)
for spw in spws:
antenna = antenna0
if spw == '':
spw = '{:d}~{:d}'.format(
next(x[0] for x in enumerate(cfreqs)
if x[1] > lowcutoff_freq),
len(cfreqs) - 1)
spwran = [s.zfill(2) for s in spw.split('~')]
freqran = [cfreqs[int(s)] for s in spw.split('~')]
cfreq = np.mean(freqran)
bmsz = max(150. / cfreq, 20.)
uvrange = ''
imsize = 512
cell = ['5arcsec']
if len(spwran) == 2:
spwstr = spwran[0] + '~' + spwran[1]
else:
spwstr = spwran[0]
restoringbeam = ['{0:.1f}arcsec'.format(bmsz)]
imagesuffix = '.synoptic.spw' + spwstr.replace('~', '-')
antenna = antenna + ';!0&1' # deselect the shortest baselines
res = ptclean3(vis=msfile,
imageprefix=imdir,
imagesuffix=imagesuffix,
twidth=len(tim),
uvrange=uvrange,
spw=spw,
ncpu=1,
niter=0,
gain=0.05,
antenna=antenna,
imsize=imsize,
cell=cell,
stokes=stokes,
doreg=True,
usephacenter=False,
overwrite=overwrite,
toTb=toTb,
restoringbeam=restoringbeam,
specmode="mfs",
deconvolver="hogbom",
datacolumn='data',
pbcor=True)
if res:
imres['Synoptic']['Succeeded'] += res['Succeeded']
imres['Synoptic']['BeginTime'] += res['BeginTime']
imres['Synoptic']['EndTime'] += res['EndTime']
imres['Synoptic']['ImageName'] += res['ImageName']
imres['Synoptic']['Spw'] += [spwstr] * len(res['ImageName'])
imres['Synoptic']['Vis'] += [msfile] * len(res['ImageName'])
else:
continue
# save it for debugging purposes
np.savez('imres.npz', imres=imres)
return imres
def importeovsa(idbfiles,
timebin=None,
width=None,
visprefix=None,
nocreatms=False,
doconcat=False,
modelms=''):
casalog.origin('importeovsa')
# # Initialize the helper class
# pdh = ParallelDataHelper("importeovsa", locals())
#
# # Validate input and output parameters
# try:
# pdh.setupIO()
# except Exception, instance:
# casalog.post('%s' % instance, 'ERROR')
# return False
if type(idbfiles) == Time:
filelist = ri.get_trange_files(idbfiles)
else:
# If input type is not Time, assume that it is the list of files to read
filelist = idbfiles
if type(filelist) == str:
filelist = [filelist]
for f in filelist:
if not os.path.exists(f):
casalog.post("Some files in filelist are invalid. Aborting...")
return False
if not visprefix:
visprefix = './'
if not timebin:
timebin = '0s'
if not width:
width = 1
if not modelms:
if nocreatms:
filename = filelist[0]
modelms = creatms(filename, visprefix)
else:
if not os.path.exists(modelms):
if nocreatms:
filename = filelist[0]
modelms = creatms(filename, visprefix)
msfile = []
time_concat = []
for filename in filelist:
uv = aipy.miriad.UV(filename)
# if filename.split('/')[-1][0:3] == 'UDB':
# uv_str = uv_hex_rm(uv)
# else:
# uv_str = uv
uv.select('antennae', 0, 1, include=True)
uv.select('polarization', -5, -5, include=True)
times = []
uv.rewind()
for preamble, data in uv.all():
uvw, t, (i, j) = preamble
times.append(t)
uv.select('clear', -1, -1, include=True)
times = jd2mjds(np.asarray(times))
inttime = np.median((times - np.roll(times, 1))[1:]) / 60
time_steps = len(times)
time_concat.append(int((times[-1] - times[0]) / 60 + inttime))
time0 = time.time()
if 'antlist' in uv.vartable:
ants = uv['antlist'].replace('\x00', '')
antlist = map(int, ants.split())
else:
antlist = [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16]
good_idx = np.where(uv['sfreq'] > 0)[0]
nf = len(good_idx)
npol = uv['npol']
nants = uv['nants']
source_id = uv['source'].replace('\x00', '')
sfreq = uv['sfreq'][good_idx]
sdf = uv['sdf'][good_idx]
ra, dec = uv['ra'], uv['dec']
nbl = nants * (nants - 1) / 2
bl2ord = bl_list2(nants)
npairs = nbl + nants
flag = np.ones((npol, nf, time_steps, npairs), dtype=bool)
out = np.zeros((npol, nf, time_steps, npairs),
dtype=np.complex64) # Cross-correlations
uvwarray = np.zeros((3, time_steps, npairs), dtype=np.float)
chan_band = get_band(sfreq=sfreq, sdf=sdf)
nband = len(chan_band)
uv.rewind()
l = -1
for preamble, data in uv.all():
uvw, t, (i0, j0) = preamble
i = antlist.index(i0 + 1)
j = antlist.index(j0 + 1)
if i > j:
# Reverse order of indices
j = antlist.index(i0 + 1)
i = antlist.index(j0 + 1)
# Assumes uv['pol'] is one of -5, -6, -7, -8
k = -5 - uv['pol']
l += 1
out[k, :, l / (npairs * npol), bl2ord[i0, j0]] = data.data
flag[k, :, l / (npairs * npol), bl2ord[i0, j0]] = data.mask
if i != j:
if k == 3:
uvwarray[:, l / (npairs * npol), bl2ord[
i0, j0]] = -uvw * constants.speed_of_light / 1e9
nrows = time_steps * npairs
out = out.reshape(npol, nf, nrows)
flag = flag.reshape(npol, nf, nrows)
uvwarray = uvwarray.reshape(3, nrows)
uvwarray = np.tile(uvwarray, (1, nband))
sigma = np.ones((4, nrows), dtype=np.float) + 1
sigma = np.tile(sigma, (1, nband))
casalog.post(
'IDB File {0} is readed in --- {1:10.2f} seconds ---'.format(
filename, (time.time() - time0)))
msname = list(filename.split('/')[-1])
msname = visprefix + ''.join(msname) + '-{:d}m.ms'.format(
int((times[-1] - times[0]) / 60 + inttime))
if not nocreatms:
modelms = creatms(filename, visprefix)
os.system('mv {} {}'.format(modelms, msname))
else:
casalog.post('----------------------------------------')
casalog.post('copying standard MS to {0}'.format(
msname, (time.time() - time0)))
casalog.post('----------------------------------------')
os.system("rm -fr %s" % msname)
os.system("cp -r " + " %s" % modelms + " %s" % msname)
casalog.post(
'Standard MS is copied to {0} in --- {1:10.2f} seconds ---'.
format(msname, (time.time() - time0)))
tb.open(msname, nomodify=False)
casalog.post('----------------------------------------')
casalog.post("Updating the main table of" '%s' % msname)
casalog.post('----------------------------------------')
for l, cband in enumerate(chan_band):
time1 = time.time()
nchannels = len(cband['cidx'])
for row in range(nrows):
tb.putcell('DATA', (row + l * nrows),
out[:, cband['cidx'][0]:cband['cidx'][-1] + 1, row])
tb.putcell(
'FLAG', (row + l * nrows),
flag[:, cband['cidx'][0]:cband['cidx'][-1] + 1, row])
casalog.post(
'---spw {0:02d} is updated in --- {1:10.2f} seconds ---'.
format((l + 1),
time.time() - time1))
tb.putcol('UVW', uvwarray)
tb.putcol('SIGMA', sigma)
tb.putcol('WEIGHT', 1.0 / sigma**2)
timearr = times
timearr = timearr.reshape(1, time_steps, 1)
timearr = np.tile(timearr, (nband, 1, npairs))
timearr = timearr.reshape(nband * npairs * time_steps)
tb.putcol('TIME', timearr)
tb.putcol('TIME_CENTROID', timearr)
scan_id = tb.getcol('SCAN_NUMBER')
scan_id *= 0
tb.putcol('SCAN_NUMBER', scan_id)
colnames = tb.colnames()
cols2rm = ["MODEL_DATA", "CORRECTED_DATA"]
for l in range(len(cols2rm)):
if cols2rm[l] in colnames:
tb.removecols(cols2rm[l])
tb.close()
casalog.post('----------------------------------------')
casalog.post("Updating the OBSERVATION table of" '%s' % msname)
casalog.post('----------------------------------------')
tb.open(msname + '/OBSERVATION', nomodify=False)
tb.putcol(
'TIME_RANGE',
np.asarray([times[0] - 0.5 * inttime,
times[-1] + 0.5 * inttime]).reshape(2, 1))
tb.putcol('OBSERVER', ['EOVSA team'])
tb.close()
casalog.post('----------------------------------------')
casalog.post("Updating the POINTING table of" '%s' % msname)
casalog.post('----------------------------------------')
tb.open(msname + '/POINTING', nomodify=False)
timearr = times.reshape(1, time_steps, 1)
timearr = np.tile(timearr, (nband, 1, nants))
timearr = timearr.reshape(nband * time_steps * nants)
tb.putcol('TIME', timearr)
tb.putcol('TIME_ORIGIN', timearr) # - 0.5 * delta_time)
direction = tb.getcol('DIRECTION')
direction[0, 0, :] = ra
direction[1, 0, :] = dec
tb.putcol('DIRECTION', direction)
target = tb.getcol('TARGET')
target[0, 0, :] = ra
target[1, 0, :] = dec
tb.putcol('TARGET', target)
tb.close()
casalog.post('----------------------------------------')
casalog.post("Updating the SOURCE table of" '%s' % msname)
casalog.post('----------------------------------------')
tb.open(msname + '/SOURCE', nomodify=False)
radec = tb.getcol('DIRECTION')
radec[0], radec[1] = ra, dec
tb.putcol('DIRECTION', radec)
name = np.array([source_id], dtype='|S{0}'.format(len(source_id) + 1))
tb.putcol('NAME', name)
tb.close()
casalog.post('----------------------------------------')
casalog.post("Updating the DATA_DESCRIPTION table of" '%s' % msname)
casalog.post('----------------------------------------')
tb.open(msname + '/DATA_DESCRIPTION/', nomodify=False)
pol_id = tb.getcol('POLARIZATION_ID')
pol_id *= 0
tb.putcol('POLARIZATION_ID', pol_id)
# spw_id = tb.getcol('SPECTRAL_WINDOW_ID')
# spw_id *= 0
# tb.putcol('SPECTRAL_WINDOW_ID', spw_id)
tb.close()
casalog.post('----------------------------------------')
casalog.post("Updating the POLARIZATION table of" '%s' % msname)
casalog.post('----------------------------------------')
tb.open(msname + '/POLARIZATION/', nomodify=False)
tb.removerows(rownrs=np.arange(1, nband, dtype=int))
tb.close()
casalog.post('----------------------------------------')
casalog.post("Updating the FIELD table of" '%s' % msname)
casalog.post('----------------------------------------')
tb.open(msname + '/FIELD/', nomodify=False)
delay_dir = tb.getcol('DELAY_DIR')
delay_dir[0], delay_dir[1] = ra, dec
tb.putcol('DELAY_DIR', delay_dir)
phase_dir = tb.getcol('PHASE_DIR')
phase_dir[0], phase_dir[1] = ra, dec
tb.putcol('PHASE_DIR', phase_dir)
reference_dir = tb.getcol('REFERENCE_DIR')
reference_dir[0], reference_dir[1] = ra, dec
tb.putcol('REFERENCE_DIR', reference_dir)
name = np.array([source_id], dtype='|S{0}'.format(len(source_id) + 1))
tb.putcol('NAME', name)
tb.close()
# FIELD: DELAY_DIR, PHASE_DIR, REFERENCE_DIR, NAME
del out, flag, uvwarray, uv, timearr, sigma
gc.collect() #
if not (timebin == '0s' and width == 1):
split(vis=msname,
outputvis=msname + '.split',
datacolumn='data',
timebin=timebin,
width=width,
keepflags=False)
os.system('rm -rf {}'.format(msname))
msfile.append(msname + '.split')
else:
msfile.append(msname)
casalog.post("finished in --- %s seconds ---" % (time.time() - time0))
if doconcat:
msname = list(filelist[0].split('/')[-1])
concatvis = visprefix + ''.join(msname) + '-{:d}m.ms'.format(
int(sum(time_concat)))
concat(vis=msfile, concatvis=concatvis, timesort=True)
# Change all observation ids to be the same (zero)
tb.open(concatvis + '/OBSERVATION', nomodify=False)
nobs = tb.nrows()
tb.removerows([i + 1 for i in range(nobs - 1)])
tb.close()
tb.open(concatvis, nomodify=False)
obsid = tb.getcol('OBSERVATION_ID')
newobsid = np.zeros(len(obsid), dtype='int')
tb.putcol('OBSERVATION_ID', newobsid)
tb.close()
for ll in msfile:
os.system('rm -rf {}'.format(ll))
return True
# flag CH3OH maser
#flagdata(vis=ms, mode='manual',
# spw='44054800170~44084179830Hz:44054800170~44084179830Hz')
tb.open(ms)
if 'CORRECTED_DATA' in tb.colnames():
datacolumn = 'corrected'
else:
datacolumn = 'data'
tb.close()
split(
vis=ms,
outputvis=cont_ms,
field=",".join(field_names),
width=16,
datacolumn=datacolumn,
spw=Kmses[fullpathms],
)
# Unflag CH3OH maser
#flagdata(vis=ms, mode='unflag',
# spw='44054800170~44084179830Hz:44054800170~44084179830Hz')
with open('done_recalibrating_{0}'.format(ms), 'w') as fh:
fh.write(datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S") + "\n")
fh.write("cont_ms={0}, ms={1}".format(cont_ms, ms) + "\n")
os.remove('WORKING')
os.chdir(cwd)
field = 'Sgr B2 MN K,Sgr B2 MS K,Sgr B2 SDS K'
restfreq = '23.722633335GHz'
selfcal_spw = '0~5:23.7178~23.7185GHz'
merged_ms = 'NH322_merged.ms'
if not os.path.exists(merged_ms):
splitvises = []
for ms, spw in spws.items():
basems = os.path.split(ms)[-1]
outputvis = 'NH322_{0}'.format(basems)
if not os.path.exists(outputvis):
split(
vis=ms,
outputvis=outputvis,
field=field,
# have to pick these carefully, they're topo.
spw='{0}:23.714~23.723GHz'.format(spw),
)
msmd.open(outputvis)
ref0 = msmd.reffreq(0)
msmd.close()
if ref0['refer'] == 'TOPO':
mst.open(outputvis, nomodify=False)
mst.regridspw(outframe='LSRK',
mode='vrad',
restfreq=23.722633335e9)
mst.close()
def get_dspec(vis=None, savespec=True, specfile=None, bl='', uvrange='', field='', scan='', datacolumn='data',
domedian=False, timeran=None, spw=None, timebin='0s', verbose=False):
from split_cli import split_cli as split
msfile = vis
if not spw:
spw = ''
if not timeran:
timeran = ''
if not bl:
bl = ''
if domedian:
if not uvrange:
uvrange = '0.2~0.8km'
else:
uvrange = ''
# Open the ms and plot dynamic spectrum
if verbose:
print('Splitting selected data...')
vis_spl = './tmpms.splitted'
if os.path.exists(vis_spl):
os.system('rm -rf ' + vis_spl)
split(vis=msfile, outputvis=vis_spl, timerange=timeran, antenna=bl, field=field, scan=scan, spw=spw,
uvrange=uvrange, timebin=timebin, datacolumn=datacolumn)
ms.open(vis_spl, nomodify=False)
if verbose:
print('Regridding into a single spectral window...')
# print('Reading data spw by spw')
ms.cvel(outframe='LSRK', mode='frequency', interp='nearest')
ms.selectinit(datadescid=0, reset=True)
data = ms.getdata(['amplitude', 'time', 'axis_info'], ifraxis=True)
ms.close()
os.system('rm -rf ' + vis_spl)
specamp = data['amplitude']
(npol, nfreq, nbl, ntim) = specamp.shape
if verbose:
print('npol, nfreq, nbl, ntime:', data['amplitude'].shape)
spec = np.swapaxes(specamp, 2, 1)
freq = data['axis_info']['freq_axis']['chan_freq'].reshape(nfreq)
tim = data['time']
if domedian:
if verbose:
print('doing median of all the baselines')
# mask zero values before median
spec_masked = np.ma.masked_where(spec < 1e-9, spec)
spec_med = np.ma.filled(np.ma.median(spec_masked, axis=1), fill_value=0.)
nbl = 1
ospec = spec_med.reshape((npol, nbl, nfreq, ntim))
else:
ospec = spec
# Save the dynamic spectral data
if savespec:
if not specfile:
specfile = msfile + '.dspec.npz'
if os.path.exists(specfile):
os.system('rm -rf ' + specfile)
np.savez(specfile, spec=ospec, tim=tim, freq=freq,
timeran=timeran, spw=spw, bl=bl, uvrange=uvrange)
if verbose:
print('Median dynamic spectrum saved as: ' + specfile)
return {'spec': ospec, 'tim': tim, 'freq': freq, 'timeran': timeran, 'spw': spw, 'bl': bl, 'uvrange': uvrange}
def get_dspec(vis=None, savespec=True, specfile=None, bl='', uvrange='', field='', scan='', datacolumn='data',
domedian=False, timeran=None, spw=None, timebin='0s', regridfreq=False, fillnan=None, verbose=False,
usetbtool=False):
# from split_cli import split_cli as split
if vis.endswith('/'):
vis = vis[:-1]
msfile = vis
if not spw:
spw = ''
if not timeran:
timeran = ''
if domedian:
if not uvrange:
uvrange = '0.2~0.8km'
# bl = ''
else:
uvrange = ''
if not bl:
bl = ''
else:
uvrange = ''
# Open the ms and plot dynamic spectrum
if verbose:
print('Splitting selected data...')
if usetbtool:
try:
tb.open(vis + '/POLARIZATION')
corrtype = tb.getcell('CORR_TYPE', 0)
pols = [stokesenum[p] for p in corrtype]
tb.close()
except:
pols = []
antmask = []
if uvrange is not '' or bl is not '':
ms.open(vis)
ms.selectinit(datadescid=0)
mdata = ms.metadata()
antlist = mdata.antennaids()
mdata.done()
staql = {'uvdist': uvrange, 'baseline': bl, 'spw': spw, 'field': field, 'scan': scan, 'timerange': timeran}
### todo the selection only works for uvrange and bl. To make the selection of other items works,
## I need to make mask for other items.
a = ms.msselect(staql)
mdata = ms.metadata()
baselines = mdata.baselines()
for lidx, l in enumerate(antlist):
antmask.append(baselines[l][antlist[lidx:]])
antmask = np.hstack(antmask)
mdata.done()
ms.close()
tb.open(vis)
spwtb = tbtool()
spwtb.open(vis + '/SPECTRAL_WINDOW')
ptb = tbtool()
ptb.open(vis + '/POLARIZATION')
ms.open(vis)
spwlist = []
mdata = ms.metadata()
nspw = mdata.nspw()
nbl = mdata.nbaselines() + mdata.nantennas()
nscans = mdata.nscans()
spw_nfrq = [] # List of number of frequencies in each spw
for i in range(nspw):
spw_nfrq.append(mdata.nchan(i))
spw_nfrq = np.array(spw_nfrq)
nf = np.sum(spw_nfrq)
smry = mdata.summary()
scan_ntimes = [] # List of number of times in each scan
for iscan in range(nscans):
scan_ntimes.append(
smry['observationID=0']['arrayID=0']['scan=' + str(iscan)]['fieldID=0']['nrows'] / nspw / nbl)
scan_ntimes = np.array(scan_ntimes)
scan_ntimes_integer = scan_ntimes.astype(np.int)
if len(np.where(scan_ntimes % scan_ntimes_integer != 0)[0]) != 0:
# if True:
scan_ntimes = [] # List of number of times in each scan
for iscan in range(nscans):
scan_ntimes.append(
len(smry['observationID=0']['arrayID=0']['scan=' + str(iscan)]['fieldID=0'].keys()) - 6)
scan_ntimes = np.array(scan_ntimes)
else:
scan_ntimes = scan_ntimes_integer
nt = np.sum(scan_ntimes)
times = tb.getcol('TIME')
if times[nbl] - times[0] != 0:
# This is frequency/scan sort order
order = 'f'
elif times[nbl * nspw - 1] - times[0] != 0:
# This is time sort order
order = 't'
npol = ptb.getcol('NUM_CORR', 0, 1)[0]
ptb.close()
freq = np.zeros(nf, float)
times = np.zeros(nt, float)
if order == 't':
specamp = np.zeros((npol, nf, nbl, nt), np.complex)
for j in range(nt):
fptr = 0
# Loop over spw
for i in range(nspw):
# Get channel frequencies for this spw (annoyingly comes out as shape (nf, 1)
cfrq = spwtb.getcol('CHAN_FREQ', i, 1)[:, 0]
if j == 0:
# Only need this the first time through
spwlist += [i] * len(cfrq)
if i == 0:
times[j] = tb.getcol('TIME', nbl * (i + nspw * j), 1) # Get the time
spec_ = tb.getcol('DATA', nbl * (i + nspw * j), nbl) # Get complex data for this spw
flag = tb.getcol('FLAG', nbl * (i + nspw * j), nbl) # Get flags for this spw
nfrq = len(cfrq)
# Apply flags
if type(fillnan) in [int, float]:
spec_[flag] = float(fillnan)
else:
spec_[flag] = 0.0
# Insert data for this spw into larger array
specamp[:, fptr:fptr + nfrq, :, j] = spec_
freq[fptr:fptr + nfrq] = cfrq
fptr += nfrq
else:
specf = np.zeros((npol, nf, nt, nbl), np.complex) # Array indexes are swapped
iptr = 0
for j in range(nscans):
# Loop over scans
for i in range(nspw):
# Loop over spectral windows
s = scan_ntimes[j]
f = spw_nfrq[i]
s1 = np.sum(scan_ntimes[:j]) # Start time index
s2 = np.sum(scan_ntimes[:j + 1]) # End time index
f1 = np.sum(spw_nfrq[:i]) # Start freq index
f2 = np.sum(spw_nfrq[:i + 1]) # End freq index
spec_ = tb.getcol('DATA', iptr, nbl * s)
flag = tb.getcol('FLAG', iptr, nbl * s)
if j == 0:
cfrq = spwtb.getcol('CHAN_FREQ', i, 1)[:, 0]
freq[f1:f2] = cfrq
spwlist += [i] * len(cfrq)
times[s1:s2] = tb.getcol('TIME', iptr, nbl * s).reshape(s, nbl)[:, 0] # Get the times
iptr += nbl * s
# Apply flags
if type(fillnan) in [int, float]:
spec_[flag] = float(fillnan)
else:
spec_[flag] = 0.0
# Insert data for this spw into larger array
specf[:, f1:f2, s1:s2] = spec_.reshape(npol, f, s, nbl)
# Swap the array indexes back to the desired order
specamp = np.swapaxes(specf, 2, 3)
tb.close()
spwtb.close()
ms.close()
if len(antmask) > 0:
specamp = specamp[:, :, np.where(antmask)[0], :]
(npol, nfreq, nbl, ntim) = specamp.shape
tim = times
else:
# Open the ms and plot dynamic spectrum
if verbose:
print('Splitting selected data...')
vis_spl = './tmpms.splitted'
if os.path.exists(vis_spl):
os.system('rm -rf ' + vis_spl)
# split(vis=msfile, outputvis=vis_spl, timerange=timeran, antenna=bl, field=field, scan=scan, spw=spw,
# uvrange=uvrange, timebin=timebin, datacolumn=datacolumn)
try:
from split_cli import split_cli as split
split(vis=msfile, outputvis=vis_spl, datacolumn=datacolumn, timerange=timeran, spw=spw, antenna=bl,
field=field,
scan=scan, uvrange=uvrange, timebin=timebin)
except:
ms.open(msfile, nomodify=True)
ms.split(outputms=vis_spl, whichcol=datacolumn, time=timeran, spw=spw, baseline=bl, field=field, scan=scan,
uvrange=uvrange, timebin=timebin)
ms.close()
if verbose:
print('Regridding into a single spectral window...')
# print('Reading data spw by spw')
try:
tb.open(vis_spl + '/POLARIZATION')
corrtype = tb.getcell('CORR_TYPE', 0)
pols = [stokesenum[p] for p in corrtype]
tb.close()
except:
pols = []
if regridfreq:
ms.open(vis_spl, nomodify=False)
ms.cvel(outframe='LSRK', mode='frequency', interp='nearest')
ms.selectinit(datadescid=0, reset=True)
data = ms.getdata(['amplitude', 'time', 'axis_info'], ifraxis=True)
specamp = data['amplitude']
freq = data['axis_info']['freq_axis']['chan_freq']
else:
ms.open(vis_spl)
ms.selectinit(datadescid=0, reset=True)
spwinfo = ms.getspectralwindowinfo()
specamp = []
freq = []
time = []
for descid in range(len(spwinfo.keys())):
ms.selectinit(datadescid=0, reset=True)
ms.selectinit(datadescid=descid)
data = ms.getdata(['amplitude', 'time', 'axis_info'], ifraxis=True)
specamp_ = data['amplitude']
freq_ = data['axis_info']['freq_axis']['chan_freq']
time_ = data['time']
if fillnan is not None:
flag_ = ms.getdata(['flag', 'time', 'axis_info'], ifraxis=True)['flag']
if type(fillnan) in [int, float, long]:
specamp_[flag_] = float(fillnan)
else:
specamp_[flag_] = 0.0
specamp.append(specamp_)
freq.append(freq_)
time.append(time_)
specamp = np.concatenate(specamp, axis=1)
freq = np.concatenate(freq, axis=0)
ms.selectinit(datadescid=0, reset=True)
ms.close()
os.system('rm -rf ' + vis_spl)
(npol, nfreq, nbl, ntim) = specamp.shape
freq = freq.reshape(nfreq)
tim = data['time']
if verbose:
print('npol, nfreq, nbl, ntime:', (npol, nfreq, nbl, ntim))
spec = np.swapaxes(specamp, 2, 1)
if domedian:
if verbose:
print('doing median of all the baselines')
# mask zero values before median
# spec_masked = np.ma.masked_where(spec < 1e-9, spec)
# spec_masked2 = np.ma.masked_invalid(spec)
# spec_masked = np.ma.masked_array(spec, mask=np.logical_or(spec_masked.mask, spec_masked2.mask))
# spec_med = np.ma.filled(np.ma.median(spec_masked, axis=1), fill_value=0.)
spec = np.abs(spec)
spec_med = np.nanmedian(spec, axis=1)
nbl = 1
ospec = spec_med.reshape((npol, nbl, nfreq, ntim))
else:
ospec = spec
# Save the dynamic spectral data
if savespec:
if not specfile:
specfile = msfile + '.dspec.npz'
if os.path.exists(specfile):
os.system('rm -rf ' + specfile)
np.savez(specfile, spec=ospec, tim=tim, freq=freq,
timeran=timeran, spw=spw, bl=bl, uvrange=uvrange, pol=pols)
if verbose:
print('Median dynamic spectrum saved as: ' + specfile)
return {'spec': ospec, 'tim': tim, 'freq': freq, 'timeran': timeran, 'spw': spw, 'bl': bl, 'uvrange': uvrange,
'pol': pols}
def mk_qlook_image(trange, doimport=False, docalib=False, ncpu=10, twidth=12, stokes=None, antenna='0~12',
#imagedir=None, spws=['1~3','4~6','7~9','10~13','14~18','19~28'],verbose=False):
imagedir=None, spws=['1~5','6~10','11~15','16~25'], toTb=True, overwrite=True,
doslfcal=False, verbose=False):
'''
trange: can be 1) a single Time() object: use the entire day
2) a range of Time(), e.g., Time(['2017-08-01 00:00','2017-08-01 23:00'])
3) a single or a list of UDBms file(s)
4) None -- use current date Time.now()
'''
if type(trange) == Time:
mslist = trange2ms(trange=trange, doimport=doimport)
vis = mslist['ms']
tsts = [l.to_datetime() for l in mslist['tstlist']]
subdir = [tst.strftime("%Y/%m/%d/") for tst in tsts]
if type(trange) == str:
try:
date = Time(trange)
mslist = trange2ms(trange=trange, doimport=doimport)
vis = mslist['ms']
except:
vis = [trange]
subdir = ['/']
for idx, f in enumerate(vis):
if f[-1] == '/':
vis[idx] = f[:-1]
if not stokes:
stokes = 'XX'
if not imagedir:
imagedir='./'
imres = {'Succeeded': [], 'BeginTime': [], 'EndTime': [], 'ImageName': [], 'Spw': [], 'Vis': []}
for n, msfile in enumerate(vis):
msfilebs=os.path.basename(msfile)
imdir = imagedir + subdir[n]
if not os.path.exists(imdir):
os.makedirs(imdir)
if doslfcal:
slfcalms = './'+msfilebs+'.xx'
split(msfile,outputvis=slfcalms,datacolumn='corrected',correlation='XX')
for spw in spws:
spwran = [s.zfill(2) for s in spw.split('~')]
freqran = [int(s)*0.5+2.9 for s in spw.split('~')]
cfreq=np.mean(freqran)
bmsz=max(150./cfreq,20.)
uvrange='<10klambda'
if doslfcal:
slfcal_img = './'+msfilebs+'.slf.spw'+spw.replace('~','-')+'.slfimg'
slfcal_tb = './'+msfilebs+'.slf.spw'+spw.replace('~','-')+'.slftb'
try:
clean(vis=slfcalms,
antenna=antenna,
imagename=slfcal_img,
spw=spw,
mode='mfs',
timerange='',
imagermode='csclean',
psfmode='clark',
imsize=[512,512],
cell=['5arcsec'],
niter=100,
gain=0.05,
stokes='I',
weighting='natural',
restoringbeam=[str(bmsz)+'arcsec'],
pbcor=False,
interactive=False,
usescratch=True)
except:
print 'error in cleaning spw: '+spw
break
gaincal(vis=slfcalms, refant='0',antenna=antenna,caltable=slfcal_tb,spw=spw, uvrange='',\
gaintable=[],selectdata=True,timerange='',solint='600s',gaintype='G',calmode='p',\
combine='',minblperant=3,minsnr=2,append=False)
if not os.path.exists(slfcal_tb):
print 'No solution found in spw: '+spw
break
else:
clearcal(slfcalms)
delmod(slfcalms)
applycal(vis=slfcalms,gaintable=[slfcal_tb],spw=spw,selectdata=True,\
antenna=antenna,interp='nearest',flagbackup=False,applymode='calonly',calwt=False)
msfile=slfcalms
if cfreq < 10.:
imsize=512
cell=['5arcsec']
else:
imsize=1024
cell=['2.5arcsec']
if len(spwran) == 2:
spwstr = spwran[0]+'~'+spwran[1]
else:
spwstr = spwran[0]
restoringbeam=['{0:.1f}arcsec'.format(bmsz)]
imagesuffix='.spw'+spwstr.replace('~','-')
if cfreq > 10.:
antenna=antenna+';!0&1;!0&2' #deselect the shortest baselines
res=ptclean(vis=msfile, imageprefix=imdir, imagesuffix=imagesuffix, twidth=twidth, uvrange=uvrange,
spw=spw, ncpu=ncpu, niter=1000, gain=0.05, antenna=antenna,imsize=imsize, cell=cell,
stokes=stokes, doreg=True, usephacenter=False, overwrite=overwrite, toTb=toTb, restoringbeam=restoringbeam,
uvtaper=True,outertaper=['30arcsec'])
if res:
imres['Succeeded'] += res['Succeeded']
imres['BeginTime'] += res['BeginTime']
imres['EndTime'] += res['EndTime']
imres['ImageName'] += res['ImageName']
imres['Spw'] += [spwstr]*len(res['ImageName'])
imres['Vis'] += [msfile]*len(res['ImageName'])
else:
return None
#save it for debugging purposes
np.savez('imres.npz',imres=imres)
return imres
def calibeovsa(vis=None, caltype=None, interp=None, docalib=True, doflag=True, flagant=None, doimage=False, imagedir=None, antenna=None,
timerange=None, spw=None, stokes=None, doconcat=False, msoutdir=None, concatvis=None, keep_orig_ms=True):
'''
:param vis: EOVSA visibility dataset(s) to be calibrated
:param caltype:
:param interp:
:param docalib:
:param qlookimage:
:param flagant:
:param stokes:
:param doconcat:
:return:
'''
if type(vis) == str:
vis = [vis]
for idx, f in enumerate(vis):
if f[-1] == '/':
vis[idx] = f[:-1]
for msfile in vis:
casalog.origin('calibeovsa')
if not caltype:
casalog.post("Caltype not provided. Perform reference phase calibration and daily phase calibration.")
caltype = ['refpha', 'phacal', 'fluxcal'] ## use this line after the phacal is applied # caltype = ['refcal']
if not os.path.exists(msfile):
casalog.post("Input visibility does not exist. Aborting...")
continue
if msfile.endswith('/'):
msfile = msfile[:-1]
if not msfile[-3:] in ['.ms', '.MS']:
casalog.post("Invalid visibility. Please provide a proper visibility file ending with .ms")
# if not caltable:
# caltable=[os.path.basename(vis).replace('.ms','.'+c) for c in caltype]
# get band information
tb.open(msfile + '/SPECTRAL_WINDOW')
nspw = tb.nrows()
bdname = tb.getcol('NAME')
bd_nchan = tb.getcol('NUM_CHAN')
bd = [int(b[4:]) - 1 for b in bdname] # band index from 0 to 33
# nchans = tb.getcol('NUM_CHAN')
# reffreqs = tb.getcol('REF_FREQUENCY')
# cenfreqs = np.zeros((nspw))
tb.close()
tb.open(msfile + '/ANTENNA')
nant = tb.nrows()
antname = tb.getcol('NAME')
antlist = [str(ll) for ll in range(len(antname) - 1)]
antennas = ','.join(antlist)
tb.close()
# get time stamp, use the beginning of the file
tb.open(msfile + '/OBSERVATION')
trs = {'BegTime': [], 'EndTime': []}
for ll in range(tb.nrows()):
tim0, tim1 = Time(tb.getcell('TIME_RANGE', ll) / 24 / 3600, format='mjd')
trs['BegTime'].append(tim0)
trs['EndTime'].append(tim1)
tb.close()
trs['BegTime'] = Time(trs['BegTime'])
trs['EndTime'] = Time(trs['EndTime'])
btime = np.min(trs['BegTime'])
etime = np.max(trs['EndTime'])
# ms.open(vis)
# summary = ms.summary()
# ms.close()
# btime = Time(summary['BeginTime'], format='mjd')
# etime = Time(summary['EndTime'], format='mjd')
## stop using ms.summary to avoid conflicts with importeovsa
t_mid = Time((btime.mjd + etime.mjd) / 2., format='mjd')
print "This scan observed from {} to {} UTC".format(btime.iso, etime.iso)
gaintables = []
if ('refpha' in caltype) or ('refamp' in caltype) or ('refcal' in caltype):
refcal = ra.sql2refcalX(btime)
pha = refcal['pha'] # shape is 15 (nant) x 2 (npol) x 34 (nband)
pha[np.where(refcal['flag'] == 1)] = 0.
amp = refcal['amp']
amp[np.where(refcal['flag'] == 1)] = 1.
t_ref = refcal['timestamp']
# find the start and end time of the local day when refcal is registered
try:
dhr = t_ref.LocalTime.utcoffset().total_seconds() / 60. / 60.
except:
dhr = -7.
bt = Time(np.fix(t_ref.mjd + dhr / 24.) - dhr / 24., format='mjd')
et = Time(bt.mjd + 1., format='mjd')
(yr, mon, day) = (bt.datetime.year, bt.datetime.month, bt.datetime.day)
dirname = caltbdir + str(yr) + str(mon).zfill(2) + '/'
if not os.path.exists(dirname):
os.mkdir(dirname)
# check if there is any ROACH reboot between the reference calibration found and the current data
t_rbts = db.get_reboot(Time([t_ref, btime]))
if not t_rbts:
casalog.post("Reference calibration is derived from observation at " + t_ref.iso)
print "Reference calibration is derived from observation at " + t_ref.iso
else:
casalog.post(
"Oh crap! Roach reboot detected between the reference calibration time " + t_ref.iso + ' and the current observation at ' + btime.iso)
casalog.post("Aborting...")
print "Oh crap! Roach reboot detected between the reference calibration time " + t_ref.iso + ' and the current observation at ' + btime.iso
print "Aborting..."
para_pha = []
para_amp = []
calpha = np.zeros((nspw, 15, 2))
calamp = np.zeros((nspw, 15, 2))
for s in range(nspw):
for n in range(15):
for p in range(2):
calpha[s, n, p] = pha[n, p, bd[s]]
calamp[s, n, p] = amp[n, p, bd[s]]
para_pha.append(np.degrees(pha[n, p, bd[s]]))
para_amp.append(amp[n, p, bd[s]])
if 'fluxcal' in caltype:
calfac = pc.get_calfac(Time(t_mid.iso.split(' ')[0] + 'T23:59:59'))
t_bp = Time(calfac['timestamp'], format='lv')
if int(t_mid.mjd) == int(t_bp.mjd):
accalfac = calfac['accalfac'] # (ant x pol x freq)
# tpcalfac = calfac['tpcalfac'] # (ant x pol x freq)
caltb_autoamp = dirname + t_bp.isot[:-4].replace(':', '').replace('-', '') + '.bandpass'
if not os.path.exists(caltb_autoamp):
bandpass(vis=msfile, caltable=caltb_autoamp, solint='inf', refant='eo01', minblperant=0, minsnr=0, bandtype='B', docallib=False)
tb.open(caltb_autoamp, nomodify=False) # (ant x spw)
bd_chanidx = np.hstack([[0], bd_nchan.cumsum()])
for ll in range(nspw):
antfac = np.sqrt(accalfac[:, :, bd_chanidx[ll]:bd_chanidx[ll + 1]])
# # antfac *= tpcalfac[:, :,bd_chanidx[ll]:bd_chanidx[ll + 1]]
antfac = np.moveaxis(antfac, 0, 2)
cparam = np.zeros((2, bd_nchan[ll], nant))
cparam[:, :, :-3] = 1.0 / antfac
tb.putcol('CPARAM', cparam + 0j, ll * nant, nant)
paramerr = tb.getcol('PARAMERR', ll * nant, nant)
paramerr = paramerr * 0
tb.putcol('PARAMERR', paramerr, ll * nant, nant)
bpflag = tb.getcol('FLAG', ll * nant, nant)
bpant1 = tb.getcol('ANTENNA1', ll * nant, nant)
bpflagidx, = np.where(bpant1 >= 13)
bpflag[:] = False
bpflag[:, :, bpflagidx] = True
tb.putcol('FLAG', bpflag, ll * nant, nant)
bpsnr = tb.getcol('SNR', ll * nant, nant)
bpsnr[:] = 100.0
bpsnr[:, :, bpflagidx] = 0.0
tb.putcol('SNR', bpsnr, ll * nant, nant)
tb.close()
msg_prompt = "Scaling calibration is derived for {}.".format(msfile)
casalog.post(msg_prompt)
print msg_prompt
gaintables.append(caltb_autoamp)
else:
msg_prompt = "Caution: No TPCAL is available on {}. No scaling calibration is derived for {}.".format(
t_mid.datetime.strftime('%b %d, %Y'), msfile)
casalog.post(msg_prompt)
print msg_prompt
if ('refpha' in caltype) or ('refcal' in caltype):
# caltb_pha = os.path.basename(vis).replace('.ms', '.refpha')
# check if the calibration table already exists
caltb_pha = dirname + t_ref.isot[:-4].replace(':', '').replace('-', '') + '.refpha'
if not os.path.exists(caltb_pha):
gencal(vis=msfile, caltable=caltb_pha, caltype='ph', antenna=antennas, pol='X,Y', spw='0~' + str(nspw - 1), parameter=para_pha)
gaintables.append(caltb_pha)
if ('refamp' in caltype) or ('refcal' in caltype):
# caltb_amp = os.path.basename(vis).replace('.ms', '.refamp')
caltb_amp = dirname + t_ref.isot[:-4].replace(':', '').replace('-', '') + '.refamp'
if not os.path.exists(caltb_amp):
gencal(vis=msfile, caltable=caltb_amp, caltype='amp', antenna=antennas, pol='X,Y', spw='0~' + str(nspw - 1), parameter=para_amp)
gaintables.append(caltb_amp)
# calibration for the change of delay center between refcal time and beginning of scan -- hopefully none!
xml, buf = ch.read_calX(4, t=[t_ref, btime], verbose=False)
if buf:
dly_t2 = Time(stf.extract(buf[0], xml['Timestamp']), format='lv')
dlycen_ns2 = stf.extract(buf[0], xml['Delaycen_ns'])[:15]
xml, buf = ch.read_calX(4, t=t_ref)
dly_t1 = Time(stf.extract(buf, xml['Timestamp']), format='lv')
dlycen_ns1 = stf.extract(buf, xml['Delaycen_ns'])[:15]
dlycen_ns_diff = dlycen_ns2 - dlycen_ns1
for n in range(2):
dlycen_ns_diff[:, n] -= dlycen_ns_diff[0, n]
print 'Multi-band delay is derived from delay center difference at {} & {}'.format(dly_t1.iso, dly_t2.iso)
# print '=====Delays relative to Ant 14====='
# for i, dl in enumerate(dlacen_ns_diff[:, 0] - dlacen_ns_diff[13, 0]):
# ant = antlist[i]
# print 'Ant eo{0:02d}: x {1:.2f} ns & y {2:.2f} ns'.format(int(ant) + 1, dl
# dlacen_ns_diff[i, 1] - dlacen_ns_diff[13, 1])
# caltb_mbd0 = os.path.basename(vis).replace('.ms', '.mbd0')
caltb_dlycen = dirname + dly_t2.isot[:-4].replace(':', '').replace('-', '') + '.dlycen'
if not os.path.exists(caltb_dlycen):
gencal(vis=msfile, caltable=caltb_dlycen, caltype='mbd', pol='X,Y', antenna=antennas, parameter=dlycen_ns_diff.flatten().tolist())
gaintables.append(caltb_dlycen)
if 'phacal' in caltype:
phacals = np.array(ra.sql2phacalX([bt, et], neat=True, verbose=False))
if not phacals.any() or len(phacals) == 0:
print "Found no phacal records in SQL database, will skip phase calibration"
else:
# first generate all phacal calibration tables if not already exist
t_phas = Time([phacal['t_pha'] for phacal in phacals])
# sort the array in ascending order by t_pha
sinds = t_phas.mjd.argsort()
t_phas = t_phas[sinds]
phacals = phacals[sinds]
caltbs_phambd = []
for i, phacal in enumerate(phacals):
# filter out phase cals with reference time stamp >30 min away from the provided refcal time
if (phacal['t_ref'].jd - refcal['timestamp'].jd) > 30. / 1440.:
del phacals[i]
del t_phas[i]
continue
else:
t_pha = phacal['t_pha']
phambd_ns = phacal['pslope']
for n in range(2): phambd_ns[:, n] -= phambd_ns[0, n]
# set all flagged values to be zero
phambd_ns[np.where(phacal['flag'] == 1)] = 0.
caltb_phambd = dirname + t_pha.isot[:-4].replace(':', '').replace('-', '') + '.phambd'
caltbs_phambd.append(caltb_phambd)
if not os.path.exists(caltb_phambd):
gencal(vis=msfile, caltable=caltb_phambd, caltype='mbd', pol='X,Y', antenna=antennas,
parameter=phambd_ns.flatten().tolist())
# now decides which table to apply depending on the interpolation method ("neatest" or "linear")
if interp == 'nearest':
tbind = np.argmin(np.abs(t_phas.mjd - t_mid.mjd))
dt = np.min(np.abs(t_phas.mjd - t_mid.mjd)) * 24.
print "Selected nearest phase calibration table at " + t_phas[tbind].iso
gaintables.append(caltbs_phambd[tbind])
if interp == 'linear':
# bphacal = ra.sql2phacalX(btime)
# ephacal = ra.sql2phacalX(etime,reverse=True)
bt_ind, = np.where(t_phas.mjd < btime.mjd)
et_ind, = np.where(t_phas.mjd > etime.mjd)
if len(bt_ind) == 0 and len(et_ind) == 0:
print "No phacal found before or after the ms data within the day of observation"
print "Skipping daily phase calibration"
elif len(bt_ind) > 0 and len(et_ind) == 0:
gaintables.append(caltbs_phambd[bt_ind[-1]])
elif len(bt_ind) == 0 and len(et_ind) > 0:
gaintables.append(caltbs_phambd[et_ind[0]])
elif len(bt_ind) > 0 and len(et_ind) > 0:
bphacal = phacals[bt_ind[-1]]
ephacal = phacals[et_ind[0]]
# generate a new table interpolating between two daily phase calibrations
t_pha_mean = Time(np.mean([bphacal['t_pha'].mjd, ephacal['t_pha'].mjd]), format='mjd')
phambd_ns = (bphacal['pslope'] + ephacal['pslope']) / 2.
for n in range(2): phambd_ns[:, n] -= phambd_ns[0, n]
# set all flagged values to be zero
phambd_ns[np.where(bphacal['flag'] == 1)] = 0.
phambd_ns[np.where(ephacal['flag'] == 1)] = 0.
caltb_phambd_interp = dirname + t_pha_mean.isot[:-4].replace(':', '').replace('-', '') + '.phambd'
if not os.path.exists(caltb_phambd_interp):
gencal(vis=msfile, caltable=caltb_phambd_interp, caltype='mbd', pol='X,Y', antenna=antennas,
parameter=phambd_ns.flatten().tolist())
print "Using phase calibration table interpolated between records at " + bphacal['t_pha'].iso + ' and ' + ephacal['t_pha'].iso
gaintables.append(caltb_phambd_interp)
if docalib:
clearcal(msfile)
applycal(vis=msfile, gaintable=gaintables, applymode='calflag', calwt=False)
# delete the interpolated phase calibration table
try:
caltb_phambd_interp
except:
pass
else:
if os.path.exists(caltb_phambd_interp):
shutil.rmtree(caltb_phambd_interp)
if doflag:
# flag zeros and NaNs
flagdata(vis=msfile, mode='clip', clipzeros=True)
if flagant:
try:
flagdata(vis=msfile, antenna=flagant)
except:
print "Something wrong with flagant. Abort..."
if doimage:
from matplotlib import pyplot as plt
from suncasa.utils import helioimage2fits as hf
from sunpy import map as smap
if not antenna:
antenna = '0~12'
if not stokes:
stokes = 'XX'
if not timerange:
timerange = ''
if not spw:
spw = '1~3'
if not imagedir:
imagedir = '.'
#(yr, mon, day) = (bt.datetime.year, bt.datetime.month, bt.datetime.day)
#dirname = imagedir + str(yr) + '/' + str(mon).zfill(2) + '/' + str(day).zfill(2) + '/'
#if not os.path.exists(dirname):
# os.makedirs(dirname)
bds = [spw]
nbd = len(bds)
imgs = []
for bd in bds:
if '~' in bd:
bdstr = bd.replace('~', '-')
else:
bdstr = str(bd).zfill(2)
imname = imagedir + '/' + os.path.basename(msfile).replace('.ms', '.bd' + bdstr)
print 'Cleaning image: ' + imname
try:
clean(vis=msfile, imagename=imname, antenna=antenna, spw=bd, timerange=timerange, imsize=[512], cell=['5.0arcsec'], stokes=stokes,
niter=500)
except:
print 'clean not successfull for band ' + str(bd)
else:
imgs.append(imname + '.image')
junks = ['.flux', '.mask', '.model', '.psf', '.residual']
for junk in junks:
if os.path.exists(imname + junk):
shutil.rmtree(imname + junk)
tranges = [btime.iso + '~' + etime.iso] * nbd
fitsfiles = [img.replace('.image', '.fits') for img in imgs]
hf.imreg(vis=msfile, timerange=tranges, imagefile=imgs, fitsfile=fitsfiles, usephacenter=False)
plt.figure(figsize=(6, 6))
for i, fitsfile in enumerate(fitsfiles):
plt.subplot(1, nbd, i + 1)
eomap = smap.Map(fitsfile)
sz = eomap.data.shape
if len(sz) == 4:
eomap.data = eomap.data.reshape((sz[2], sz[3]))
eomap.plot_settings['cmap'] = plt.get_cmap('jet')
eomap.plot()
eomap.draw_limb()
eomap.draw_grid()
plt.show()
if doconcat:
from suncasa.tasks import concateovsa_cli as ce
# from suncasa.eovsa import concateovsa as ce
if msoutdir is None:
msoutdir = './'
if not concatvis:
concatvis = os.path.basename(vis[0])
concatvis = msoutdir + '/' + concatvis.split('.')[0] + '_concat.ms'
else:
concatvis = os.path.join(msoutdir, concatvis)
if len(vis) > 1:
ce.concateovsa(vis, concatvis, datacolumn='corrected', keep_orig_ms=keep_orig_ms, cols2rm="model,corrected")
return [concatvis]
else:
split(vis=vis[0], outputvis=concatvis, datacolumn='corrected')
return [concatvis]
else:
return vis
mses = list(Qmses.keys())
fullpath_mses = [
'../' + ms[:-3] + "_continuum_split_for_selfcal.ms" for ms in mses
if ms in Qmses
]
cont_vis = []
for ms in fullpath_mses:
splitagain = ms[:-3] + "_SgrB2_NM_Q.ms"
myprint("{0} -> {1}".format(ms, splitagain))
if not os.path.exists(splitagain):
assert split(vis=ms,
outputvis=splitagain,
field='Sgr B2 NM Q',
datacolumn='corrected')
cont_vis.append(splitagain)
cleanbox_mask_image = 'cleanbox_mask_SgrB2.image'
cleanbox_mask = 'cleanbox_mask.mask'
mask = cleanbox_mask_image
selfcal_fields = "Sgr B2 N Q,Sgr B2 NM Q,Sgr B2 MS Q".split(",")
selfcal_fields = ['Sgr B2 NM Q']
extrapars = {
'wproject': {
'wprojplanes': 64,
#'rotatepastep': 5.0,
#'cfcache':'test_wtermmerge.cfcache',
def concateovsa(vis, concatvis, datacolumn='corrected', keep_orig_ms=True, cols2rm="model,corrected", freqtol="", dirtol="", respectname=False,
timesort=True, copypointing=True, visweightscale=[], forcesingleephemfield=""):
if concatvis[-1] == os.path.sep:
concatvis = concatvis[:-1]
if os.path.sep not in concatvis:
visprefix = './'
else:
visprefix = os.path.dirname(concatvis) + os.path.sep
msfiles = vis
msfiles_ = []
for idx, ll in enumerate(msfiles):
if str(ll).endswith('/'):
msfiles[idx] = str(ll)[:-1]
datacolumn = datacolumn.lower()
if datacolumn == 'data':
print 'DATA columns will be concatenated.'
for ll in msfiles:
clearcal(vis=str(ll), addmodel=True)
elif datacolumn == 'corrected':
# try:
print 'CORRECTED columns will be concatenated.'
tmpdir = os.path.join(visprefix, 'tmp_ms') + os.path.sep
if not os.path.exists(tmpdir):
os.makedirs(tmpdir)
for ll in msfiles:
msfile_ = os.path.join(tmpdir, os.path.basename(str(ll)))
msfiles_.append(msfile_)
split(vis=str(ll), outputvis=msfile_, datacolumn='corrected')
clearcal(vis=msfile_, addmodel=True)
else:
raise ValueError('Please set datacolumn to be "data" or "corrected"!')
if msfiles_:
concat(vis=msfiles_, concatvis=concatvis, freqtol=freqtol, dirtol=dirtol, respectname=respectname, timesort=timesort,
copypointing=copypointing, visweightscale=visweightscale, forcesingleephemfield=forcesingleephemfield)
os.system('rm -rf {}'.format(tmpdir))
else:
concat(vis=msfiles, concatvis=concatvis, freqtol=freqtol, dirtol=dirtol, respectname=respectname, timesort=timesort,
copypointing=copypointing, visweightscale=visweightscale, forcesingleephemfield=forcesingleephemfield)
# Change all observation ids to be the same (zero)
tb.open(concatvis + '/OBSERVATION', nomodify=False)
nobs = tb.nrows()
tim0 = tb.getcell('TIME_RANGE', 0)[0]
tim1 = tb.getcell('TIME_RANGE', nobs - 1)[1]
tb.removerows([i + 1 for i in range(nobs - 1)])
tb.putcell('TIME_RANGE', 0, [tim0, tim1])
tb.close()
tb.open(concatvis + '/DATA_DESCRIPTION', nomodify=False)
nrows = tb.nrows()
pol_id = tb.getcol('POLARIZATION_ID')
tb.removerows(np.where(pol_id != 0)[0])
tb.close()
tb.open(concatvis, nomodify=False)
dd_id = tb.getcol('DATA_DESC_ID')
idx_dd_id, = np.where(dd_id >= nrows / 2)
dd_id[idx_dd_id] = dd_id[idx_dd_id] - nrows / 2
tb.putcol('DATA_DESC_ID', dd_id)
tb.close()
tb.open(concatvis + '/FIELD', nomodify=False)
nobs = tb.nrows()
tb.removerows([i + 1 for i in range(nobs - 1)])
tb.close()
tb.open(concatvis + '/SOURCE', nomodify=False)
nobs = tb.nrows()
tb.removerows([i + 1 for i in range(nobs - 1)])
tb.close()
tb.open(concatvis, nomodify=False)
obsid = tb.getcol('OBSERVATION_ID')
newobsid = np.zeros(len(obsid), dtype='int')
tb.putcol('OBSERVATION_ID', newobsid)
fldid = tb.getcol('FIELD_ID')
newfldid = np.zeros(len(fldid), dtype='int')
tb.putcol('FIELD_ID', newfldid)
colnames = tb.colnames()
cols2rm = cols2rm.upper()
cols2rm = cols2rm.split(',')
for l in range(len(cols2rm)):
col = cols2rm[l] + '_DATA'
if col in colnames:
try:
tb.removecols(col)
print 'Column {} removed.'.format(col)
except:
pass
tb.close()
if msfiles_ != [] and msfiles_ != msfiles:
for ll in msfiles_:
os.system('rm -rf {}'.format(ll))
if not keep_orig_ms:
for ll in msfiles:
os.system('rm -rf {}'.format(ll))
flagdata(vis=fullpathms, mode='quack', quackinterval=10)
# flag edge channels
flagchans = ",".join(
["{0}:0~5;123~128".format(xx) for xx in Qmses[ms].split(",")])
flagdata(vis=fullpathms, mode='manual', spw=flagchans)
# flag CH3OH maser
flagdata(vis=fullpathms,
mode='manual',
spw='44054800170~44084179830Hz:44054800170~44084179830Hz')
split(
vis=fullpathms,
outputvis=cont_ms,
field=('Sgr B2 N Q,Sgr B2 NM Q,Sgr B2 MS Q,'
'Sgr B2 S Q,Sgr B2 DS1 Q,Sgr B2 DS2 Q,Sgr B2 DS3 Q'),
width=16,
spw=Qmses[ms],
)
# Unflag CH3OH maser
flagdata(vis=fullpathms,
mode='unflag',
spw='44054800170~44084179830Hz:44054800170~44084179830Hz')
clearcal(cont_ms, addmodel=True)
#ft(vis=cont_ms,
# field='Sgr B2 N Q,Sgr B2 NM Q,Sgr B2 MS Q',
# spw='',
# model=['../continuum/18A-229_mosaic_for_selfcal.model.tt0.noneg',
# '../continuum/18A-229_mosaic_for_selfcal.model.tt1.noneg'],
tint) + 'IDB20170906T' + trange.split(':')[3] + trange.split(
':')[4].split('.')[0] + '.ms.corrected.xx.slfcal'
slfcaledms = workdir + 'slfcal/No' + str(
tint) + 'IDB20170906T' + trange.split(':')[3] + trange.split(
':')[4].split('.')[0] + '.ms.corrected.xx.slfcaled'
calprefix = workdir + 'slfcal/caltbs/No' + str(
tint) + 'slf_19' + trange.split(':')[3] + trange.split(':')[4].split(
'.')[0]
#imgprefix=workdir+'slfcal/images/No'+str(tint)+'slf_19'+trange.split(':')[3]+trange.split(':')[4].split('.')[0]
imgprefix = workdir + 'slfcal/images/sbs/No' + str(
tint) + 'slf_19' + trange.split(':')[3] + trange.split(':')[4].split(
'.')[0]
if not os.path.exists(slfcalms):
split(vis=refms,
outputvis=slfcalms,
datacolumn='data',
timerange=trange,
correlation='XX')
listobs(slfcalms, listfile=slfcalms + '.listobs')
clearcal(slfcalms)
delmod(slfcalms)
if doapplyselfcal:
caltable_list = [
'/srg/ywei/data/eovsa/caltbs/slf_192920.G0/',
'/srg/ywei/data/eovsa/caltbs/slf_192920.G1/',
'/srg/ywei/data/eovsa/caltbs/slf_192920.G2/'
]
for nc in range(len(caltable_list)):
applycal(vis=slfcalms,
gaintable=caltable_list[nc],
spw=','.join(spws),
def mk_qlook_image(vis, ncpu=10, twidth=12, stokes='I,V', antenna='', imagedir=None, spws=[], toTb=True, overwrite=True, doslfcal=False,
phasecenter='', c_external=True):
vis = [vis]
subdir = ['/']
for idx, f in enumerate(vis):
if f[-1] == '/':
vis[idx] = f[:-1]
if not imagedir:
imagedir = './'
imres = {'Succeeded': [], 'BeginTime': [], 'EndTime': [], 'ImageName': [], 'Spw': [], 'Vis': [], 'Freq': []}
msfile = vis[0]
ms.open(msfile)
metadata = ms.metadata()
observatory = metadata.observatorynames()[0]
# axisInfo = ms.getdata(["axis_info"], ifraxis=True)
spwInfo = ms.getspectralwindowinfo()
# freqInfo = axisInfo["axis_info"]["freq_axis"]["chan_freq"].swapaxes(0, 1) / 1e9
# freqInfo_ravel = freqInfo.ravel()
ms.close()
if not spws:
if observatory == 'EVLA':
spws = ['0', '1', '2', '3', '4', '5', '6', '7']
if observatory == 'EOVSA':
spws = ['1~5', '6~10', '11~15', '16~25']
if observatory == 'EOVSA':
print 'Provide stokes: ' + str(stokes) + '. However EOVSA has linear feeds. Force stokes to be IV'
stokes = 'I,V'
msfilebs = os.path.basename(msfile)
imdir = imagedir + subdir[0]
if not os.path.exists(imdir):
os.makedirs(imdir)
if doslfcal:
slfcalms = './' + msfilebs + '.rr'
split(msfile, outputvis=slfcalms, datacolumn='corrected', correlation='RR')
for spw in spws:
spwran = [s.zfill(2) for s in spw.split('~')]
# freqran = [
# (int(s) * spwInfo['0']['TotalWidth'] + spwInfo['0']['RefFreq'] + spwInfo['0']['TotalWidth'] / 2.0) / 1.0e9
# for s in spw.split('~')]
spw_ = spw.split('~')
if len(spw_) == 2:
freqran = [(spwInfo['{}'.format(s)]['RefFreq'] + spwInfo['{}'.format(s)]['TotalWidth'] / 2.0) / 1.0e9 for s in spw.split('~')]
elif len(spw_) == 1:
s = spw_[0]
freqran = np.array([0, spwInfo['{}'.format(s)]['TotalWidth']]) + spwInfo['{}'.format(s)]['RefFreq']
freqran = freqran / 1.0e9
freqran = list(freqran)
else:
raise ValueError("Keyword 'spw' in wrong format")
cfreq = np.mean(freqran)
bmsz = max(30. / cfreq, 30.)
uvrange = '<3km'
if cfreq < 10.:
imsize = 512
cell = ['5arcsec']
else:
imsize = 1024
cell = ['2.5arcsec']
if len(spwran) == 2:
spwstr = spwran[0] + '~' + spwran[1]
else:
spwstr = spwran[0]
restoringbeam = ['{0:.1f}arcsec'.format(bmsz)]
imagesuffix = '.spw' + spwstr.replace('~', '-')
# if cfreq > 10.:
# antenna = antenna + ';!0&1;!0&2' # deselect the shortest baselines
sto = stokes.replace(',', '')
if c_external:
cleanscript = os.path.join(imdir, 'ptclean_external.py')
resfile = os.path.join(imdir, os.path.basename(msfile) + '.res.npz')
os.system('rm -rf {}'.format(cleanscript))
inpdict = {'vis': msfile, 'imageprefix': imdir, 'imagesuffix': imagesuffix, 'twidth': twidth, 'uvrange': uvrange, 'spw': spw,
'ncpu': ncpu, 'niter': 1000, 'gain': 0.05, 'antenna': antenna, 'imsize': imsize, 'cell': cell, 'stokes': sto, 'doreg': True,
'overwrite': overwrite, 'toTb': toTb, 'restoringbeam': restoringbeam, 'uvtaper': True, 'outertaper': ['30arcsec'],
'phasecenter': phasecenter}
for key, val in inpdict.items():
if type(val) is str:
inpdict[key] = '"{}"'.format(val)
fi = open(cleanscript, 'wb')
fi.write('from ptclean_cli import ptclean_cli as ptclean \n')
fi.write('import numpy as np \n')
fi.write(
'res = ptclean(vis={i[vis]},imageprefix={i[imageprefix]},imagesuffix={i[imagesuffix]},twidth={i[twidth]},uvrange={i[uvrange]},spw={i[spw]},ncpu={i[ncpu]},niter={i[niter]},gain={i[gain]},antenna={i[antenna]},imsize={i[imsize]},cell={i[cell]},stokes={i[stokes]},doreg={i[doreg]},overwrite={i[overwrite]},toTb={i[toTb]},restoringbeam={i[restoringbeam]},uvtaper={i[uvtaper]},outertaper={i[outertaper]},phasecenter={i[phasecenter]}) \n'.format(
i=inpdict))
fi.write('np.savez("{}",res=res) \n'.format(resfile))
fi.close()
os.system('casa --nologger -c {}'.format(cleanscript))
res = np.load(resfile)
res = res['res'].item()
else:
res = ptclean(vis=msfile, imageprefix=imdir, imagesuffix=imagesuffix, twidth=twidth, uvrange=uvrange, spw=spw, ncpu=ncpu, niter=1000,
gain=0.05, antenna=antenna, imsize=imsize, cell=cell, stokes=sto, doreg=True, overwrite=overwrite, toTb=toTb,
restoringbeam=restoringbeam, uvtaper=True, outertaper=['30arcsec'], phasecenter=phasecenter)
if res:
imres['Succeeded'] += res['Succeeded']
imres['BeginTime'] += res['BeginTime']
imres['EndTime'] += res['EndTime']
imres['ImageName'] += res['ImageName']
imres['Spw'] += [spwstr] * len(res['ImageName'])
imres['Vis'] += [msfile] * len(res['ImageName'])
imres['Freq'] += [freqran] * len(res['ImageName'])
else:
return None
# save it for debugging purposes
np.savez(os.path.join(imagedir, '{}.imres.npz'.format(os.path.basename(msfile))), imres=imres)
return imres
def importeovsa_iter(filelist, timebin, width, visprefix, nocreatms, modelms,
doscaling, keep_nsclms, fileidx):
from taskinit import tb, casalog
filename = filelist[fileidx]
uv = aipy.miriad.UV(filename)
# try:
msname0 = list(filename.split('/')[-1])
msname = visprefix + ''.join(msname0) + '.ms'
uv.select('antennae', 0, 1, include=True)
uv.select('polarization', -5, -5, include=True)
times = []
uv.rewind()
for preamble, data in uv.all():
uvw, t, (i, j) = preamble
times.append(t)
uv.select('clear', -1, -1, include=True)
times = ipe.jd2mjds(np.asarray(times))
inttime = np.median((times - np.roll(times, 1))[1:]) / 60
time_steps = len(times)
durtim = int((times[-1] - times[0]) / 60 + inttime)
time0 = time.time()
if 'antlist' in uv.vartable:
ants = uv['antlist'].replace('\x00', '')
antlist = map(int, ants.split())
else:
antlist = [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16]
good_idx = np.where(uv['sfreq'] > 0)[0]
nf = len(good_idx)
npol = uv['npol']
nants = uv['nants']
source_id = uv['source'].replace('\x00', '')
sfreq = uv['sfreq'][good_idx]
sdf = uv['sdf'][good_idx]
ra, dec = uv['ra'], uv['dec']
nbl = nants * (nants - 1) / 2
bl2ord = ipe.bl_list2(nants)
npairs = nbl + nants
flag = np.ones((npol, nf, time_steps, npairs), dtype=bool)
out = np.zeros((npol, nf, time_steps, npairs),
dtype=np.complex64) # Cross-correlations
uvwarray = np.zeros((3, time_steps, npairs), dtype=np.float)
chan_band = ipe.get_band(sfreq=sfreq, sdf=sdf)
nband = len(chan_band)
uv.rewind()
l = -1
for preamble, data in uv.all():
uvw, t, (i0, j0) = preamble
i = antlist.index(i0 + 1)
j = antlist.index(j0 + 1)
if i > j:
# Reverse order of indices
j = antlist.index(i0 + 1)
i = antlist.index(j0 + 1)
# Assumes uv['pol'] is one of -5, -6, -7, -8
k = -5 - uv['pol']
l += 1
data = ma.masked_array(ma.masked_invalid(data), fill_value=0.0)
out[k, :, l / (npairs * npol), bl2ord[i0, j0]] = data.data
flag[k, :, l / (npairs * npol), bl2ord[i0, j0]] = data.mask
# if i != j:
if k == 3:
uvwarray[:, l / (npairs * npol),
bl2ord[i0, j0]] = -uvw * constants.speed_of_light / 1e9
nrows = time_steps * npairs
if doscaling:
out2 = out.copy()
for i0 in antlist:
for j0 in antlist:
if i0 < j0:
i, j = i0 - 1, j0 - 1
out2[:, :, :,
bl2ord[i, j]] = out[:, :, :, bl2ord[i, j]] / np.sqrt(
np.abs(out[:, :, :, bl2ord[i, i]]) *
np.abs(out[:, :, :, bl2ord[j, j]]))
out2 = out2.reshape(npol, nf, nrows)
out2[np.isnan(out2)] = 0
out2[np.isinf(out2)] = 0
# out2 = ma.masked_array(ma.masked_invalid(out2), fill_value=0.0)
out = out.reshape(npol, nf, nrows) * 1e4
flag = flag.reshape(npol, nf, nrows)
uvwarray = uvwarray.reshape(3, nrows)
uvwarray = np.tile(uvwarray, (1, nband))
sigma = np.ones((4, nrows), dtype=np.float) + 1
sigma = np.tile(sigma, (1, nband))
casalog.post('IDB File {0} is readed in --- {1:10.2f} seconds ---'.format(
filename, (time.time() - time0)))
if not nocreatms:
modelms = ipe.creatms(filename, visprefix)
os.system('mv {} {}'.format(modelms, msname))
else:
casalog.post('----------------------------------------')
casalog.post('copying standard MS to {0}'.format(
msname, (time.time() - time0)))
casalog.post('----------------------------------------')
os.system("rm -fr %s" % msname)
os.system("cp -r " + " %s" % modelms + " %s" % msname)
casalog.post(
'Standard MS is copied to {0} in --- {1:10.2f} seconds ---'.format(
msname, (time.time() - time0)))
tb.open(msname, nomodify=False)
casalog.post('----------------------------------------')
casalog.post("Updating the main table of" '%s' % msname)
casalog.post('----------------------------------------')
for l, cband in enumerate(chan_band):
time1 = time.time()
nchannels = len(cband['cidx'])
for row in range(nrows):
if not doscaling or keep_nsclms:
tb.putcell('DATA', (row + l * nrows),
out[:, cband['cidx'][0]:cband['cidx'][-1] + 1, row])
tb.putcell('FLAG', (row + l * nrows),
flag[:, cband['cidx'][0]:cband['cidx'][-1] + 1, row])
casalog.post(
'---spw {0:02d} is updated in --- {1:10.2f} seconds ---'.format(
(l + 1),
time.time() - time1))
tb.putcol('UVW', uvwarray)
tb.putcol('SIGMA', sigma)
tb.putcol('WEIGHT', 1.0 / sigma**2)
timearr = times
timearr = timearr.reshape(1, time_steps, 1)
timearr = np.tile(timearr, (nband, 1, npairs))
timearr = timearr.reshape(nband * npairs * time_steps)
tb.putcol('TIME', timearr)
tb.putcol('TIME_CENTROID', timearr)
scan_id = tb.getcol('SCAN_NUMBER')
scan_id *= 0
tb.putcol('SCAN_NUMBER', scan_id)
colnames = tb.colnames()
cols2rm = ["MODEL_DATA", "CORRECTED_DATA"]
for l in range(len(cols2rm)):
if cols2rm[l] in colnames:
tb.removecols(cols2rm[l])
tb.close()
casalog.post('----------------------------------------')
casalog.post("Updating the OBSERVATION table of" '%s' % msname)
casalog.post('----------------------------------------')
tb.open(msname + '/OBSERVATION', nomodify=False)
tb.putcol(
'TIME_RANGE',
np.asarray([times[0] - 0.5 * inttime,
times[-1] + 0.5 * inttime]).reshape(2, 1))
tb.putcol('OBSERVER', ['EOVSA team'])
tb.close()
casalog.post('----------------------------------------')
casalog.post("Updating the POINTING table of" '%s' % msname)
casalog.post('----------------------------------------')
tb.open(msname + '/POINTING', nomodify=False)
timearr = times.reshape(1, time_steps, 1)
timearr = np.tile(timearr, (nband, 1, nants))
timearr = timearr.reshape(nband * time_steps * nants)
tb.putcol('TIME', timearr)
tb.putcol('TIME_ORIGIN', timearr) # - 0.5 * delta_time)
direction = tb.getcol('DIRECTION')
direction[0, 0, :] = ra
direction[1, 0, :] = dec
tb.putcol('DIRECTION', direction)
target = tb.getcol('TARGET')
target[0, 0, :] = ra
target[1, 0, :] = dec
tb.putcol('TARGET', target)
tb.close()
casalog.post('----------------------------------------')
casalog.post("Updating the SOURCE table of" '%s' % msname)
casalog.post('----------------------------------------')
tb.open(msname + '/SOURCE', nomodify=False)
radec = tb.getcol('DIRECTION')
radec[0], radec[1] = ra, dec
tb.putcol('DIRECTION', radec)
name = np.array([source_id], dtype='|S{0}'.format(len(source_id) + 1))
tb.putcol('NAME', name)
tb.close()
casalog.post('----------------------------------------')
casalog.post("Updating the DATA_DESCRIPTION table of" '%s' % msname)
casalog.post('----------------------------------------')
tb.open(msname + '/DATA_DESCRIPTION/', nomodify=False)
pol_id = tb.getcol('POLARIZATION_ID')
pol_id *= 0
tb.putcol('POLARIZATION_ID', pol_id)
# spw_id = tb.getcol('SPECTRAL_WINDOW_ID')
# spw_id *= 0
# tb.putcol('SPECTRAL_WINDOW_ID', spw_id)
tb.close()
# casalog.post('----------------------------------------')
# casalog.post("Updating the POLARIZATION table of" '%s' % msname)
# casalog.post('----------------------------------------')
# tb.open(msname + '/POLARIZATION/', nomodify=False)
# tb.removerows(rownrs=np.arange(1, nband, dtype=int))
# tb.close()
casalog.post('----------------------------------------')
casalog.post("Updating the FIELD table of" '%s' % msname)
casalog.post('----------------------------------------')
tb.open(msname + '/FIELD/', nomodify=False)
delay_dir = tb.getcol('DELAY_DIR')
delay_dir[0], delay_dir[1] = ra, dec
tb.putcol('DELAY_DIR', delay_dir)
phase_dir = tb.getcol('PHASE_DIR')
phase_dir[0], phase_dir[1] = ra, dec
tb.putcol('PHASE_DIR', phase_dir)
reference_dir = tb.getcol('REFERENCE_DIR')
reference_dir[0], reference_dir[1] = ra, dec
tb.putcol('REFERENCE_DIR', reference_dir)
name = np.array([source_id], dtype='|S{0}'.format(len(source_id) + 1))
tb.putcol('NAME', name)
tb.close()
# FIELD: DELAY_DIR, PHASE_DIR, REFERENCE_DIR, NAME
# del out, flag, uvwarray, uv, timearr, sigma
# gc.collect() #
if doscaling:
if keep_nsclms:
msname_scl = visprefix + ''.join(msname0) + '_scl.ms'
os.system('cp -r {} {}'.format(msname, msname_scl))
else:
msname_scl = msname
tb.open(msname_scl, nomodify=False)
casalog.post('----------------------------------------')
casalog.post("Updating the main table of" '%s' % msname_scl)
casalog.post('----------------------------------------')
for l, cband in enumerate(chan_band):
time1 = time.time()
for row in range(nrows):
tb.putcell(
'DATA', (row + l * nrows),
out2[:, cband['cidx'][0]:cband['cidx'][-1] + 1, row])
casalog.post(
'---spw {0:02d} is updated in --- {1:10.2f} seconds ---'.
format((l + 1),
time.time() - time1))
tb.close()
if not (timebin == '0s' and width == 1):
msfile = msname + '.split'
if doscaling:
split(vis=msname_scl,
outputvis=msname_scl + '.split',
datacolumn='data',
timebin=timebin,
width=width,
keepflags=False)
os.system('rm -rf {}'.format(msname_scl))
msfile_scl = msname_scl + '.split'
if not (doscaling and not keep_nsclms):
split(vis=msname,
outputvis=msname + '.split',
datacolumn='data',
timebin=timebin,
width=width,
keepflags=False)
os.system('rm -rf {}'.format(msname))
else:
msfile = msname
if doscaling:
msfile_scl = msname_scl
casalog.post("finished in --- %s seconds ---" % (time.time() - time0))
if doscaling:
return [True, msfile, msfile_scl, durtim]
else:
return [True, msfile, durtim]
def importeovsa(idbfiles, timebin=None, width=None, visprefix=None, nocreatms=True, doconcat=False):
casalog.origin('importeovsa')
# # Initialize the helper class
# pdh = ParallelDataHelper("importeovsa", locals())
#
# # Validate input and output parameters
# try:
# pdh.setupIO()
# except Exception, instance:
# casalog.post('%s' % instance, 'ERROR')
# return False
if type(idbfiles) == Time:
filelist = ri.get_trange_files(idbfiles)
else:
# If input type is not Time, assume that it is the list of files to read
filelist = idbfiles
if type(filelist) == str:
filelist = [filelist]
for f in filelist:
if not os.path.exists(f):
casalog.post("Some files in filelist are invalid. Aborting...")
return False
if not visprefix:
visprefix = './'
if not timebin:
timebin = '0s'
if not width:
width = 1
if nocreatms:
filename = filelist[0]
modelms = creatms(filename, visprefix)
msfile = []
for filename in filelist:
uv = aipy.miriad.UV(filename)
# if uv['source'].lower() == 'sun':
# visprefix = visprefix + 'sun/'
# if not os.path.exists(visprefix):
# os.mkdir(visprefix)
# else:
# visprefix = visprefix + 'calibrator/'
# if not os.path.exists(visprefix):
# os.mkdir(visprefix)
# uv.rewind()
start_time = 0 # The start and stop times are referenced to ref_time_jd in second
end_time = 600
delta_time = 1
time_steps = (end_time - start_time) / delta_time
time0 = time.time()
if 'antlist' in uv.vartable:
ants = uv['antlist']
antlist = map(int, ants.split())
else:
antlist = [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16]
good_idx = np.arange(len(uv['sfreq']))
ref_time_jd = uv['time']
ref_time_mjd = (ref_time_jd - 2400000.5) * 24. * 3600. + 0.5 * delta_time
nf = len(good_idx)
npol = uv['npol']
nants = uv['nants']
source_id = uv['source']
ra, dec = uv['ra'], uv['dec']
nbl = nants * (nants - 1) / 2
bl2ord = bl_list2(nants)
npairs = nbl + nants
flag = np.ones((npol, nf, time_steps, npairs), dtype=bool)
out = np.zeros((npol, nf, time_steps, npairs), dtype=np.complex64) # Cross-correlations
uvwarray = np.zeros((3, time_steps, npairs), dtype=np.float)
bandedge = get_band_edge(nband=34)
nband = len(bandedge) - 1
uv.rewind()
l = -1
for preamble, data in uv.all():
uvw, t, (i0, j0) = preamble
i = antlist.index(i0 + 1)
j = antlist.index(j0 + 1)
if i > j:
# Reverse order of indices
j = antlist.index(i0 + 1)
i = antlist.index(j0 + 1)
# Assumes uv['pol'] is one of -5, -6, -7, -8
k = -5 - uv['pol']
l += 1
out[k, :, l / (npairs * npol), bl2ord[i0, j0]] = data.data
flag[k, :, l / (npairs * npol), bl2ord[i0, j0]] = data.mask
if i != j:
if k == 3:
uvwarray[:, l / (npairs * npol), bl2ord[i0, j0]] = -uvw * constants.speed_of_light / 1e9
nrows = time_steps * npairs
out = out.reshape(npol, nf, nrows)
flag = flag.reshape(npol, nf, nrows)
uvwarray = uvwarray.reshape(3, nrows)
uvwarray = np.tile(uvwarray, (1, nband))
sigma = np.ones((4, nrows), dtype=np.float) + 1
sigma = np.tile(sigma, (1, nband))
casalog.post('IDB File {0} is readed in --- {1:10.2f} seconds ---'.format(filename, (time.time() - time0)))
msname = list(filename.split('/')[-1])
msname.insert(11, 'T')
msname = visprefix + source_id.upper() + '_' + ''.join(msname[3:]) + '-10m.ms'
if not nocreatms:
modelms = creatms(filename, visprefix)
os.system('mv {} {}'.format(modelms, msname))
else:
casalog.post('----------------------------------------')
casalog.post('copying standard MS to {0}'.format(msname, (time.time() - time0)))
casalog.post('----------------------------------------')
os.system("rm -fr %s" % msname)
os.system("cp -r " + " %s" % modelms + " %s" % msname)
casalog.post(
'Standard MS is copied to {0} in --- {1:10.2f} seconds ---'.format(msname, (time.time() - time0)))
tb.open(msname, nomodify=False)
casalog.post('----------------------------------------')
casalog.post("Updating the main table of" '%s' % msname)
casalog.post('----------------------------------------')
for l, bdedge in enumerate(bandedge[:-1]):
time1 = time.time()
nchannels = (bandedge[l + 1] - bandedge[l])
for row in range(nrows):
tb.putcell('DATA', (row + l * nrows), out[:, bandedge[l]:bandedge[l + 1], row])
tb.putcell('FLAG', (row + l * nrows), flag[:, bandedge[l]:bandedge[l + 1], row])
casalog.post('---spw {0:02d} is updated in --- {1:10.2f} seconds ---'.format((l + 1), time.time() - time1))
tb.putcol('UVW', uvwarray)
tb.putcol('SIGMA', sigma)
tb.putcol('WEIGHT', 1.0 / sigma ** 2)
timearr = np.arange((time_steps), dtype=np.float)
timearr = timearr.reshape(1, time_steps, 1)
timearr = np.tile(timearr, (nband, 1, npairs))
timearr = timearr.reshape(nband * npairs * time_steps) + ref_time_mjd
tb.putcol('TIME', timearr)
tb.putcol('TIME_CENTROID', timearr)
scan_id = tb.getcol('SCAN_NUMBER')
scan_id *= 0
tb.putcol('SCAN_NUMBER', scan_id)
colnames = tb.colnames()
cols2rm = ["MODEL_DATA", "CORRECTED_DATA"]
for l in range(len(cols2rm)):
if cols2rm[l] in colnames:
tb.removecols(cols2rm[l])
tb.close()
casalog.post('----------------------------------------')
casalog.post("Updating the OBSERVATION table of" '%s' % msname)
casalog.post('----------------------------------------')
tb.open(msname + '/OBSERVATION', nomodify=False)
tb.putcol('TIME_RANGE',
np.asarray([ref_time_mjd - 0.5 * delta_time, ref_time_mjd + end_time - 0.5 * delta_time]).reshape(
2, 1))
tb.putcol('OBSERVER', ['EOVSA team'])
tb.close()
casalog.post('----------------------------------------')
casalog.post("Updating the POINTING table of" '%s' % msname)
casalog.post('----------------------------------------')
tb.open(msname + '/POINTING', nomodify=False)
timearr = np.arange((time_steps), dtype=np.float).reshape(1, time_steps, 1)
timearr = np.tile(timearr, (nband, 1, nants))
timearr = timearr.reshape(nband * time_steps * nants) + ref_time_mjd
tb.putcol('TIME', timearr)
tb.putcol('TIME_ORIGIN', timearr - 0.5 * delta_time)
direction = tb.getcol('DIRECTION')
direction[0, 0, :] = ra
direction[1, 0, :] = dec
tb.putcol('DIRECTION', direction)
target = tb.getcol('TARGET')
target[0, 0, :] = ra
target[1, 0, :] = dec
tb.putcol('TARGET', target)
tb.close()
casalog.post('----------------------------------------')
casalog.post("Updating the SOURCE table of" '%s' % msname)
casalog.post('----------------------------------------')
tb.open(msname + '/SOURCE', nomodify=False)
radec = tb.getcol('DIRECTION')
radec[0], radec[1] = ra, dec
tb.putcol('DIRECTION', radec)
name = np.array([source_id], dtype='|S{0}'.format(len(source_id) + 1))
tb.putcol('NAME', name)
tb.close()
casalog.post('----------------------------------------')
casalog.post("Updating the DATA_DESCRIPTION table of" '%s' % msname)
casalog.post('----------------------------------------')
tb.open(msname + '/DATA_DESCRIPTION/', nomodify=False)
pol_id = tb.getcol('POLARIZATION_ID')
pol_id *= 0
tb.putcol('POLARIZATION_ID', pol_id)
# spw_id = tb.getcol('SPECTRAL_WINDOW_ID')
# spw_id *= 0
# tb.putcol('SPECTRAL_WINDOW_ID', spw_id)
tb.close()
casalog.post('----------------------------------------')
casalog.post("Updating the POLARIZATION table of" '%s' % msname)
casalog.post('----------------------------------------')
tb.open(msname + '/POLARIZATION/', nomodify=False)
tb.removerows(rownrs=np.arange(1, nband, dtype=int))
tb.close()
casalog.post('----------------------------------------')
casalog.post("Updating the FIELD table of" '%s' % msname)
casalog.post('----------------------------------------')
tb.open(msname + '/FIELD/', nomodify=False)
delay_dir = tb.getcol('DELAY_DIR')
delay_dir[0], delay_dir[1] = ra, dec
tb.putcol('DELAY_DIR', delay_dir)
phase_dir = tb.getcol('PHASE_DIR')
phase_dir[0], phase_dir[1] = ra, dec
tb.putcol('PHASE_DIR', phase_dir)
reference_dir = tb.getcol('REFERENCE_DIR')
reference_dir[0], reference_dir[1] = ra, dec
tb.putcol('REFERENCE_DIR', reference_dir)
name = np.array([source_id], dtype='|S{0}'.format(len(source_id) + 1))
tb.putcol('NAME', name)
tb.close()
# FIELD: DELAY_DIR, PHASE_DIR, REFERENCE_DIR, NAME
del out, flag, uvwarray, uv, timearr, sigma
gc.collect() #
if not (timebin == '0s' and width == 1):
split(vis=msname, outputvis=msname + '.split', datacolumn='data', timebin=timebin, width=width)
os.system('rm -rf {}'.format(msname))
msfile.append(msname + '.split')
else:
msfile.append(msname)
casalog.post("finished in --- %s seconds ---" % (time.time() - time0))
# msname = visprefix + source_id.upper() + '_' + ''.join(msname[3:]) + '-10m.ms'
if doconcat:
msname = list(filelist[0].split('/')[-1])
msname.insert(11, 'T')
concatvis = visprefix + ''.join(msname[3:]) + '-{}m.ms'.format(10.0 * len(msfile))
concat(vis=msfile, concatvis=concatvis)
for ll in msfile:
os.system('rm -rf {}'.format(ll))
return True
from astropy.io import fits
from astropy import wcs
mses = list(Qmses.keys())
fullpath_mses = ['../' + ms[:-3] + "_continuum.ms"
for ms in mses if ms in Qmses]
raw_and_corr_vis = 'continuum_concatenated_raw_and_corr.ms'
if not os.path.exists(raw_and_corr_vis):
assert concat(vis=fullpath_mses, concatvis=raw_and_corr_vis,
# should be used but isn't freqtol='5MHz',
)
cont_vis = 'continuum_concatenated_selfcal_wterms.ms'
if not os.path.exists(cont_vis):
assert split(vis=raw_and_corr_vis, outputvis=cont_vis,
datacolumn='corrected')
# this is OK because there should be no corrected column
clearcal(vis=cont_vis, addmodel=True)
imagename = '18A-229_Q_mosaic_selfcal_iter0_dirty'
if not os.path.exists(imagename+".image.tt0.pbcor"):
# do a full-mosaic clean to enable mask creation
tclean(
vis=cont_vis,
spw='',
field="Sgr B2 N Q,Sgr B2 NM Q,Sgr B2 MS Q,Sgr B2 S Q",
phasecenter='J2000 17h47m19.693 -28d23m11.527',
imsize=[9000,9000],
cell='0.02arcsec',
#
# this is OK because there should be no corrected column
#clearcal(vis=cont_vis, addmodel=True)
def myprint(x):
print(x)
casalog.post(str(x), origin='singlefield')
selfcal_mses = {}
for ms in fullpath_mses:
outms = ms[:-3]+"_split_for_selfcal_mergedates.ms"
if not os.path.exists(outms):
# TODO: come back later & keep in DS...
split(vis=ms, outputvis=outms,
datacolumn='corrected',
field="Sgr B2 N Q,Sgr B2 NM Q,Sgr B2 MS Q,Sgr B2 S Q")
msname = ms[16:21]
if msname == '03_06':
if 'T12' in ms:
msname = '03_06_T12'
else:
msname = '03_06_T01'
myprint(msname)
selfcal_mses[msname] = outms
myprint(selfcal_mses)
cont_vis = list(selfcal_mses.values())
