def oldMSpcd(msFile):
"""
get phase center
works for most data without uv-regrid
Parameters
----------
msFile: string
measurement set filename
Returns
-------
pcd:
phase center
Note
----
by Shane
works for data without uv-regrid using CASA's fixvis()
the following will give the old phase center otherwise
"""
from taskinit import tb
tb.open(msFile + '/SOURCE')
pcd_ra = tb.getcol('DIRECTION')[0][0] * 180 / numpy.pi
if pcd_ra < 0:
pcd_ra += 360
pcd_dec = tb.getcol('DIRECTION')[1][0] * 180 / numpy.pi
tb.close()
pcd = [pcd_ra, pcd_dec]
return pcd
def get_array_config(vis):
msmd.open(vis)
obstime = Time(msmd.timerangeforobs(0)['begin']['m0']['value'],
format='mjd')
antennadiameter = msmd.antennadiameter(0)['value']
msmd.close()
if antennadiameter == 12:
if os.path.exists(vis + "/ASDM_EXECBLOCK"):
tb.open(vis + "/ASDM_EXECBLOCK")
else:
asdm_execblock = vis.replace(
"calibrated", "calibrated_pipeline") + "/ASDM_EXECBLOCK"
if os.path.exists(asdm_execblock):
tb.open(asdm_execblock)
else:
raise IOError("No ASDM_EXECBLOCK found for vis " + vis)
mous = tb.getcol('sessionReference')[0].split('"')[1].split("/")[-1]
configname = str(tb.getcol('configName')[0])
tb.close()
else:
configname = '7M'
mous = '7M'
return obstime.datetime, configname
def getAntennaPosition(vis):
tb.open(vis + '/ANTENNA')
position = tb.getcol('POSITION')
diameter = tb.getcol('DISH_DIAMETER')
antenna = tb.getcol('NAME')
tb.close()
return position, diameter, antenna
def get_all_freqs(msname):
tb.open(msname + "::SPECTRAL_WINDOW")
chanfreqs = tb.getcol('CHAN_FREQ')
n_chanfreqs, n_swids = chanfreqs.shape
ref_freq = tb.getcol('REF_FREQUENCY')[0]
n_freqs = n_swids * n_chanfreqs
dfs = tb.getcol('CHAN_WIDTH')
assert utils.allequal(dfs)
df = dfs[0, 0]
min_chan_freq = min(chanfreqs[0])
freqs = min_chan_freq + df * np.arange(n_freqs)
offsets = ((chanfreqs[0] - min_chan_freq) / df).astype(int)
return freqs, n_chanfreqs, n_freqs, df, offsets, ref_freq
def get_freqs(msname, swid):
tb.open(msname + "::SPECTRAL_WINDOW")
freqs = tb.getcol('CHAN_FREQ')[:, swid]
ref_freq = tb.getcol('REF_FREQUENCY')[swid]
(n_freqs, ) = freqs.shape
dfreqs = freqs - freqs[0]
#Patch to reproduce off by 1:
#freqs = freqs[0] + dfreqs * (n_freqs+1)/n_freqs
# Ends here
dfs = tb.getcol('CHAN_WIDTH')[:, swid]
dfreq = dfs[0]
assert all(np.equal(dfs, dfreq))
return freqs, n_freqs, dfreq, ref_freq
def getspwfreq(vis):
'''
:param vis:
:return: mid frequencies in GHz of each spw in the vis
'''
tb.open(vis + '/SPECTRAL_WINDOW')
reffreqs = tb.getcol('REF_FREQUENCY')
bdwds = tb.getcol('TOTAL_BANDWIDTH')
cfreqs = reffreqs + bdwds / 2.
tb.close()
cfreqs = cfreqs / 1.0e9
return cfreqs
def find_expected_beams(vis, spw, baseline_percentile=95):
'''
Return the expected synthesized beam (approximately) and the primary
beam size based on the baselines.
Parameters
----------
vis : str
Name of MS.
spw : int
Which SPW in the MS to consider.
baseline_percentile : int or float between 0 and 100
The percentile of the longest baseline to estimate the synthesized
beam with.
Returns
-------
syn_beam : float
Approximate Synthesized beam in arcseconds
prim_beam : float
Primary beam size in arcseconds.
'''
# Get percentile of max baseline and dish size
bline_max = getBaselinePercentile(vis, baseline_percentile)
tb.open(os.path.join(vis, 'ANTENNA'))
dishs = tb.getcol('DISH_DIAMETER')
dish_min = min(dishs)
tb.close()
tb.open(os.path.join(vis, 'SPECTRAL_WINDOW'))
ref_freqs = tb.getcol('REF_FREQUENCY')
try:
freq = ref_freqs[spw]
except IndexError:
raise IndexError("Given SPW ({0}) is not within the range of SPWs"
"found ({1})".format(spw, len(ref_freqs)))
# Find the beam
# XXX
# When astropy is easier to install (CASA 4.7??), switch to using the
# defined constants.
centre_lambda = 299792458.0 / freq
syn_beam = (centre_lambda / bline_max) * 180 / np.pi * 3600
prim_beam = (centre_lambda / dish_min) * 180 / np.pi * 3600
return syn_beam, prim_beam
def get_freq_wlcm_bm(vis=None):
tb.open(vis + '/SPECTRAL_WINDOW')
tb.open(vis + '/SPECTRAL_WINDOW')
reffreqs = tb.getcol('REF_FREQUENCY')
bdwds = tb.getcol('TOTAL_BANDWIDTH')
cfreqs = reffreqs + bdwds / 2.
tb.close()
sbeam = 40.
bm = np.zeros(30)
wlcm = np.zeros(30)
for sp in range(30):
cfreq = cfreqs[sp]
wlcm[sp] = 30000000000 / cfreq
bm[sp] = max(sbeam * cfreqs[1] / cfreq, 10.)
return (cfreqs, wlcm, bm)
def pcdload(visfile):
checker = visfile.find('uvfits')
if checker == -1:
uvfits = False
else:
uvfits = True
if uvfits:
# uv fits format
visdata = fits.open(visfile)
visheader = visdata[0].header
if visheader['NAXIS'] == 7:
# identify the phase center
try:
pcd_ra = visdata['AIPS SU '].data['RAEPO'][0]
pcd_dec = visdata['AIPS SU '].data['DECEPO'][0]
except:
pcd_ra = visheader['CRVAL6'] #RP mod was 5
pcd_dec = visheader['CRVAL7'] #RP mod was 6
if pcd_ra < 0:
pcd_ra += 360
pcd = [pcd_ra, pcd_dec]
return pcd
if visheader['NAXIS'] == 6:
# identify the channel frequency(ies):
pcd_ra = visdata[0].header['CRVAL5']
pcd_dec = visdata[0].header['CRVAL6']
if pcd_ra < 0:
pcd_ra += 360
pcd = [pcd_ra, pcd_dec]
return pcd
else:
# CASA MS
from taskinit import tb
tb.open(visfile + '/SOURCE')
pcd_ra = tb.getcol('DIRECTION')[0][0] * 180 / numpy.pi
if pcd_ra < 0:
pcd_ra += 360
pcd_dec = tb.getcol('DIRECTION')[1][0] * 180 / numpy.pi
tb.close()
pcd = [pcd_ra, pcd_dec]
return pcd
def pcdload(visfile):
checker = visfile.find('uvfits')
if checker == -1:
uvfits = False
else:
uvfits = True
if uvfits:
# uv fits format
visdata = fits.open(visfile)
visheader = visdata[0].header
if visheader['NAXIS'] == 7:
# identify the phase center
try:
pcd_ra = visdata['AIPS SU '].data['RAEPO'][0]
pcd_dec = visdata['AIPS SU '].data['DECEPO'][0]
except:
pcd_ra = visheader['CRVAL5']
pcd_dec = visheader['CRVAL6']
if pcd_ra < 0:
pcd_ra += 360
pcd = [pcd_ra, pcd_dec]
return pcd
if visheader['NAXIS'] == 6:
# identify the channel frequency(ies):
pcd_ra = visdata[0].header['CRVAL5']
pcd_dec = visdata[0].header['CRVAL6']
if pcd_ra < 0:
pcd_ra += 360
pcd = [pcd_ra, pcd_dec]
return pcd
else:
# CASA MS
from taskinit import tb
tb.open(visfile + '/SOURCE')
pcd_ra = tb.getcol('DIRECTION')[0][0] * 180 / numpy.pi
if pcd_ra < 0:
pcd_ra += 360
pcd_dec = tb.getcol('DIRECTION')[1][0] * 180 / numpy.pi
tb.close()
pcd = [pcd_ra, pcd_dec]
return pcd
def concateovsa(msname,
msfiles,
visprefix='./',
doclearcal=True,
keep_orig_ms=False,
cols2rm=["MODEL_DATA", "CORRECTED_DATA"]):
concatvis = visprefix + msname
msfiles_ = []
for idx, ll in enumerate(msfiles):
if str(ll).endswith('/'):
msfiles[idx] = str(ll)[:-1]
if doclearcal:
print 'Warning: Corrected column in the input ms file will be cleared!!!'
for ll in msfiles:
clearcal(vis=str(ll), addmodel=True)
else:
try:
tmpdir = visprefix + '/tmp_ms/'
if not os.path.exists(tmpdir):
os.makedirs(tmpdir)
for ll in msfiles:
msfile_ = tmpdir + os.path.basename(str(ll))
msfiles_.append(msfile_)
split(vis=str(ll), outputvis=msfile_, datacolumn='corrected')
clearcal(vis=msfile_, addmodel=True)
except:
print 'Warning: Corrected column not found in the input ms file.'
msfiles_ = msfiles
if msfiles_:
concat(vis=msfiles_, concatvis=concatvis, timesort=True)
else:
concat(vis=msfiles, concatvis=concatvis, timesort=True)
# Change all observation ids to be the same (zero)
tb.open(concatvis + '/OBSERVATION', nomodify=False)
nobs = tb.nrows()
tim0 = tb.getcell('TIME_RANGE', 0)[0]
tim1 = tb.getcell('TIME_RANGE', nobs - 1)[1]
tb.removerows([i + 1 for i in range(nobs - 1)])
tb.putcell('TIME_RANGE', 0, [tim0, tim1])
tb.close()
tb.open(concatvis, nomodify=False)
obsid = tb.getcol('OBSERVATION_ID')
newobsid = np.zeros(len(obsid), dtype='int')
tb.putcol('OBSERVATION_ID', newobsid)
colnames = tb.colnames()
for l in range(len(cols2rm)):
if cols2rm[l] in colnames:
try:
tb.removecols(cols2rm[l])
except:
pass
tb.close()
if msfiles_ != [] and msfiles_ != msfiles:
for ll in msfiles_:
os.system('rm -rf {}'.format(ll))
if not keep_orig_ms:
for ll in msfiles:
os.system('rm -rf {}'.format(ll))
def concat_slftb(tb_in=[], tb_out=None):
if not tb_in:
print('tb_in not provided. Abort...')
if os.path.exists(tb_out):
os.system('rm -r {}'.format(tb_out))
os.system('cp -r {} {}'.format(tb_in[0], tb_out))
tbdata = {}
tb.open(tb_out)
cols = tb.colnames()
tb.close()
cols.remove('WEIGHT')
for col in cols:
tbdata[col] = []
for tbidx, ctb in enumerate(tb_in):
tb.open(ctb, nomodify=True)
tim0 = tb.getcol(cols[0])
if len(tim0) == 0:
continue
else:
for col in cols:
if tbidx == 1 and col in ['CPARAM', 'PARAMERR', 'FLAG', 'SNR']:
tbdata[col].append(tb.getcol(col)[::-1,...])
else:
tbdata[col].append(tb.getcol(col))
tb.close()
if len(tbdata[cols[0]]) == 0:
print('tables have no data. Return')
return -1
else:
for col in cols:
if col in ['CPARAM', 'PARAMERR', 'FLAG', 'SNR']:
tbdata[col] = np.concatenate(tbdata[col], axis=2)
else:
tbdata[col] = np.concatenate(tbdata[col])
tb.open(tb_out, nomodify=False)
nrows = tb.nrows()
nrows_new = len(tbdata[cols[0]])
tb.addrows(nrows_new - nrows)
for col in cols:
tb.putcol(col, tbdata[col])
tb.close()
return tb_out
def flagcaltboutliers(caltable, limit=[]):
import numpy as np
import numpy.ma as ma
# def removeOutliers(x, outlierConstant):
# a = np.array(x)
# idx, = np.where(np.diff(np.sort(datamag[0, 0, :]))>)
# upper_quartile = np.percentile(a, 80)
# lower_quartile = np.percentile(a, 20)
# IQR = (upper_quartile - lower_quartile) * outlierConstant
# quartileSet = (lower_quartile - IQR, upper_quartile + IQR)
# return ma.masked_outside(x, quartileSet[1], quartileSet[0])
if not os.path.exists(caltable): return 0
if isinstance(limit, list):
if len(limit) == 2:
tb.open(caltable, nomodify=False)
# subt = tb.query("ANTENNA1==1 && SPECTRAL_WINDOW_ID=10")
# data = subt.getcol('CPARAM')
# flag = subt.getcol('FLAG')
# spw = subt.getcol('SPECTRAL_WINDOW_ID')
# datamag = np.abs(data)
# mdatamag = ma.masked_outside(datamag, limit[0], limit[1])
# mask = np.logical_or(mdatamag.mask, flag)
# dataidx1 = datamag<limit[0]
# dataidx2 = datamag>limit[1]
# mdatamag = ma.masked_array(mdatamag, mask)
# mdatamag[0, 0, :] = removeOutliers(mdatamag[0, 0, :], 5)
# mdatamag[1, 0, :] = removeOutliers(mdatamag[1, 0, :], 5)
data = tb.getcol('CPARAM')
flag = tb.getcol('FLAG')
datamag = np.abs(data)
dataidx1 = datamag < limit[0]
dataidx2 = datamag > limit[1]
flag[dataidx1] = True
flag[dataidx2] = True
tb.putcol('FLAG', flag)
return 1
else:
print('limit must have two elements. Aborted!')
return 0
else:
print('limit must be a list. Aborted!')
def make_pbfile(vis, pbfile):
from taskinit import im, ms, ia, qa, tb
import numpy as np
from scipy.constants import c
ms.open(vis)
fields = ms.range('field_id')['field_id']
ms.done()
im.open(vis)
im.selectvis(field=fields[0])
ms.open(vis)
freq = np.mean(ms.range('chan_freq')['chan_freq'])
phase_dir = ms.range('phase_dir')['phase_dir']['direction']
ms.done()
phase_dir = phase_dir[0][0], phase_dir[1][0]
phase_dir = [
qa.formxxx(str(phase_dir[0]) + 'rad', format='hms'),
qa.formxxx(str(phase_dir[1]) + 'rad', format='dms')
]
phase_dir = 'J2000 ' + ' '.join(phase_dir)
tb.open(vis + '/ANTENNA/')
dishdia = np.min(tb.getcol('DISH_DIAMETER'))
tb.done()
# pb of 512 pix cover pb down to 0.001
# ensure largest pixel to pixel var to .01
minpb = 0.001
nx = 512
cellconv = (nx * np.sqrt(np.log(2) / np.log(1 / minpb)))**-1
beam = c / freq / dishdia
cell = {}
cell['value'] = beam * cellconv
cell['unit'] = 'rad'
# nx = int(3*3e8/freq/dishdia*1.22*180/
# math.pi*3600/qa.convert(advise['cell'],
# 'arcsec')['value'])
# Chosen as to be 3 times fwhm of primary beam,
# should include up to approximately .01 of peak flux
im.defineimage(nx=nx, ny=nx, cellx=cell, celly=cell, phasecenter=phase_dir)
im.setvp(dovp=True)
im.makeimage(type='pb', image=pbfile)
im.done()
ia.open(pbfile)
cs = ia.coordsys()
cs.setreferencevalue(type='direction', value=[0., 0.])
ia.setcoordsys(cs.torecord())
ia.maskhandler('delete', 'mask0')
ia.done()
def getFreqfromtb(msFile):
"""get a list of IF from the measurement set table
same as ('CHAN_FREQ') if
Parameters
----------
msFile: string
CASA measurement set
Returns
-------
freq0: numpy.ndarray
reference frequency in Hz
"""
from taskinit import tb
tb.open(msFile+'/SPECTRAL_WINDOW')
freq0 = tb.getcol('REF_FREQUENCY')
nchan = tb.getcol('NUM_CHAN')
tb.close()
return freq0
def MSpcd(msFile):
"""
Explore different ways to get phase center consistent with data that has been regridded from one epoch to another
Parameters
----------
msFile: string
measurement set filename
Returns
-------
pcd: list
phase center
Note
----
which method gives the phase center that correspond to that used in CASA imaging needs further investigation
"""
from taskinit import tb
tb.open(msFile + '/FIELD')
old = tb.getcol('DELAY_DIR') #, fieldid
new = tb.getcol('PHASE_DIR')
def shiftRA(ra):
if ra < 0.: ra += numpy.pi * 2
return ra
old[0] = shiftRA(old[0])
new[0] = shiftRA(new[0])
# old phase center
# _pcd_ra, _pcd_dec = old[0] * 180. / numpy.pi, old[1] * 180. / numpy.pi
# new phase center
pcd_ra, pcd_dec = new[0] * 180. / numpy.pi, new[1] * 180. / numpy.pi
tb.close()
pcd = pcd_ra[0][0], pcd_dec[0][0]
return list(pcd)
def clearflagrow(msfile, mode='clear'):
'''
:param msfile:
:param mode: FLAG_ROW operation
default: 'clear': (default) clear the FLAG_ROW
'list': to list existing FLAG_ROW
:return:
'''
if mode == 'list':
tb.open(msfile, nomodify=True)
a = tb.getcol('FLAG_ROW')
nfrows = np.sum(a)
nrows = float(len(a))
print('{:0d} out of {:.0f} ({:.0f}%) rows are flagged in {}'.format(nfrows, nrows, nfrows / nrows * 100,
os.path.basename(msfile)))
elif mode == 'clear':
tb.open(msfile, nomodify=False)
a = tb.getcol('FLAG_ROW')
a[:] = False
tb.putcol('FLAG_ROW', a)
print('reset successfully')
tb.close()
def insertdiskmodel(vis, sizescale=1.0, fdens=None, dsize=None, xmlfile='SOLDISK.xml', writediskinfoonly=False,
active=False, overwrite=True):
# Apply size scale adjustment (default is no adjustment)
for i in range(len(dsize)):
num, unit = dsize[i].split('arc')
dsize[i] = str(float(num) * sizescale)[:6] + 'arc' + unit
msfile = vis
ms.open(msfile)
spwinfo = ms.getspectralwindowinfo()
nspw = len(spwinfo.keys())
ms.close()
diskimdir = 'diskim/'
if not os.path.exists(diskimdir):
os.makedirs(diskimdir)
frq = []
spws = range(nspw)
for sp in spws:
spw = spwinfo[str(sp)]
frq.append('{:.4f}GHz'.format((spw['RefFreq'] + spw['TotalWidth'] / 2.0) / 1e9))
frq = np.array(frq)
writediskxml(dsize, fdens, frq, xmlfile=xmlfile)
if not writediskinfoonly:
tb.open(msfile + '/FIELD')
phadir = tb.getcol('PHASE_DIR').flatten()
tb.close()
ra = phadir[0]
dec = phadir[1]
direction = 'J2000 ' + str(ra) + 'rad ' + str(dec) + 'rad'
diskim = []
for sp in tqdm(spws, desc='Generating {} disk models'.format(nspw), ascii=True):
diskim.append(
mk_diskmodel(outname=diskimdir + 'disk{:02d}_'.format(sp), bdwidth=spwinfo[str(sp)],
direction=direction,
reffreq=frq[sp],
flux=fdens[sp], eqradius=dsize[sp], polradius=dsize[sp], overwrite=overwrite))
if not active:
delmod(msfile, otf=True, scr=True)
for sp in tqdm(spws, desc='Inserting disk model', ascii=True):
ft(vis=msfile, spw=str(sp), field='', model=str(diskim[sp]), nterms=1,
reffreq="", complist="", incremental=True, usescratch=True)
return msfile, diskim
def guesspb(vis):
"""
Produces a PrimaryBeamModel from a measurementset
"""
print(vis)
from taskinit import ms, tb, qa
ms.open(vis)
freq = (np.mean(ms.range('chan_freq')['chan_freq']) / 1e9)
ms.done()
tb.open(vis + '/OBSERVATION')
telescope = tb.getcol('TELESCOPE_NAME')[0]
tb.done()
pbfile = '{0}-{1:.1f}GHz.pb'.format(telescope, freq)
print(pbfile)
if not os.access(pbfile, os.F_OK):
stacker.make_pbfile(vis, pbfile)
return MSPrimaryBeamModel(pbfile)
def getBaselinePercentile(msFile, percentile):
"""
Based on getBaselineExtrema from analysisUtils
"""
tb.open(msFile + '/ANTENNA')
positions = np.transpose(tb.getcol('POSITION'))
tb.close()
all_lengths = []
for i in range(len(positions)):
for j in range(i + 1, len(positions)):
length = au.computeBaselineLength(positions[i],
positions[j])
if length != 0.0:
all_lengths.append(length)
all_lengths = np.array(all_lengths)
return np.percentile(all_lengths, percentile)
def set_imagermode(vis, source):
tb.open(os.path.join(vis, 'FIELD'))
names = tb.getcol('NAME')
tb.close()
moscount = 0
for name in names:
chsrc = name.find(source)
if chsrc != -1:
moscount = moscount + 1
if moscount > 1:
imagermode = "mosaic"
else:
imagermode = "csclean"
return imagermode
def writeVis(vis_complex, visdataloc, modelvisloc, miriad=False):
if miriad:
os.system('rm -rf ' + modelvisloc)
cmd = 'cp ' + visdataloc + ' ' + modelvisloc
os.system(cmd)
# get the real and imaginary components
real = numpy.real(vis_complex)
imag = numpy.imag(vis_complex)
# replace data visibilities with model visibilities
visfile = fits.open(modelvisloc, mode='update')
visibilities = visfile[0].data
visheader = visfile[0].header
if visheader['NAXIS'] == 7:
visibilities['DATA'][:, 0, 0, :, :, :, 0] = real
visibilities['DATA'][:, 0, 0, :, :, :, 1] = imag
elif visheader['NAXIS'] == 6:
visibilities['DATA'][:, 0, 0, :, :, 0] = real
visibilities['DATA'][:, 0, 0, :, :, 1] = imag
else:
print("Visibility dataset has >7 or <6 axes. I can't read this.")
#visibilities['DATA'][:, 0, 0, :, :, 2] = wgt
# replace the data visibilities with the model visibilities
visfile[0].data = visibilities
visfile.flush()
else:
from taskinit import tb
print("Writing visibility data to " + modelvisloc)
os.system('rm -rf ' + modelvisloc)
tb.open(visdataloc)
tb.copy(modelvisloc)
tb.close()
tb.open(modelvisloc, nomodify=False)
datashape = tb.getcol('DATA').shape
vis_complex = vis_complex.reshape(datashape)
tb.putcol('DATA', vis_complex)
tb.close()
def writeVis(vis_complex, visdataloc, modelvisloc, miriad=False):
if miriad:
os.system('rm -rf ' + modelvisloc)
cmd = 'cp ' + visdataloc + ' ' + modelvisloc
os.system(cmd)
# get the real and imaginary components
real = numpy.real(vis_complex)
imag = numpy.imag(vis_complex)
# replace data visibilities with model visibilities
visfile = fits.open(modelvisloc, mode='update')
visibilities = visfile[0].data
visheader = visfile[0].header
if visheader['NAXIS'] == 7:
visibilities['DATA'][:, 0, 0, :, :, :, 0] = real
visibilities['DATA'][:, 0, 0, :, :, :, 1] = imag
elif visheader['NAXIS'] == 6:
visibilities['DATA'][:, 0, 0, :, :, 0] = real
visibilities['DATA'][:, 0, 0, :, :, 1] = imag
else:
print("Visibility dataset has >7 or <6 axes. I can't read this.")
#visibilities['DATA'][:, 0, 0, :, :, 2] = wgt
# replace the data visibilities with the model visibilities
visfile[0].data = visibilities
visfile.flush()
else:
from taskinit import tb
print("Writing visibility data to " + modelvisloc)
os.system('rm -rf ' + modelvisloc)
tb.open(visdataloc)
tb.copy(modelvisloc)
tb.close()
tb.open(modelvisloc, nomodify=False)
datashape = tb.getcol('DATA').shape
vis_complex = vis_complex.reshape(datashape)
tb.putcol('DATA', vis_complex)
tb.close()
def has_field(vis, source):
'''
Check if source is contained in at least one of the field names.
'''
tb.open(os.path.join(vis, 'FIELD'))
names = tb.getcol('NAME')
tb.close()
moscount = 0
for name in names:
chsrc = name.find(source)
if chsrc != -1:
moscount = moscount + 1
if moscount == 0:
return False
return True
def writeVis(vis_complex, visdataloc, modelvisloc, miriad=False):
"""
Parameters
----------
vis_complex: numpy.array
visdataloc: string
uv-data filename
miriad: Boolean
True: not using CASA, just python data manipulation, hence can run in ipython,
False: use CASA taskinit module, must run inside CASA
Returns
-------
modelvisloc: string
uv-model filename
"""
if miriad:
os.system('rm -rf ' + modelvisloc)
cmd = 'cp ' + visdataloc + ' ' + modelvisloc
os.system(cmd)
# get the real and imaginary components
real = numpy.real(vis_complex)
imag = numpy.imag(vis_complex)
# replace data visibilities with model visibilities
visfile = fits.open(modelvisloc, mode='update')
visibilities = visfile[0].data
visheader = visfile[0].header
if visheader['NAXIS'] == 7:
# expect uu.dim = 4, hence real.ndim also 4
nfreq = visibilities['DATA'][0, 0, 0, 0, :, 0, 0].size
# to match uvutil.uvload() instead of fixing array size mismatch with miriad= in the function
if nfreq > 1:
visibilities['DATA'][:, 0, 0, :, :, :, 0] = real
visibilities['DATA'][:, 0, 0, :, :, :, 1] = imag
else:
try:
visibilities['DATA'][:, 0, 0, :, :, :, 0] = real
visibilities['DATA'][:, 0, 0, :, :, :, 1] = imag
except ValueError:
# mistmatach arise when uu.ndim is 3
# which would be the case if nfreq = 0
# vis, 0, 0, spw, chan(freq), pol, (real, imag, weights)
visibilities['DATA'][:, 0, 0, :, 0, :, 0] = real
visibilities['DATA'][:, 0, 0, :, 0, :, 1] = imag
elif visheader['NAXIS'] == 6:
visibilities['DATA'][:, 0, 0, :, :, 0] = real
visibilities['DATA'][:, 0, 0, :, :, 1] = imag
else:
print("Visibility dataset has >7 or <6 axes. I can't read this.")
#visibilities['DATA'][:, 0, 0, :, :, 2] = wgt
# replace the data visibilities with the model visibilities
visfile[0].data = visibilities
visfile.flush()
else:
from taskinit import tb
print("Writing visibility data to " + modelvisloc)
os.system('rm -rf ' + modelvisloc)
tb.open(visdataloc)
tb.copy(modelvisloc)
tb.close()
tb.open(modelvisloc, nomodify=False)
datashape = tb.getcol('DATA').shape
vis_complex = vis_complex.reshape(datashape)
tb.putcol('DATA', vis_complex)
tb.close()
def subtract_outliers(vis, outlier_coords, field='M33*', split_fields=True,
stokes='I', interactive=True, weighting='natural',
threshold='5mJy/beam', cell='3arcsec', cleanup=False,
datacolumn="CORRECTED", imsize=64, save_space=False,
masks=None, multiscale=[]):
'''
Subtract an outlier at the given coordinates. Splits out each field,
tries to image at that coordinate, then subracts of the model in the UV
plane. The fields are concatenated into a final ms.
'''
vis = vis.rstrip("/")
# Get some info from the image
tb.open(os.path.join(vis, 'FIELD'))
all_fields = tb.getcol('NAME')
tb.close()
regex = re.compile(field)
fields = [f for f in all_fields if re.match(regex, f)]
# If only one position given, convert to list for iteration.
if isinstance(outlier_coords, six.string_types):
outlier_coords = [outlier_coords]
if masks is not None:
if len(masks) != len(outlier_coords):
raise Warning("The number of specified masks must match the"
" number of coordinates.")
try:
os.mkdir('temp_files')
except OSError:
warnings.warn("temp_files already exists. "
"Going to remove all ms files from it.")
catch_fail(rmtables, tablenames='temp_files/*')
# Loop through the fields
for f in fields:
fieldvis = os.path.join('temp_files', f+".ms")
fieldimg = os.path.join('temp_files', f)
# Split the field off
catch_fail(split, vis=vis, outputvis=fieldvis, field=f,
datacolumn=datacolumn)
# Now image the data, keeping the phasecenter at the outlier
# Image each outlier at its phasecenter, then uvsub
for i, coord in enumerate(outlier_coords):
outfield_img = fieldimg + "_" + str(i)
if masks is not None:
mask = masks[i]
else:
mask = None
catch_fail(clean, vis=fieldvis, imagename=outfield_img, mode='mfs',
phasecenter=coord, niter=10000, usescratch=True,
interactive=interactive, cell='3arcsec',
imsize=imsize, threshold=threshold, weighting=weighting,
minpb=0.0, mask=mask, multiscale=multiscale)
# Subtract out the model from the imaging
catch_fail(uvsub, vis=fieldvis)
# Remove the individual images
if cleanup:
catch_fail(rmtables, tablenames=outfield_img+"*")
if save_space:
fieldvis_corronly = \
os.path.join('temp_files', f+"_corrected.ms")
catch_fail(split, vis=fieldvis, outputvis=fieldvis_corronly,
field=f, datacolumn="CORRECTED")
catch_fail(rmtables, tablenames=fieldvis)
# Now append the uvsub fields back together
individ_ms = glob.glob("temp_files/*.ms")
catch_fail(concat, vis=individ_ms,
concatvis=vis.rstrip(".ms")+"_outsub.ms",
respectname=True)
if cleanup:
catch_fail(rmtables, tablenames='temp_files/*')
def uvload(visfile):
checker = visfile.find('uvfits')
if checker == -1:
uvfits = False
else:
uvfits = True
if uvfits:
visdata = fits.open(visfile)
visibilities = visdata[0].data
visheader = visdata[0].header
if visheader['NAXIS'] == 7:
# identify the channel frequency(ies):
visfreq = visdata[1].data
freq0 = visheader['CRVAL4']
dfreq = visheader['CDELT4']
cfreq = visheader['CRPIX4']
nvis = visibilities['DATA'][:, 0, 0, 0, 0, 0, 0].size
nspw = visibilities['DATA'][0, 0, 0, :, 0, 0, 0].size
nfreq = visibilities['DATA'][0, 0, 0, 0, :, 0, 0].size
npol = visibilities['DATA'][0, 0, 0, 0, 0, :, 0].size
if True or nfreq > 1: #RP mod inserted True
uu = numpy.zeros([nvis, nspw, nfreq, npol])
vv = numpy.zeros([nvis, nspw, nfreq, npol])
ww = numpy.zeros([nvis, nspw, nfreq, npol])
else:
uu = numpy.zeros([nvis, nspw, npol])
vv = numpy.zeros([nvis, nspw, npol])
ww = numpy.zeros([nvis, nspw, npol])
#wgt = numpy.zeros([nvis, nspw, nfreq, npol])
for ispw in range(nspw):
if nspw > 1:
freqif = freq0 + visfreq['IF FREQ'][0][ispw]
else:
try:
freqif = freq0 + visfreq['IF FREQ'][0]
except:
freqif = freq0
#uu[:, ispw] = freqif * visibilities['UU']
#vv[:, ispw] = freqif * visibilities['VV']
for ipol in range(npol):
# then compute the spatial frequencies:
if True or nfreq > 1: #RP mod inserted True
freq = (numpy.arange(nfreq) - cfreq +
1) * dfreq + freqif
freqvis = numpy.meshgrid(freq, visibilities['UU'])
uu[:, ispw, :, ipol] = freqvis[0] * freqvis[1]
freqvis = numpy.meshgrid(freq, visibilities['VV'])
vv[:, ispw, :, ipol] = freqvis[0] * freqvis[1]
freqvis = numpy.meshgrid(freq, visibilities['WW'])
ww[:, ispw, :, ipol] = freqvis[0] * freqvis[1]
else:
uu[:, ispw, ipol] = freqif * visibilities['UU']
vv[:, ispw, ipol] = freqif * visibilities['VV']
ww[:, ispw, ipol] = freqif * visibilities['WW']
if visheader['NAXIS'] == 6:
# identify the channel frequency(ies):
freq0 = visheader['CRVAL4']
dfreq = visheader['CDELT4']
cfreq = visheader['CRPIX4']
nvis = visibilities['DATA'][:, 0, 0, 0, 0, 0].size
nfreq = visibilities['DATA'][0, 0, 0, :, 0, 0].size
npol = visibilities['DATA'][0, 0, 0, 0, :, 0].size
if nfreq > 1:
uu = numpy.zeros([nvis, nfreq, npol])
vv = numpy.zeros([nvis, nfreq, npol])
ww = numpy.zeros([nvis, nfreq, npol])
else:
uu = numpy.zeros([nvis, npol])
vv = numpy.zeros([nvis, npol])
ww = numpy.zeros([nvis, npol])
#wgt = numpy.zeros([nvis, nspw, nfreq, npol])
freqif = freq0
#uu[:, ispw] = freqif * visibilities['UU']
#vv[:, ispw] = freqif * visibilities['VV']
for ipol in range(npol):
# then compute the spatial frequencies:
if nfreq > 1:
freq = (numpy.arange(nfreq) - cfreq + 1) * dfreq + freqif
freqvis = numpy.meshgrid(freq, visibilities['UU'])
uu[:, 0, :, ipol] = freqvis[0] * freqvis[1]
freqvis = numpy.meshgrid(freq, visibilities['VV'])
vv[:, 0, :, ipol] = freqvis[0] * freqvis[1]
freqvis = numpy.meshgrid(freq, visibilities['WW'])
ww[:, 0, :, ipol] = freqvis[0] * freqvis[1]
else:
uu[:, ipol] = freqif * visibilities['UU']
vv[:, ipol] = freqif * visibilities['VV']
ww[:, ipol] = freqif * visibilities['WW']
else:
from taskinit import tb
# read in the uvfits data
tb.open(visfile)
uvw = tb.getcol('UVW')
uvspw = tb.getcol('DATA_DESC_ID')
tb.close()
tb.open(visfile + '/SPECTRAL_WINDOW')
freq = tb.getcol('CHAN_FREQ')
tb.close()
tb.open(visfile + '/POLARIZATION')
polinfo = tb.getcol('NUM_CORR')
tb.close()
npol = polinfo[0]
nspw = len(freq[0])
for ispw in range(nspw):
ilam = 3e8 / freq[0][ispw]
indx_spw = uvspw == ispw
uvw[:, indx_spw] /= ilam
uu = []
vv = []
ww = []
for ipol in range(npol):
uu.append(uvw[0, :])
vv.append(uvw[1, :])
ww.append(uvw[2, :])
uu = numpy.array(uu)
vv = numpy.array(vv)
ww = numpy.array(ww)
if uu[:, 0].size == 1:
uu = uu.flatten()
vv = vv.flatten()
ww = ww.flatten()
return uu, vv, ww
antennas = '' # default is to use all 13 EOVSA 2-m antennas.
npix = 128 # number of pixels in the image
cell = '2arcsec' # pixel scale in arcsec
pol = 'XX' # polarization to image, use XX for now
pbcor = True # correct for primary beam response?
grid_spacing = 2. * u.deg # spacing for plotting longitude and latitude grid in degrees
outdir = './image_series/' # Specify where you want to save the output fits files
imresfile = 'imres.npz' # File to write the imageing results summary
outimgpre = 'EO' # Something to add to the image name
#################################################################################
################### (OPTIONAL) DEFINE RESTORING BEAM SIZE ##################
### (Optional) the following is to define beam size ###
# read frequency information from visibility data
tb.open(vis + '/SPECTRAL_WINDOW')
reffreqs = tb.getcol('REF_FREQUENCY')
bdwds = tb.getcol('TOTAL_BANDWIDTH')
cfreqs = reffreqs + bdwds / 2.
tb.close()
spws = [str(s + 1) for s in range(30)]
sbeam = 30.
bmsz = [max(sbeam * cfreqs[1] / f, 6.) for f in cfreqs]
#################################################################################
############# This block checks if the script is being run on Virgo ############
import socket
if socket.gethostname() == 'ip-172-26-5-203.ec2.internal':
print(
'!!!!!!Caution!!!!!!: We detected that you are trying to run this computationally intensive script on Virgo.'
)
def importeovsa_iter(filelist, timebin, width, visprefix, nocreatms, modelms,
doscaling, keep_nsclms, fileidx):
from taskinit import tb, casalog
filename = filelist[fileidx]
uv = aipy.miriad.UV(filename)
# try:
msname0 = list(filename.split('/')[-1])
msname = visprefix + ''.join(msname0) + '.ms'
uv.select('antennae', 0, 1, include=True)
uv.select('polarization', -5, -5, include=True)
times = []
uv.rewind()
for preamble, data in uv.all():
uvw, t, (i, j) = preamble
times.append(t)
uv.select('clear', -1, -1, include=True)
times = ipe.jd2mjds(np.asarray(times))
inttime = np.median((times - np.roll(times, 1))[1:]) / 60
time_steps = len(times)
durtim = int((times[-1] - times[0]) / 60 + inttime)
time0 = time.time()
if 'antlist' in uv.vartable:
ants = uv['antlist'].replace('\x00', '')
antlist = map(int, ants.split())
else:
antlist = [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16]
good_idx = np.where(uv['sfreq'] > 0)[0]
nf = len(good_idx)
npol = uv['npol']
nants = uv['nants']
source_id = uv['source'].replace('\x00', '')
sfreq = uv['sfreq'][good_idx]
sdf = uv['sdf'][good_idx]
ra, dec = uv['ra'], uv['dec']
nbl = nants * (nants - 1) / 2
bl2ord = ipe.bl_list2(nants)
npairs = nbl + nants
flag = np.ones((npol, nf, time_steps, npairs), dtype=bool)
out = np.zeros((npol, nf, time_steps, npairs),
dtype=np.complex64) # Cross-correlations
uvwarray = np.zeros((3, time_steps, npairs), dtype=np.float)
chan_band = ipe.get_band(sfreq=sfreq, sdf=sdf)
nband = len(chan_band)
uv.rewind()
l = -1
for preamble, data in uv.all():
uvw, t, (i0, j0) = preamble
i = antlist.index(i0 + 1)
j = antlist.index(j0 + 1)
if i > j:
# Reverse order of indices
j = antlist.index(i0 + 1)
i = antlist.index(j0 + 1)
# Assumes uv['pol'] is one of -5, -6, -7, -8
k = -5 - uv['pol']
l += 1
data = ma.masked_array(ma.masked_invalid(data), fill_value=0.0)
out[k, :, l / (npairs * npol), bl2ord[i0, j0]] = data.data
flag[k, :, l / (npairs * npol), bl2ord[i0, j0]] = data.mask
# if i != j:
if k == 3:
uvwarray[:, l / (npairs * npol),
bl2ord[i0, j0]] = -uvw * constants.speed_of_light / 1e9
nrows = time_steps * npairs
if doscaling:
out2 = out.copy()
for i0 in antlist:
for j0 in antlist:
if i0 < j0:
i, j = i0 - 1, j0 - 1
out2[:, :, :,
bl2ord[i, j]] = out[:, :, :, bl2ord[i, j]] / np.sqrt(
np.abs(out[:, :, :, bl2ord[i, i]]) *
np.abs(out[:, :, :, bl2ord[j, j]]))
out2 = out2.reshape(npol, nf, nrows)
out2[np.isnan(out2)] = 0
out2[np.isinf(out2)] = 0
# out2 = ma.masked_array(ma.masked_invalid(out2), fill_value=0.0)
out = out.reshape(npol, nf, nrows) * 1e4
flag = flag.reshape(npol, nf, nrows)
uvwarray = uvwarray.reshape(3, nrows)
uvwarray = np.tile(uvwarray, (1, nband))
sigma = np.ones((4, nrows), dtype=np.float) + 1
sigma = np.tile(sigma, (1, nband))
casalog.post('IDB File {0} is readed in --- {1:10.2f} seconds ---'.format(
filename, (time.time() - time0)))
if not nocreatms:
modelms = ipe.creatms(filename, visprefix)
os.system('mv {} {}'.format(modelms, msname))
else:
casalog.post('----------------------------------------')
casalog.post('copying standard MS to {0}'.format(
msname, (time.time() - time0)))
casalog.post('----------------------------------------')
os.system("rm -fr %s" % msname)
os.system("cp -r " + " %s" % modelms + " %s" % msname)
casalog.post(
'Standard MS is copied to {0} in --- {1:10.2f} seconds ---'.format(
msname, (time.time() - time0)))
tb.open(msname, nomodify=False)
casalog.post('----------------------------------------')
casalog.post("Updating the main table of" '%s' % msname)
casalog.post('----------------------------------------')
for l, cband in enumerate(chan_band):
time1 = time.time()
nchannels = len(cband['cidx'])
for row in range(nrows):
if not doscaling or keep_nsclms:
tb.putcell('DATA', (row + l * nrows),
out[:, cband['cidx'][0]:cband['cidx'][-1] + 1, row])
tb.putcell('FLAG', (row + l * nrows),
flag[:, cband['cidx'][0]:cband['cidx'][-1] + 1, row])
casalog.post(
'---spw {0:02d} is updated in --- {1:10.2f} seconds ---'.format(
(l + 1),
time.time() - time1))
tb.putcol('UVW', uvwarray)
tb.putcol('SIGMA', sigma)
tb.putcol('WEIGHT', 1.0 / sigma**2)
timearr = times
timearr = timearr.reshape(1, time_steps, 1)
timearr = np.tile(timearr, (nband, 1, npairs))
timearr = timearr.reshape(nband * npairs * time_steps)
tb.putcol('TIME', timearr)
tb.putcol('TIME_CENTROID', timearr)
scan_id = tb.getcol('SCAN_NUMBER')
scan_id *= 0
tb.putcol('SCAN_NUMBER', scan_id)
colnames = tb.colnames()
cols2rm = ["MODEL_DATA", "CORRECTED_DATA"]
for l in range(len(cols2rm)):
if cols2rm[l] in colnames:
tb.removecols(cols2rm[l])
tb.close()
casalog.post('----------------------------------------')
casalog.post("Updating the OBSERVATION table of" '%s' % msname)
casalog.post('----------------------------------------')
tb.open(msname + '/OBSERVATION', nomodify=False)
tb.putcol(
'TIME_RANGE',
np.asarray([times[0] - 0.5 * inttime,
times[-1] + 0.5 * inttime]).reshape(2, 1))
tb.putcol('OBSERVER', ['EOVSA team'])
tb.close()
casalog.post('----------------------------------------')
casalog.post("Updating the POINTING table of" '%s' % msname)
casalog.post('----------------------------------------')
tb.open(msname + '/POINTING', nomodify=False)
timearr = times.reshape(1, time_steps, 1)
timearr = np.tile(timearr, (nband, 1, nants))
timearr = timearr.reshape(nband * time_steps * nants)
tb.putcol('TIME', timearr)
tb.putcol('TIME_ORIGIN', timearr) # - 0.5 * delta_time)
direction = tb.getcol('DIRECTION')
direction[0, 0, :] = ra
direction[1, 0, :] = dec
tb.putcol('DIRECTION', direction)
target = tb.getcol('TARGET')
target[0, 0, :] = ra
target[1, 0, :] = dec
tb.putcol('TARGET', target)
tb.close()
casalog.post('----------------------------------------')
casalog.post("Updating the SOURCE table of" '%s' % msname)
casalog.post('----------------------------------------')
tb.open(msname + '/SOURCE', nomodify=False)
radec = tb.getcol('DIRECTION')
radec[0], radec[1] = ra, dec
tb.putcol('DIRECTION', radec)
name = np.array([source_id], dtype='|S{0}'.format(len(source_id) + 1))
tb.putcol('NAME', name)
tb.close()
casalog.post('----------------------------------------')
casalog.post("Updating the DATA_DESCRIPTION table of" '%s' % msname)
casalog.post('----------------------------------------')
tb.open(msname + '/DATA_DESCRIPTION/', nomodify=False)
pol_id = tb.getcol('POLARIZATION_ID')
pol_id *= 0
tb.putcol('POLARIZATION_ID', pol_id)
# spw_id = tb.getcol('SPECTRAL_WINDOW_ID')
# spw_id *= 0
# tb.putcol('SPECTRAL_WINDOW_ID', spw_id)
tb.close()
# casalog.post('----------------------------------------')
# casalog.post("Updating the POLARIZATION table of" '%s' % msname)
# casalog.post('----------------------------------------')
# tb.open(msname + '/POLARIZATION/', nomodify=False)
# tb.removerows(rownrs=np.arange(1, nband, dtype=int))
# tb.close()
casalog.post('----------------------------------------')
casalog.post("Updating the FIELD table of" '%s' % msname)
casalog.post('----------------------------------------')
tb.open(msname + '/FIELD/', nomodify=False)
delay_dir = tb.getcol('DELAY_DIR')
delay_dir[0], delay_dir[1] = ra, dec
tb.putcol('DELAY_DIR', delay_dir)
phase_dir = tb.getcol('PHASE_DIR')
phase_dir[0], phase_dir[1] = ra, dec
tb.putcol('PHASE_DIR', phase_dir)
reference_dir = tb.getcol('REFERENCE_DIR')
reference_dir[0], reference_dir[1] = ra, dec
tb.putcol('REFERENCE_DIR', reference_dir)
name = np.array([source_id], dtype='|S{0}'.format(len(source_id) + 1))
tb.putcol('NAME', name)
tb.close()
# FIELD: DELAY_DIR, PHASE_DIR, REFERENCE_DIR, NAME
# del out, flag, uvwarray, uv, timearr, sigma
# gc.collect() #
if doscaling:
if keep_nsclms:
msname_scl = visprefix + ''.join(msname0) + '_scl.ms'
os.system('cp -r {} {}'.format(msname, msname_scl))
else:
msname_scl = msname
tb.open(msname_scl, nomodify=False)
casalog.post('----------------------------------------')
casalog.post("Updating the main table of" '%s' % msname_scl)
casalog.post('----------------------------------------')
for l, cband in enumerate(chan_band):
time1 = time.time()
for row in range(nrows):
tb.putcell(
'DATA', (row + l * nrows),
out2[:, cband['cidx'][0]:cband['cidx'][-1] + 1, row])
casalog.post(
'---spw {0:02d} is updated in --- {1:10.2f} seconds ---'.
format((l + 1),
time.time() - time1))
tb.close()
if not (timebin == '0s' and width == 1):
msfile = msname + '.split'
if doscaling:
split(vis=msname_scl,
outputvis=msname_scl + '.split',
datacolumn='data',
timebin=timebin,
width=width,
keepflags=False)
os.system('rm -rf {}'.format(msname_scl))
msfile_scl = msname_scl + '.split'
if not (doscaling and not keep_nsclms):
split(vis=msname,
outputvis=msname + '.split',
datacolumn='data',
timebin=timebin,
width=width,
keepflags=False)
os.system('rm -rf {}'.format(msname))
else:
msfile = msname
if doscaling:
msfile_scl = msname_scl
casalog.post("finished in --- %s seconds ---" % (time.time() - time0))
if doscaling:
return [True, msfile, msfile_scl, durtim]
else:
return [True, msfile, durtim]
def visload(visfile):
checker = visfile.find('uvfits')
if checker == -1:
uvfits = False
else:
uvfits = True
if uvfits:
visdata = fits.open(visfile)
# get the telescope name
visheader = visdata[0].header
#telescop = visheader['TELESCOP']
# if we are dealing with SMA data
if visheader['NAXIS'] == 6:
nfreq = visdata[0].data['DATA'][0, 0, 0, :, 0, 0].size
if nfreq > 1:
data_real = visdata[0].data['DATA'][:,0,0,:,:,0]
data_imag = visdata[0].data['DATA'][:,0,0,:,:,1]
data_wgt = visdata[0].data['DATA'][:,0,0,:,:,2]
else:
data_real = visdata[0].data['DATA'][:,0,0,0,:,0]
data_imag = visdata[0].data['DATA'][:,0,0,0,:,1]
data_wgt = visdata[0].data['DATA'][:,0,0,0,:,2]
# if we are dealing with ALMA or PdBI data
if visheader['NAXIS'] == 7:
nfreq = visdata[0].data['DATA'][0, 0, 0, 0, :, 0, 0].size
if nfreq > 1:
data_real = visdata[0].data['DATA'][:,0,0,:,:,:,0]
data_imag = visdata[0].data['DATA'][:,0,0,:,:,:,1]
data_wgt = visdata[0].data['DATA'][:,0,0,:,:,:,2]
else:
data_real = visdata[0].data['DATA'][:,0,0,:,0,:,0]
data_imag = visdata[0].data['DATA'][:,0,0,:,0,:,1]
data_wgt = visdata[0].data['DATA'][:,0,0,:,0,:,2]
data_complex = numpy.array(data_real) + \
1j * numpy.array(data_imag)
else:
from taskinit import tb
# read in the CASA MS
tb.open(visfile)
vis_complex = tb.getcol('DATA')
vis_weight = tb.getcol('WEIGHT')
tb.close()
#tb.open(visfile + '/POLARIZATION')
#polinfo = tb.getcol('NUM_CORR')
#npol = polinfo[0]
data_complex = vis_complex
data_wgt = vis_weight
wgtshape = data_wgt.shape
if len(wgtshape) == 2:
npol = wgtshape[0]
nrow = wgtshape[1]
wgtshape = (npol, 1, nrow)
data_wgt = data_wgt.reshape(wgtshape)
#data_complex = []
#data_wgt = []
#for ipol in range(npol):
# data_complex.append(vis_complex[ipol, 0, :])
# data_wgt.append(vis_weight[ipol, :])
#data_complex = numpy.array(data_complex)
#data_wgt = numpy.array(data_wgt)
return data_complex, data_wgt
def calibeovsa(vis=None,
caltype=None,
interp=None,
docalib=True,
doflag=True,
flagant=None,
doimage=False,
imagedir=None,
antenna=None,
timerange=None,
spw=None,
stokes=None,
doconcat=False,
msoutdir=None,
keep_orig_ms=True):
'''
:param vis: EOVSA visibility dataset(s) to be calibrated
:param caltype:
:param interp:
:param docalib:
:param qlookimage:
:param flagant:
:param stokes:
:param doconcat:
:return:
'''
if type(vis) == str:
vis = [vis]
for idx, f in enumerate(vis):
if f[-1] == '/':
vis[idx] = f[:-1]
for msfile in vis:
casalog.origin('calibeovsa')
if not caltype:
casalog.post(
"Caltype not provided. Perform reference phase calibration and daily phase calibration."
)
caltype = [
'refpha', 'phacal', 'fluxcal'
] ## use this line after the phacal is applied # caltype = ['refcal']
if not os.path.exists(msfile):
casalog.post("Input visibility does not exist. Aborting...")
continue
if msfile.endswith('/'):
msfile = msfile[:-1]
if not msfile[-3:] in ['.ms', '.MS']:
casalog.post(
"Invalid visibility. Please provide a proper visibility file ending with .ms"
)
# if not caltable:
# caltable=[os.path.basename(vis).replace('.ms','.'+c) for c in caltype]
# get band information
tb.open(msfile + '/SPECTRAL_WINDOW')
nspw = tb.nrows()
bdname = tb.getcol('NAME')
bd_nchan = tb.getcol('NUM_CHAN')
bd = [int(b[4:]) - 1 for b in bdname] # band index from 0 to 33
# nchans = tb.getcol('NUM_CHAN')
# reffreqs = tb.getcol('REF_FREQUENCY')
# cenfreqs = np.zeros((nspw))
tb.close()
tb.open(msfile + '/ANTENNA')
nant = tb.nrows()
antname = tb.getcol('NAME')
antlist = [str(ll) for ll in range(len(antname) - 1)]
antennas = ','.join(antlist)
tb.close()
# get time stamp, use the beginning of the file
tb.open(msfile + '/OBSERVATION')
trs = {'BegTime': [], 'EndTime': []}
for ll in range(tb.nrows()):
tim0, tim1 = Time(tb.getcell('TIME_RANGE', ll) / 24 / 3600,
format='mjd')
trs['BegTime'].append(tim0)
trs['EndTime'].append(tim1)
tb.close()
trs['BegTime'] = Time(trs['BegTime'])
trs['EndTime'] = Time(trs['EndTime'])
btime = np.min(trs['BegTime'])
etime = np.max(trs['EndTime'])
# ms.open(vis)
# summary = ms.summary()
# ms.close()
# btime = Time(summary['BeginTime'], format='mjd')
# etime = Time(summary['EndTime'], format='mjd')
## stop using ms.summary to avoid conflicts with importeovsa
t_mid = Time((btime.mjd + etime.mjd) / 2., format='mjd')
print "This scan observed from {} to {} UTC".format(
btime.iso, etime.iso)
gaintables = []
if ('refpha' in caltype) or ('refamp' in caltype) or ('refcal'
in caltype):
refcal = ra.sql2refcalX(btime)
pha = refcal['pha'] # shape is 15 (nant) x 2 (npol) x 34 (nband)
pha[np.where(refcal['flag'] == 1)] = 0.
amp = refcal['amp']
amp[np.where(refcal['flag'] == 1)] = 1.
t_ref = refcal['timestamp']
# find the start and end time of the local day when refcal is registered
try:
dhr = t_ref.LocalTime.utcoffset().total_seconds() / 60. / 60.
except:
dhr = -7.
bt = Time(np.fix(t_ref.mjd + dhr / 24.) - dhr / 24., format='mjd')
et = Time(bt.mjd + 1., format='mjd')
(yr, mon, day) = (bt.datetime.year, bt.datetime.month,
bt.datetime.day)
dirname = caltbdir + str(yr) + str(mon).zfill(2) + '/'
if not os.path.exists(dirname):
os.mkdir(dirname)
# check if there is any ROACH reboot between the reference calibration found and the current data
t_rbts = db.get_reboot(Time([t_ref, btime]))
if not t_rbts:
casalog.post(
"Reference calibration is derived from observation at " +
t_ref.iso)
print "Reference calibration is derived from observation at " + t_ref.iso
else:
casalog.post(
"Oh crap! Roach reboot detected between the reference calibration time "
+ t_ref.iso + ' and the current observation at ' +
btime.iso)
casalog.post("Aborting...")
print "Oh crap! Roach reboot detected between the reference calibration time " + t_ref.iso + ' and the current observation at ' + btime.iso
print "Aborting..."
para_pha = []
para_amp = []
calpha = np.zeros((nspw, 15, 2))
calamp = np.zeros((nspw, 15, 2))
for s in range(nspw):
for n in range(15):
for p in range(2):
calpha[s, n, p] = pha[n, p, bd[s]]
calamp[s, n, p] = amp[n, p, bd[s]]
para_pha.append(np.degrees(pha[n, p, bd[s]]))
para_amp.append(amp[n, p, bd[s]])
if 'fluxcal' in caltype:
calfac = pc.get_calfac(Time(t_mid.iso.split(' ')[0] + 'T23:59:59'))
t_bp = Time(calfac['timestamp'], format='lv')
if int(t_mid.mjd) == int(t_bp.mjd):
accalfac = calfac['accalfac'] # (ant x pol x freq)
# tpcalfac = calfac['tpcalfac'] # (ant x pol x freq)
caltb_autoamp = dirname + t_bp.isot[:-4].replace(
':', '').replace('-', '') + '.bandpass'
if not os.path.exists(caltb_autoamp):
bandpass(vis=msfile,
caltable=caltb_autoamp,
solint='inf',
refant='eo01',
minblperant=0,
minsnr=0,
bandtype='B',
docallib=False)
tb.open(caltb_autoamp, nomodify=False) # (ant x spw)
bd_chanidx = np.hstack([[0], bd_nchan.cumsum()])
for ll in range(nspw):
antfac = np.sqrt(
accalfac[:, :, bd_chanidx[ll]:bd_chanidx[ll + 1]])
# # antfac *= tpcalfac[:, :,bd_chanidx[ll]:bd_chanidx[ll + 1]]
antfac = np.moveaxis(antfac, 0, 2)
cparam = np.zeros((2, bd_nchan[ll], nant))
cparam[:, :, :-3] = 1.0 / antfac
tb.putcol('CPARAM', cparam + 0j, ll * nant, nant)
paramerr = tb.getcol('PARAMERR', ll * nant, nant)
paramerr = paramerr * 0
tb.putcol('PARAMERR', paramerr, ll * nant, nant)
bpflag = tb.getcol('FLAG', ll * nant, nant)
bpant1 = tb.getcol('ANTENNA1', ll * nant, nant)
bpflagidx, = np.where(bpant1 >= 13)
bpflag[:] = False
bpflag[:, :, bpflagidx] = True
tb.putcol('FLAG', bpflag, ll * nant, nant)
bpsnr = tb.getcol('SNR', ll * nant, nant)
bpsnr[:] = 100.0
bpsnr[:, :, bpflagidx] = 0.0
tb.putcol('SNR', bpsnr, ll * nant, nant)
tb.close()
msg_prompt = "Scaling calibration is derived for {}.".format(
msfile)
casalog.post(msg_prompt)
print msg_prompt
gaintables.append(caltb_autoamp)
else:
msg_prompt = "Caution: No TPCAL is available on {}. No scaling calibration is derived for {}.".format(
t_mid.datetime.strftime('%b %d, %Y'), msfile)
casalog.post(msg_prompt)
print msg_prompt
if ('refpha' in caltype) or ('refcal' in caltype):
# caltb_pha = os.path.basename(vis).replace('.ms', '.refpha')
# check if the calibration table already exists
caltb_pha = dirname + t_ref.isot[:-4].replace(':', '').replace(
'-', '') + '.refpha'
if not os.path.exists(caltb_pha):
gencal(vis=msfile,
caltable=caltb_pha,
caltype='ph',
antenna=antennas,
pol='X,Y',
spw='0~' + str(nspw - 1),
parameter=para_pha)
gaintables.append(caltb_pha)
if ('refamp' in caltype) or ('refcal' in caltype):
# caltb_amp = os.path.basename(vis).replace('.ms', '.refamp')
caltb_amp = dirname + t_ref.isot[:-4].replace(':', '').replace(
'-', '') + '.refamp'
if not os.path.exists(caltb_amp):
gencal(vis=msfile,
caltable=caltb_amp,
caltype='amp',
antenna=antennas,
pol='X,Y',
spw='0~' + str(nspw - 1),
parameter=para_amp)
gaintables.append(caltb_amp)
# calibration for the change of delay center between refcal time and beginning of scan -- hopefully none!
xml, buf = ch.read_calX(4, t=[t_ref, btime], verbose=False)
if buf:
dly_t2 = Time(stf.extract(buf[0], xml['Timestamp']), format='lv')
dlycen_ns2 = stf.extract(buf[0], xml['Delaycen_ns'])[:15]
xml, buf = ch.read_calX(4, t=t_ref)
dly_t1 = Time(stf.extract(buf, xml['Timestamp']), format='lv')
dlycen_ns1 = stf.extract(buf, xml['Delaycen_ns'])[:15]
dlycen_ns_diff = dlycen_ns2 - dlycen_ns1
for n in range(2):
dlycen_ns_diff[:, n] -= dlycen_ns_diff[0, n]
print 'Multi-band delay is derived from delay center difference at {} & {}'.format(
dly_t1.iso, dly_t2.iso)
# print '=====Delays relative to Ant 14====='
# for i, dl in enumerate(dlacen_ns_diff[:, 0] - dlacen_ns_diff[13, 0]):
# ant = antlist[i]
# print 'Ant eo{0:02d}: x {1:.2f} ns & y {2:.2f} ns'.format(int(ant) + 1, dl
# dlacen_ns_diff[i, 1] - dlacen_ns_diff[13, 1])
# caltb_mbd0 = os.path.basename(vis).replace('.ms', '.mbd0')
caltb_dlycen = dirname + dly_t2.isot[:-4].replace(':', '').replace(
'-', '') + '.dlycen'
if not os.path.exists(caltb_dlycen):
gencal(vis=msfile,
caltable=caltb_dlycen,
caltype='mbd',
pol='X,Y',
antenna=antennas,
parameter=dlycen_ns_diff.flatten().tolist())
gaintables.append(caltb_dlycen)
if 'phacal' in caltype:
phacals = np.array(
ra.sql2phacalX([bt, et], neat=True, verbose=False))
if not phacals.any() or len(phacals) == 0:
print "Found no phacal records in SQL database, will skip phase calibration"
else:
# first generate all phacal calibration tables if not already exist
t_phas = Time([phacal['t_pha'] for phacal in phacals])
# sort the array in ascending order by t_pha
sinds = t_phas.mjd.argsort()
t_phas = t_phas[sinds]
phacals = phacals[sinds]
caltbs_phambd = []
for i, phacal in enumerate(phacals):
# filter out phase cals with reference time stamp >30 min away from the provided refcal time
if (phacal['t_ref'].jd -
refcal['timestamp'].jd) > 30. / 1440.:
del phacals[i]
del t_phas[i]
continue
else:
t_pha = phacal['t_pha']
phambd_ns = phacal['pslope']
for n in range(2):
phambd_ns[:, n] -= phambd_ns[0, n]
# set all flagged values to be zero
phambd_ns[np.where(phacal['flag'] == 1)] = 0.
caltb_phambd = dirname + t_pha.isot[:-4].replace(
':', '').replace('-', '') + '.phambd'
caltbs_phambd.append(caltb_phambd)
if not os.path.exists(caltb_phambd):
gencal(vis=msfile,
caltable=caltb_phambd,
caltype='mbd',
pol='X,Y',
antenna=antennas,
parameter=phambd_ns.flatten().tolist())
# now decides which table to apply depending on the interpolation method ("neatest" or "linear")
if interp == 'nearest':
tbind = np.argmin(np.abs(t_phas.mjd - t_mid.mjd))
dt = np.min(np.abs(t_phas.mjd - t_mid.mjd)) * 24.
print "Selected nearest phase calibration table at " + t_phas[
tbind].iso
gaintables.append(caltbs_phambd[tbind])
if interp == 'linear':
# bphacal = ra.sql2phacalX(btime)
# ephacal = ra.sql2phacalX(etime,reverse=True)
bt_ind, = np.where(t_phas.mjd < btime.mjd)
et_ind, = np.where(t_phas.mjd > etime.mjd)
if len(bt_ind) == 0 and len(et_ind) == 0:
print "No phacal found before or after the ms data within the day of observation"
print "Skipping daily phase calibration"
elif len(bt_ind) > 0 and len(et_ind) == 0:
gaintables.append(caltbs_phambd[bt_ind[-1]])
elif len(bt_ind) == 0 and len(et_ind) > 0:
gaintables.append(caltbs_phambd[et_ind[0]])
elif len(bt_ind) > 0 and len(et_ind) > 0:
bphacal = phacals[bt_ind[-1]]
ephacal = phacals[et_ind[0]]
# generate a new table interpolating between two daily phase calibrations
t_pha_mean = Time(np.mean(
[bphacal['t_pha'].mjd, ephacal['t_pha'].mjd]),
format='mjd')
phambd_ns = (bphacal['pslope'] +
ephacal['pslope']) / 2.
for n in range(2):
phambd_ns[:, n] -= phambd_ns[0, n]
# set all flagged values to be zero
phambd_ns[np.where(bphacal['flag'] == 1)] = 0.
phambd_ns[np.where(ephacal['flag'] == 1)] = 0.
caltb_phambd_interp = dirname + t_pha_mean.isot[:-4].replace(
':', '').replace('-', '') + '.phambd'
if not os.path.exists(caltb_phambd_interp):
gencal(vis=msfile,
caltable=caltb_phambd_interp,
caltype='mbd',
pol='X,Y',
antenna=antennas,
parameter=phambd_ns.flatten().tolist())
print "Using phase calibration table interpolated between records at " + bphacal[
't_pha'].iso + ' and ' + ephacal['t_pha'].iso
gaintables.append(caltb_phambd_interp)
if docalib:
clearcal(msfile)
applycal(vis=msfile,
gaintable=gaintables,
applymode='calflag',
calwt=False)
# delete the interpolated phase calibration table
try:
caltb_phambd_interp
except:
pass
else:
if os.path.exists(caltb_phambd_interp):
shutil.rmtree(caltb_phambd_interp)
if doflag:
# flag zeros and NaNs
flagdata(vis=msfile, mode='clip', clipzeros=True)
if flagant:
try:
flagdata(vis=msfile, antenna=flagant)
except:
print "Something wrong with flagant. Abort..."
if doimage:
from matplotlib import pyplot as plt
from suncasa.utils import helioimage2fits as hf
from sunpy import map as smap
if not antenna:
antenna = '0~12'
if not stokes:
stokes = 'XX'
if not timerange:
timerange = ''
if not spw:
spw = '1~3'
if not imagedir:
imagedir = '.'
#(yr, mon, day) = (bt.datetime.year, bt.datetime.month, bt.datetime.day)
#dirname = imagedir + str(yr) + '/' + str(mon).zfill(2) + '/' + str(day).zfill(2) + '/'
#if not os.path.exists(dirname):
# os.makedirs(dirname)
bds = [spw]
nbd = len(bds)
imgs = []
for bd in bds:
if '~' in bd:
bdstr = bd.replace('~', '-')
else:
bdstr = str(bd).zfill(2)
imname = imagedir + '/' + os.path.basename(msfile).replace(
'.ms', '.bd' + bdstr)
print 'Cleaning image: ' + imname
try:
clean(vis=msfile,
imagename=imname,
antenna=antenna,
spw=bd,
timerange=timerange,
imsize=[512],
cell=['5.0arcsec'],
stokes=stokes,
niter=500)
except:
print 'clean not successfull for band ' + str(bd)
else:
imgs.append(imname + '.image')
junks = ['.flux', '.mask', '.model', '.psf', '.residual']
for junk in junks:
if os.path.exists(imname + junk):
shutil.rmtree(imname + junk)
tranges = [btime.iso + '~' + etime.iso] * nbd
fitsfiles = [img.replace('.image', '.fits') for img in imgs]
hf.imreg(vis=msfile,
timerange=tranges,
imagefile=imgs,
fitsfile=fitsfiles,
usephacenter=False)
plt.figure(figsize=(6, 6))
for i, fitsfile in enumerate(fitsfiles):
plt.subplot(1, nbd, i + 1)
eomap = smap.Map(fitsfile)
sz = eomap.data.shape
if len(sz) == 4:
eomap.data = eomap.data.reshape((sz[2], sz[3]))
eomap.plot_settings['cmap'] = plt.get_cmap('jet')
eomap.plot()
eomap.draw_limb()
eomap.draw_grid()
plt.show()
if doconcat:
if len(vis) > 1:
# from suncasa.eovsa import concateovsa as ce
from suncasa.tasks import concateovsa_cli as ce
if msoutdir is None:
msoutdir = './'
concatvis = os.path.basename(vis[0])
concatvis = msoutdir + '/' + concatvis.split('.')[0] + '_concat.ms'
ce.concateovsa(vis,
concatvis,
datacolumn='corrected',
keep_orig_ms=keep_orig_ms,
cols2rm="model,corrected")
return [concatvis]
else:
return vis
def concateovsa(vis, concatvis, datacolumn='corrected', keep_orig_ms=True, cols2rm="model,corrected", freqtol="", dirtol="", respectname=False,
timesort=True, copypointing=True, visweightscale=[], forcesingleephemfield=""):
if concatvis[-1] == os.path.sep:
concatvis = concatvis[:-1]
if os.path.sep not in concatvis:
visprefix = './'
else:
visprefix = os.path.dirname(concatvis) + os.path.sep
msfiles = vis
msfiles_ = []
for idx, ll in enumerate(msfiles):
if str(ll).endswith('/'):
msfiles[idx] = str(ll)[:-1]
datacolumn = datacolumn.lower()
if datacolumn == 'data':
print 'DATA columns will be concatenated.'
for ll in msfiles:
clearcal(vis=str(ll), addmodel=True)
elif datacolumn == 'corrected':
# try:
print 'CORRECTED columns will be concatenated.'
tmpdir = os.path.join(visprefix, 'tmp_ms') + os.path.sep
if not os.path.exists(tmpdir):
os.makedirs(tmpdir)
for ll in msfiles:
msfile_ = os.path.join(tmpdir, os.path.basename(str(ll)))
msfiles_.append(msfile_)
split(vis=str(ll), outputvis=msfile_, datacolumn='corrected')
clearcal(vis=msfile_, addmodel=True)
else:
raise ValueError('Please set datacolumn to be "data" or "corrected"!')
if msfiles_:
concat(vis=msfiles_, concatvis=concatvis, freqtol=freqtol, dirtol=dirtol, respectname=respectname, timesort=timesort,
copypointing=copypointing, visweightscale=visweightscale, forcesingleephemfield=forcesingleephemfield)
os.system('rm -rf {}'.format(tmpdir))
else:
concat(vis=msfiles, concatvis=concatvis, freqtol=freqtol, dirtol=dirtol, respectname=respectname, timesort=timesort,
copypointing=copypointing, visweightscale=visweightscale, forcesingleephemfield=forcesingleephemfield)
# Change all observation ids to be the same (zero)
tb.open(concatvis + '/OBSERVATION', nomodify=False)
nobs = tb.nrows()
tim0 = tb.getcell('TIME_RANGE', 0)[0]
tim1 = tb.getcell('TIME_RANGE', nobs - 1)[1]
tb.removerows([i + 1 for i in range(nobs - 1)])
tb.putcell('TIME_RANGE', 0, [tim0, tim1])
tb.close()
tb.open(concatvis + '/DATA_DESCRIPTION', nomodify=False)
nrows = tb.nrows()
pol_id = tb.getcol('POLARIZATION_ID')
tb.removerows(np.where(pol_id != 0)[0])
tb.close()
tb.open(concatvis, nomodify=False)
dd_id = tb.getcol('DATA_DESC_ID')
idx_dd_id, = np.where(dd_id >= nrows / 2)
dd_id[idx_dd_id] = dd_id[idx_dd_id] - nrows / 2
tb.putcol('DATA_DESC_ID', dd_id)
tb.close()
tb.open(concatvis + '/FIELD', nomodify=False)
nobs = tb.nrows()
tb.removerows([i + 1 for i in range(nobs - 1)])
tb.close()
tb.open(concatvis + '/SOURCE', nomodify=False)
nobs = tb.nrows()
tb.removerows([i + 1 for i in range(nobs - 1)])
tb.close()
tb.open(concatvis, nomodify=False)
obsid = tb.getcol('OBSERVATION_ID')
newobsid = np.zeros(len(obsid), dtype='int')
tb.putcol('OBSERVATION_ID', newobsid)
fldid = tb.getcol('FIELD_ID')
newfldid = np.zeros(len(fldid), dtype='int')
tb.putcol('FIELD_ID', newfldid)
colnames = tb.colnames()
cols2rm = cols2rm.upper()
cols2rm = cols2rm.split(',')
for l in range(len(cols2rm)):
col = cols2rm[l] + '_DATA'
if col in colnames:
try:
tb.removecols(col)
print 'Column {} removed.'.format(col)
except:
pass
tb.close()
if msfiles_ != [] and msfiles_ != msfiles:
for ll in msfiles_:
os.system('rm -rf {}'.format(ll))
if not keep_orig_ms:
for ll in msfiles:
os.system('rm -rf {}'.format(ll))
def pimporteovsa(idbfiles,
ncpu=8,
timebin=None,
width=None,
visprefix=None,
nocreatms=False,
doconcat=False,
modelms=''):
casalog.origin('pimporteovsa')
# # Initialize the helper class
# pdh = ParallelDataHelper("importeovsa", locals())
#
# # Validate input and output parameters
# try:
# pdh.setupIO()
# except Exception, instance:
# casalog.post('%s' % instance, 'ERROR')
# return False
# import glob
# idbfiles = glob.glob('IDB*_unrot')[:4]
# ncpu = 4
# timebin = None
# width = None
# visprefix = None
# nocreatms = False
# doconcat = False
# modelms = ''
# if not (type(ncpu) is int):
# casalog.post('ncpu should be an integer')
# ncpu = 8
if type(idbfiles) == Time:
filelist = ri.get_trange_files(idbfiles)
else:
# If input type is not Time, assume that it is the list of files to read
filelist = idbfiles
if type(filelist) == str:
filelist = [filelist]
for f in filelist:
if not os.path.exists(f):
casalog.post("Some files in filelist are invalid. Aborting...")
# return False
if not visprefix:
visprefix = './'
if not timebin:
timebin = '0s'
if not width:
width = 1
if not modelms:
if nocreatms:
filename = filelist[0]
modelms = ipe.creatms(filename, visprefix)
else:
if not os.path.exists(modelms):
if nocreatms:
filename = filelist[0]
modelms = ipe.creatms(filename, visprefix)
iterable = range(len(filelist))
imppart = partial(importeovsa_iter, filelist, timebin, width, visprefix,
nocreatms, modelms)
t0 = time.time()
casalog.post('Perform importeovsa in parallel ...')
pool = mp.Pool(ncpu)
res = pool.map(imppart, iterable)
pool.close()
pool.join()
t1 = time.time()
timelapse = t1 - t0
print 'It took %f secs to complete' % timelapse
results = pd.DataFrame({'succeeded': [], 'msfile': [], 'durtim': []})
for r in res:
results = results.append(
pd.DataFrame({
'succeeded': [r[0]],
'msfile': [r[1]],
'durtim': [r[2]]
}))
if doconcat:
msname = list(os.path.basename(filelist[0]))
concatvis = visprefix + ''.join(msname) + '-{:d}m.ms'.format(
int(results['durtim'].sum()))
msfile = results.loc[results['succeeded'] == True, :].msfile.tolist()
concat(vis=msfile, concatvis=concatvis, timesort=True)
# Change all observation ids to be the same (zero)
tb.open(concatvis + '/OBSERVATION', nomodify=False)
nobs = tb.nrows()
tb.removerows([i + 1 for i in range(nobs - 1)])
tb.close()
tb.open(concatvis, nomodify=False)
obsid = tb.getcol('OBSERVATION_ID')
newobsid = np.zeros(len(obsid), dtype='int')
tb.putcol('OBSERVATION_ID', newobsid)
tb.close()
for ll in msfile:
os.system('rm -rf {}'.format(ll))
return True
def coords2cfg(lons,
lats,
antnames=None,
antidxs=None,
cfgfile='ksc-7m.cfg',
arrayname='KSC',
dishdiam=7,
write2casa=True,
verbose=True):
util = su()
xs = []
ys = []
zs = []
for (lon, lat) in zip(lons, lats):
x, y, z = util.long2xyz(np.radians(lon),
np.radians(lat),
3.,
datum='WGS84')
xs.append(x)
ys.append(y)
zs.append(z)
if antidxs:
xs = [xs[i] for i in antidxs]
ys = [ys[i] for i in antidxs]
zs = [zs[i] for i in antidxs]
xs = np.array(xs)
ys = np.array(ys)
zs = np.array(zs)
f = open(cfgfile, 'w')
f.write('#observatory=KSC\n')
# f.write('#COFA={0:.5f}, {1:.5f}\n'.format(lats[idx0], lons[idx0]))
f.write('#coordsys=XYZ\n')
f.write('# x y z diam pad\n')
for i in range(len(xs)):
f.write('{0} {1} {2} {3} K{4:02d}\n'.format(xs[i], ys[i], zs[i],
dishdiam, i))
f.close()
if write2casa:
repodir = os.getenv("CASAPATH").split(' ')[0] + "/data/alma/simmos/"
os.system('cp ' + cfgfile + ' ' + repodir)
casalog.post(
'Antenna configuration file {0:s} written to CASA'.format(cfgfile))
if verbose:
print('Antenna configuration file {0:s} written to CASA'.format(
cfgfile))
obstable = os.getenv("CASAPATH").split(
' ')[0] + "/data/geodetic/Observatories"
tb.open(obstable, nomodify=True)
if arrayname not in tb.getcol('Name'):
print('{0:s} not in Observatories. Adding a record to {1:s}'.format(
arrayname, obstable))
obsdict_dsc = {
'MJD': 59147.0,
'Name': 'KSC',
'Type': 'WGS84',
'Long': -80.693,
'Lat': 28.508,
'Height': 3.00,
'X': 0.0,
'Y': 0.0,
'Z': 0.0,
'Source': 'EOVSA Team'
}
tb.open(obstable, nomodify=False)
nrows = tb.nrows()
tb.addrows(1)
for i in obsdict_dsc.keys():
tb.putcell(i, nrows, obsdict_dsc[i])
tb.close()
def visload(visfile):
checker = visfile.find('uvfits')
if checker == -1:
uvfits = False
else:
uvfits = True
if uvfits:
visdata = fits.open(visfile)
# get the telescope name
visheader = visdata[0].header
#telescop = visheader['TELESCOP']
# if we are dealing with SMA data
if visheader['NAXIS'] == 6:
nfreq = visdata[0].data['DATA'][0, 0, 0, :, 0, 0].size
if nfreq > 1:
data_real = visdata[0].data['DATA'][:, 0, 0, :, :, 0]
data_imag = visdata[0].data['DATA'][:, 0, 0, :, :, 1]
data_wgt = visdata[0].data['DATA'][:, 0, 0, :, :, 2]
else:
data_real = visdata[0].data['DATA'][:, 0, 0, 0, :, 0]
data_imag = visdata[0].data['DATA'][:, 0, 0, 0, :, 1]
data_wgt = visdata[0].data['DATA'][:, 0, 0, 0, :, 2]
# if we are dealing with ALMA or PdBI data
if visheader['NAXIS'] == 7:
nfreq = visdata[0].data['DATA'][0, 0, 0, 0, :, 0, 0].size
if True or nfreq > 1: #RP mod inserted True
data_real = visdata[0].data['DATA'][:, 0, 0, :, :, :, 0]
data_imag = visdata[0].data['DATA'][:, 0, 0, :, :, :, 1]
data_wgt = visdata[0].data['DATA'][:, 0, 0, :, :, :, 2]
else:
data_real = visdata[0].data['DATA'][:, 0, 0, :, 0, :, 0]
data_imag = visdata[0].data['DATA'][:, 0, 0, :, 0, :, 1]
data_wgt = visdata[0].data['DATA'][:, 0, 0, :, 0, :, 2]
data_complex = numpy.array(data_real) + \
1j * numpy.array(data_imag)
else:
from taskinit import tb
# read in the CASA MS
tb.open(visfile)
vis_complex = tb.getcol('DATA')
vis_weight = tb.getcol('WEIGHT')
tb.close()
#tb.open(visfile + '/POLARIZATION')
#polinfo = tb.getcol('NUM_CORR')
#npol = polinfo[0]
data_complex = vis_complex
data_wgt = vis_weight
wgtshape = data_wgt.shape
if len(wgtshape) == 2:
npol = wgtshape[0]
nrow = wgtshape[1]
wgtshape = (npol, 1, nrow)
data_wgt = data_wgt.reshape(wgtshape)
#data_complex = []
#data_wgt = []
#for ipol in range(npol):
# data_complex.append(vis_complex[ipol, 0, :])
# data_wgt.append(vis_weight[ipol, :])
#data_complex = numpy.array(data_complex)
#data_wgt = numpy.array(data_wgt)
return data_complex, data_wgt
def svplot(vis, timerange=None, spw='', workdir='./', specfile=None, bl=None, uvrange=None,
stokes='RR,LL', dmin=None, dmax=None,
goestime=None, reftime=None,
xycen=None, fov=[500.,500.], xyrange=None, restoringbeam=[''], robust=0.0,
niter=500, imsize=[512], cell=['5.0arcsec'],interactive=False,
usemsphacenter=True, imagefile=None, fitsfile=None, plotaia=True,
aiawave=171, aiafits=None, savefig=False, mkmovie=False, overwrite=True, ncpu=10, twidth=1, verbose=True):
'''
Required inputs:
vis: calibrated CASA measurement set
Important optional inputs:
timerange: timerange for clean. Standard CASA time selection format.
If not provided, use the entire range (*BE CAREFUL, COULD BE VERY SLOW*)
spw: spectral window selection following the CASA syntax.
Examples: spw='1:2~60' (spw id 1, channel range 2-60); spw='*:1.2~1.3GHz' (selects all channels within 1.2-1.3 GHz; note the *)
specfile: supply dynamic spectrum save file (from suncasa.utils.dspec2.get_dspec()). Otherwise
generate a median dynamic spectrum on the fly
Optional inputs:
bl: baseline to generate dynamic spectrum
uvrange: uvrange to select baselines for generating dynamic spectrum
stokes: polarization of the clean image, can be 'RR,LL' or 'I,V'
dmin,dmax: color bar parameter
goestime: goes plot time, example ['2016/02/18 18:00:00','2016/02/18 23:00:00']
rhessisav: rhessi savefile
reftime: reftime for the image
xycen: center of the image in helioprojective coordinates (HPLN/HPLT), in arcseconds. Example: [900, -150.]
fov: field of view in arcsecs. Example: [500., 500.]
xyrange: field of view in solar XY coordinates. Format: [[x1,x2],[y1,y2]]. Example: [[900., 1200.],[0,300]]
***NOTE: THIS PARAMETER OVERWRITES XYCEN AND FOV***
aiawave: wave length of aia file in a
imagefile: if imagefile provided, use it. Otherwise do clean and generate a new one.
fitsfile: if fitsfile provided, use it. Otherwise generate a new one
savefig: whether to save the figure
Example:
'''
if xycen:
xc, yc = xycen
xlen, ylen = fov
if parse_version(sunpy.__version__)>parse_version('0.8.0'):
xyrange = [[xc - xlen / 2.0, yc - ylen / 2.0], [xc + xlen / 2.0, yc + ylen / 2.0]]
else:
xyrange = [[xc - xlen / 2.0, xc + xlen / 2.0], [yc - ylen / 2.0, yc + ylen / 2.0]]
stokes_allowed = ['RR,LL', 'I,V', 'RRLL', 'IV']
if not stokes in stokes_allowed:
print 'wrong stokes parameter ' + str(stokes) + '. Allowed values are ' + ', '.join(stokes_allowed)
return -1
if stokes == 'RRLL':
stokes = 'RR,LL'
if stokes == 'IV':
stokes = 'I,V'
if vis[-1] == '/':
vis = vis[:-1]
if not os.path.exists(vis):
print 'input measurement not exist'
return -1
if aiafits is None:
aiafits = ''
# split the data
# generating dynamic spectrum
if not os.path.exists(workdir):
os.makedirs(workdir)
if specfile:
try:
specdata = np.load(specfile)
except:
print('Provided dynamic spectrum file not numpy npz. Generating one from the visibility data')
specfile = os.path.join(workdir, os.path.basename(vis) + '.dspec.npz')
dspec_external(vis, workdir=workdir, specfile=specfile)
specdata = np.load(specfile) # specdata = ds.get_dspec(vis, domedian=True, verbose=True)
else:
print('Dynamic spectrum file not provided; Generating one from the visibility data')
# specdata = ds.get_dspec(vis, domedian=True, verbose=True)
specfile = os.path.join(workdir, os.path.basename(vis) + '.dspec.npz')
dspec_external(vis, workdir=workdir, specfile=specfile)
specdata = np.load(specfile)
tb.open(vis)
starttim = Time(tb.getcell('TIME', 0) / 24. / 3600., format='mjd')
endtim = Time(tb.getcell('TIME', tb.nrows() - 1) / 24. / 3600., format='mjd')
tb.close()
datstr = starttim.iso[:10]
if timerange is None or timerange == '':
starttim1 = starttim
endtim1 = endtim
timerange = '{0}~{1}'.format(starttim.iso.replace('-', '/').replace(' ', '/'), endtim.iso.replace('-', '/').replace(' ', '/'))
else:
try:
(tstart, tend) = timerange.split('~')
if tstart[2] == ':':
starttim1 = Time(datstr + 'T' + tstart)
endtim1 = Time(datstr + 'T' + tend)
timerange = '{0}/{1}~{0}/{2}'.format(datstr.replace('-', '/'), tstart, tend)
else:
starttim1 = Time(qa.quantity(tstart, 'd')['value'], format='mjd')
endtim1 = Time(qa.quantity(tend, 'd')['value'], format='mjd')
except ValueError:
print "keyword 'timerange' in wrong format"
midtime_mjd = (starttim1.mjd + endtim1.mjd) / 2.
if vis.endswith('/'):
vis = vis[:-1]
visname = os.path.basename(vis)
bt = starttim1.plot_date
et = endtim1.plot_date
# find out min and max frequency for plotting in dynamic spectrum
ms.open(vis)
metadata = ms.metadata()
observatory = metadata.observatorynames()[0]
spwInfo = ms.getspectralwindowinfo()
nspw = len(spwInfo)
if not spw:
spw = '0~' + str(nspw - 1)
staql = {'timerange': timerange, 'spw': spw}
if ms.msselect(staql, onlyparse=True):
ndx = ms.msselectedindices()
chan_sel = ndx['channel']
nspw = chan_sel.shape[0]
bspw = chan_sel[0, 0]
bchan = chan_sel[0, 1]
espw = chan_sel[-1, 0]
echan = chan_sel[-1, 2]
bfreq = spwInfo[str(bspw)]['Chan1Freq'] + spwInfo[str(bspw)]['ChanWidth'] * bchan
efreq = spwInfo[str(espw)]['Chan1Freq'] + spwInfo[str(espw)]['ChanWidth'] * echan
bfreqghz = bfreq / 1e9
efreqghz = efreq / 1e9
if verbose:
print 'selected timerange {}'.format(timerange)
print 'selected frequency range {0:6.3f} to {1:6.3f} GHz'.format(bfreqghz, efreqghz)
else:
print "spw or timerange selection failed. Aborting..."
ms.close()
return -1
ms.close()
if observatory == 'EOVSA':
print 'Provide stokes: ' + str(stokes) + '. However EOVSA has linear feeds. Force stokes to be IV'
stokes = 'I,V'
if mkmovie:
plt.ioff()
# fig = plt.figure(figsize=(12, 7.5), dpi=100)
if fitsfile:
pass
else:
if not imagefile:
# from ptclean_cli import ptclean_cli as ptclean
eph = hf.read_horizons(t0=Time(midtime_mjd, format='mjd'))
if observatory == 'EOVSA' or (not usemsphacenter):
phasecenter = ''
else:
phasecenter = 'J2000 ' + str(eph['ra'][0])[:15] + 'rad ' + str(eph['dec'][0])[:15] + 'rad'
print 'use phasecenter: ' + phasecenter
qlookfitsdir = os.path.join(workdir, 'qlookfits/')
qlookfigdir = os.path.join(workdir, 'qlookimgs/')
imresfile = os.path.join(qlookfitsdir, '{}.imres.npz'.format(os.path.basename(vis)))
if overwrite:
imres = mk_qlook_image(vis, twidth=twidth, ncpu=ncpu, imagedir=qlookfitsdir, phasecenter=phasecenter, stokes=stokes,
c_external=True)
else:
if os.path.exists(imresfile):
imres = np.load(imresfile)
imres = imres['imres'].item()
else:
print('Image results file not found; Creating new images.')
imres = mk_qlook_image(vis, twidth=twidth, ncpu=ncpu, imagedir=qlookfitsdir, phasecenter=phasecenter, stokes=stokes,
c_external=True)
if not os.path.exists(qlookfigdir):
os.makedirs(qlookfigdir)
plt_qlook_image(imres, figdir=qlookfigdir, specdata=specdata, verbose=True, stokes=stokes, fov=xyrange)
else:
spec = specdata['spec']
(npol, nbl, nfreq, ntim) = spec.shape
tidx = range(ntim)
fidx = range(nfreq)
tim = specdata['tim']
freq = specdata['freq']
freqghz = freq / 1e9
spec_tim = Time(specdata['tim'] / 3600. / 24., format='mjd')
timstrr = spec_tim.plot_date
plt.ion()
fig = plt.figure(figsize=(12, 7), dpi=100)
gs1 = gridspec.GridSpec(3, 1)
gs1.update(left=0.08, right=0.32, wspace=0.05)
gs2 = gridspec.GridSpec(2, 2)
gs2.update(left=0.38, right=0.98, hspace=0.02, wspace=0.02)
spec_1 = np.absolute(spec[0, 0, :, :])
spec_2 = np.absolute(spec[1, 0, :, :])
if observatory == 'EVLA':
# circular feeds
polstr = ['RR', 'LL']
if observatory == 'EOVSA' or observatory == 'ALMA':
# linear feeds
polstr = ['XX', 'YY']
print 'plot the dynamic spectrum in pol ' + ' & '.join(polstr)
ax1 = plt.subplot(gs1[0])
ax1.pcolormesh(timstrr, freqghz, spec_1, cmap='jet', vmin=dmin, vmax=dmax)
ax1.set_xlim(timstrr[tidx[0]], timstrr[tidx[-1]])
ax1.xaxis_date()
ax1.xaxis.set_major_formatter(DateFormatter("%H:%M:%S"))
# ax1.set_xticklabels(['']*10)
ax1.set_ylim(freqghz[fidx[0]], freqghz[fidx[-1]])
ax1.set_ylabel('Frequency (GHz)', fontsize=10)
ax1.set_title(observatory + ' ' + datstr + ' ' + polstr[0] + ' & ' + polstr[1], fontsize=12)
ax1.set_autoscale_on(False)
ax1.add_patch(patches.Rectangle((bt, bfreqghz), et - bt, efreqghz - bfreqghz, ec='w', fill=False))
ax1.plot([(bt + et) / 2.], [(bfreqghz + efreqghz) / 2.], '*w', ms=12)
for tick in ax1.get_xticklabels():
tick.set_fontsize(8)
for tick in ax1.get_yticklabels():
tick.set_fontsize(8)
ax2 = plt.subplot(gs1[1])
ax2.pcolormesh(timstrr, freqghz, spec_2, cmap='jet', vmin=dmin, vmax=dmax)
ax2.set_xlim(timstrr[tidx[0]], timstrr[tidx[-1]])
ax2.xaxis_date()
ax2.xaxis.set_major_formatter(DateFormatter("%H:%M:%S"))
ax2.set_ylim(freqghz[fidx[0]], freqghz[fidx[-1]])
ax2.set_ylabel('Frequency (GHz)', fontsize=10)
for tick in ax2.get_xticklabels():
tick.set_fontsize(8)
for tick in ax2.get_yticklabels():
tick.set_fontsize(8)
ax2.set_autoscale_on(False)
ax2.add_patch(patches.Rectangle((bt, bfreqghz), et - bt, efreqghz - bfreqghz, ec='w', fill=False))
ax2.plot([(bt + et) / 2.], [(bfreqghz + efreqghz) / 2.], '*w', ms=12)
# Second part: GOES plot
if goestime:
btgoes = goestime[0]
etgoes = goestime[1]
else:
datstrg = datstr.replace('-', '/')
btgoes = datstrg + ' ' + qa.time(qa.quantity(tim[0] - 1800, 's'), form='clean', prec=9)[0]
etgoes = datstrg + ' ' + qa.time(qa.quantity(tim[tidx[-1] - 1] + 1800, 's'), form='clean', prec=9)[0]
if verbose:
print 'Acquire GOES soft X-ray data in from ' + btgoes + ' to ' + etgoes
ax3 = plt.subplot(gs1[2])
try:
from sunpy import lightcurve as lc
from sunpy.time import TimeRange
goest = lc.GOESLightCurve.create(TimeRange(btgoes, etgoes))
except:
goesscript = os.path.join(workdir, 'goes.py')
goesdatafile = os.path.join(workdir, 'goes.dat')
os.system('rm -rf {}'.format(goesscript))
fi = open(goesscript, 'wb')
fi.write('import os \n')
fi.write('from sunpy.time import TimeRange \n')
fi.write('from sunpy import lightcurve as lc \n')
fi.write('import pickle \n')
fi.write('goesplottim = TimeRange("{0}", "{1}") \n'.format(btgoes, etgoes))
fi.write('goes = lc.GOESLightCurve.create(goesplottim) \n')
fi.write('fi2 = open("{}", "wb") \n'.format(goesdatafile))
fi.write('pickle.dump(goes, fi2) \n')
fi.write('fi2.close()')
fi.close()
try:
os.system('python {}'.format(goesscript))
os.system('rm -rf {}'.format(goesscript))
except NameError:
print "Bad input names"
except ValueError:
print "Bad input values"
except:
print "Unexpected error:", sys.exc_info()[0]
print "Error in generating GOES light curves. Proceed without GOES..."
if os.path.exists(goesdatafile):
fi1 = file(goesdatafile, 'rb')
goest = pickle.load(fi1)
fi1.close()
try:
dates = mpl.dates.date2num(parse_time(goest.data.index))
goesdif = np.diff(goest.data['xrsb'])
gmax = np.nanmax(goesdif)
gmin = np.nanmin(goesdif)
ran = gmax - gmin
db = 2.8 / ran
goesdifp = goesdif * db + gmin + (-6)
ax3.plot_date(dates, np.log10(goest.data['xrsb']), '-', label='1.0--8.0 $\AA$', color='red', lw=2)
ax3.plot_date(dates[0:-1], goesdifp, '-', label='derivate', color='blue', lw=0.4)
ax3.set_ylim([-7, -3])
ax3.set_yticks([-7, -6, -5, -4, -3])
ax3.set_yticklabels([r'$10^{-7}$', r'$10^{-6}$', r'$10^{-5}$', r'$10^{-4}$', r'$10^{-3}$'])
ax3.set_title('Goes Soft X-ray', fontsize=12)
ax3.set_ylabel('Watts m$^{-2}$')
ax3.set_xlabel(datetime.datetime.isoformat(goest.data.index[0])[0:10])
ax3.axvspan(dates[899], dates[dates.size - 899], alpha=0.2)
ax2 = ax3.twinx()
# ax2.set_yscale("log")
ax2.set_ylim([-7, -3])
ax2.set_yticks([-7, -6, -5, -4, -3])
ax2.set_yticklabels(['B', 'C', 'M', 'X', ''])
ax3.yaxis.grid(True, 'major')
ax3.xaxis.grid(False, 'major')
ax3.legend(prop={'size': 6})
formatter = mpl.dates.DateFormatter('%H:%M')
ax3.xaxis.set_major_formatter(formatter)
ax3.fmt_xdata = mpl.dates.DateFormatter('%H:%M')
except:
print 'Error in downloading GOES soft X-ray data. Proceeding with out soft X-ray plot.'
# third part
# start to download the fits files
if plotaia:
if not aiafits:
newlist = []
items = glob.glob('*.fits')
for names in items:
str1 = starttim1.iso[:4] + '_' + starttim1.iso[5:7] + '_' + starttim1.iso[8:10] + 't' + starttim1.iso[
11:13] + '_' + starttim1.iso[14:16]
str2 = str(aiawave)
if names.endswith(".fits"):
if names.find(str1) != -1 and names.find(str2) != -1:
newlist.append(names)
newlist.append('0')
if newlist and os.path.exists(newlist[0]):
aiafits = newlist[0]
else:
print 'downloading the aiafits file'
wave1 = aiawave - 3
wave2 = aiawave + 3
t1 = Time(starttim1.mjd - 0.02 / 24., format='mjd')
t2 = Time(endtim1.mjd + 0.02 / 24., format='mjd')
try:
from sunpy.net import vso
client = vso.VSOClient()
qr = client.query(vso.attrs.Time(t1.iso, t2.iso), vso.attrs.Instrument('aia'), vso.attrs.Wave(wave1 * u.AA, wave2 * u.AA))
res = client.get(qr, path='{file}')
except:
SdoDownloadscript = os.path.join(workdir, 'SdoDownload.py')
os.system('rm -rf {}'.format(SdoDownloadscript))
fi = open(SdoDownloadscript, 'wb')
fi.write('from sunpy.net import vso \n')
fi.write('from astropy import units as u \n')
fi.write('client = vso.VSOClient() \n')
fi.write(
"qr = client.query(vso.attrs.Time('{0}', '{1}'), vso.attrs.Instrument('aia'), vso.attrs.Wave({2} * u.AA, {3} * u.AA)) \n".format(
t1.iso, t2.iso, wave1, wave2))
fi.write("res = client.get(qr, path='{file}') \n")
fi.close()
try:
os.system('python {}'.format(SdoDownloadscript))
except NameError:
print "Bad input names"
except ValueError:
print "Bad input values"
except:
print "Unexpected error:", sys.exc_info()[0]
print "Error in Downloading AIA fits files. Proceed without AIA..."
# Here something is needed to check whether it has finished downloading the fits files or not
if not aiafits:
newlist = []
items = glob.glob('*.fits')
for nm in items:
str1 = starttim1.iso[:4] + '_' + starttim1.iso[5:7] + '_' + starttim1.iso[8:10] + 't' + starttim1.iso[
11:13] + '_' + starttim1.iso[14:16]
str2 = str(aiawave)
if nm.find(str1) != -1 and nm.find(str2) != -1:
newlist.append(nm)
if newlist:
aiafits = newlist[0]
print 'AIA fits ' + aiafits + ' selected'
else:
print 'no AIA fits files found. Proceed without AIA'
try:
aiamap = smap.Map(aiafits)
except:
print 'error in reading aiafits. Proceed without AIA'
# RCP or I
ax4 = plt.subplot(gs2[0, 0])
ax5 = plt.subplot(gs2[1, 0])
# LCP or V
ax6 = plt.subplot(gs2[0, 1])
ax7 = plt.subplot(gs2[1, 1])
if fitsfile:
pass
else:
if not imagefile:
eph = hf.read_horizons(t0=Time(midtime_mjd, format='mjd'))
if observatory == 'EOVSA' or (not usemsphacenter):
print 'This is EOVSA data'
# use RA and DEC from FIELD ID 0
tb.open(vis+'/FIELD')
phadir = tb.getcol('PHASE_DIR').flatten()
tb.close()
ra0 = phadir[0]
dec0 = phadir[1]
if stokes == 'RRLL' or stokes == 'RR,LL':
print 'Provide stokes: ' + str(stokes) + '. However EOVSA has linear feeds. Force stokes to be IV'
stokes = 'I,V'
else:
ra0 = eph['ra'][0]
dec0 = eph['dec'][0]
if not xycen:
# use solar disk center as default
phasecenter = 'J2000 ' + str(ra0) + 'rad ' + str(dec0) + 'rad'
else:
x0 = np.radians(xycen[0]/3600.)
y0 = np.radians(xycen[1]/3600.)
p0 = np.radians(eph['p0'][0]) # p angle in radians
raoff = -((x0) * np.cos(p0) - y0 * np.sin(p0))/np.cos(eph['dec'][0])
decoff = (x0) * np.sin(p0) + y0 * np.cos(p0)
newra = ra0 + raoff
newdec = dec0 + decoff
phasecenter = 'J2000 ' + str(newra) + 'rad ' + str(newdec) + 'rad'
imagename = os.path.join(workdir, visname + '.outim')
if os.path.exists(imagename + '.image') or os.path.exists(imagename + '.flux'):
os.system('rm -rf ' + imagename + '.*')
sto = stokes.replace(',', '')
print 'do clean for ' + timerange + ' in spw ' + spw + ' stokes ' + sto
print 'Original phasecenter: '+ str(ra0) + str(dec0)
print 'use phasecenter: ' + phasecenter
clean(vis=vis, imagename=imagename, selectdata=True, spw=spw, timerange=timerange, stokes=sto,
niter=niter, interactive=interactive, npercycle=50, imsize=imsize, cell=cell, restoringbeam=restoringbeam,
weighting='briggs', robust=robust, phasecenter=phasecenter)
os.system('rm -rf ' + imagename + '.psf')
os.system('rm -rf ' + imagename + '.flux')
os.system('rm -rf ' + imagename + '.model')
os.system('rm -rf ' + imagename + '.mask')
os.system('rm -rf ' + imagename + '.residual')
imagefile = imagename + '.image'
fitsfile = imagefile + '.fits'
hf.imreg(vis=vis, ephem=eph, imagefile=imagefile, timerange=timerange, reftime=reftime, fitsfile=fitsfile, verbose=True, overwrite=True)
print 'fits file ' + fitsfile + ' selected'
ax4.cla()
ax5.cla()
ax6.cla()
ax7.cla()
rfits = fitsfile
try:
hdulist = fits.open(rfits)
hdu = hdulist[0]
(npol, nf, nx, ny) = hdu.data.shape
rmap = smap.Map(hdu.data[0, 0, :, :], hdu.header)
except:
print 'radio fits file not recognized by sunpy.map. Aborting...'
return -1
if npol > 1:
rmap1 = smap.Map(hdu.data[0, 0, :, :], hdu.header)
rmap2 = smap.Map(hdu.data[1, 0, :, :], hdu.header)
XX, YY = np.meshgrid(np.arange(rmap.data.shape[1]), np.arange(rmap.data.shape[0]))
try:
rmapx, rmapy = rmap.pixel_to_data(XX * u.pix, YY * u.pix)
except:
rmapxy = rmap.pixel_to_data(XX * u.pix, YY * u.pix)
rmapx = rmapxy.Tx
rmapy = rmapxy.Ty
if not xyrange:
if xycen:
x0 = xycen[0] * u.arcsec
y0 = xycen[1] * u.arcsec
if not xycen:
row, col = rmap1.data.shape
positon = np.nanargmax(rmap1.data)
m, n = divmod(positon, col)
x0 = rmap1.xrange[0] + rmap1.scale[1] * (n + 0.5) * u.pix
y0 = rmap1.yrange[0] + rmap1.scale[0] * (m + 0.5) * u.pix
if len(fov) == 1:
fov=[fov]*2
sz_x = fov[0] * u.arcsec
sz_y = fov[1] * u.arcsec
x1 = x0 - sz_x/2.
x2 = x0 + sz_x/2.
y1 = y0 - sz_y/2.
y2 = y0 + sz_y/2.
xyrange = [[x1.value, x2.value], [y1.value, y2.value]]
else:
sz_x = xyrange[0][1] - xyrange[0][0]
sz_y = xyrange[1][1] - xyrange[1][0]
clevels1 = np.linspace(0.2, 0.9, 5)
if stokes.split(',')[1] == 'V':
clevels2 = np.array([0.8, -0.6, -0.4, -0.2, 0.2, 0.4, 0.6, 0.8])
else:
clevels2 = np.linspace(0.2, 0.9, 5)
if 'aiamap' in vars():
aiamap.plot_settings['cmap'] = plt.get_cmap('binary')
if rmap:
title = 'AIA {0:.0f} + {1} {2:6.3f} GHz'.format(aiamap.wavelength.value, observatory, (bfreqghz + efreqghz) / 2.0)
else:
title = 'AIA {0:.0f}'.format(aiamap.wavelength.value)
aiamap.plot(axes=ax4)
ax4.set_title(title + ' ' + stokes.split(',')[0], fontsize=12)
aiamap.draw_limb()
aiamap.draw_grid()
aiamap.draw_rectangle((xyrange[0][0], xyrange[1][0]) * u.arcsec, sz_x, sz_y)
aiamap.plot(axes=ax6)
ax6.set_title(title + ' ' + stokes.split(',')[1], fontsize=12)
aiamap.draw_limb()
aiamap.draw_grid()
aiamap.draw_rectangle((xyrange[0][0], xyrange[1][0]) * u.arcsec, sz_x, sz_y)
if rmap:
ax4.contour(rmapx.value, rmapy.value, rmap1.data, levels=clevels1 * np.nanmax(rmap1.data), cmap=cm.jet)
ax6.contour(rmapx.value, rmapy.value, rmap2.data, levels=clevels2 * np.nanmax(rmap2.data), cmap=cm.RdBu)
ax4.text(0.02, 0.02, 'AIA {0:.0f} '.format(aiamap.wavelength.value) + aiamap.date.strftime('%H:%M:%S'),
verticalalignment='bottom', horizontalalignment='left', transform=ax4.transAxes, color='k',
fontsize=10)
ax6.text(0.02, 0.02, 'AIA {0:.0f} '.format(aiamap.wavelength.value) + aiamap.date.strftime('%H:%M:%S'),
verticalalignment='bottom', horizontalalignment='left', transform=ax6.transAxes, color='k',
fontsize=10)
else:
title = '{0} {1:6.3f} GHz'.format(observatory, (bfreqghz + efreqghz) / 2.0)
rmap1.plot(axes=ax4, cmap=cm.jet)
ax4.set_title(title + ' ' + stokes.split(',')[0], fontsize=12)
rmap1.draw_limb()
rmap1.draw_grid()
rmap1.draw_rectangle((xyrange[0][0], xyrange[1][0]) * u.arcsec, sz_x, sz_y)
rmap2.plot(axes=ax6, cmap=cm.RdBu)
ax6.set_title(title + ' ' + stokes.split(',')[1], fontsize=12)
rmap2.draw_limb()
rmap2.draw_grid()
# ax4.contour(rmapx.value, rmapy.value, rmap1.data, levels=np.linspace(0.2, 0.9, 5) * np.nanmax(rmap1.data),
# cmap=cm.gray)
# ax6.contour(rmapx.value, rmapy.value, rmap2.data, levels=np.linspace(0.2, 0.9, 5) * np.nanmax(rmap2.data),
# cmap=cm.gray)
rmap2.draw_rectangle((xyrange[0][0], xyrange[1][0]) * u.arcsec, sz_x, sz_y)
ax4.set_xlim(-1200, 1200)
ax4.set_ylim(-1200, 1200)
ax6.set_xlim(-1200, 1200)
ax6.set_ylim(-1200, 1200)
try:
subrmap1 = rmap1.submap(xyrange[0] * u.arcsec, xyrange[1] * u.arcsec)
subrmap2 = rmap2.submap(xyrange[0] * u.arcsec, xyrange[1] * u.arcsec)
except:
bl = SkyCoord(xyrange[0][0] * u.arcsec, xyrange[1][0] * u.arcsec, frame=rmap1.coordinate_frame)
tr = SkyCoord(xyrange[0][1] * u.arcsec, xyrange[1][1] * u.arcsec, frame=rmap1.coordinate_frame)
subrmap1 = rmap1.submap(bl, tr)
subrmap2 = rmap2.submap(bl, tr)
XX, YY = np.meshgrid(np.arange(subrmap1.data.shape[1]), np.arange(subrmap1.data.shape[0]))
try:
subrmapx, subrmapy = subrmap1.pixel_to_data(XX * u.pix, YY * u.pix)
except:
subrmapxy = subrmap1.pixel_to_data(XX * u.pix, YY * u.pix)
subrmapx = subrmapxy.Tx
subrmapy = subrmapxy.Ty
if 'aiamap' in vars():
try:
subaiamap = aiamap.submap(xyrange[0] * u.arcsec, xyrange[1] * u.arcsec)
except:
bl = SkyCoord(xyrange[0][0] * u.arcsec, xyrange[1][0] * u.arcsec, frame=aiamap.coordinate_frame)
tr = SkyCoord(xyrange[0][1] * u.arcsec, xyrange[1][1] * u.arcsec, frame=aiamap.coordinate_frame)
subaiamap = aiamap.submap(bl, tr)
subaiamap.plot(axes=ax5, title='')
subaiamap.draw_limb()
subaiamap.draw_grid()
subaiamap.plot(axes=ax7, title='')
subaiamap.draw_limb()
subaiamap.draw_grid()
ax5.contour(subrmapx.value, subrmapy.value, subrmap1.data, levels=clevels1 * np.nanmax(subrmap1.data), cmap=cm.jet)
ax7.contour(subrmapx.value, subrmapy.value, subrmap2.data, levels=clevels2 * np.nanmax(subrmap2.data),
cmap=cm.RdBu) # subaiamap.draw_rectangle((fov[0][0], fov[1][0]) * u.arcsec, 400 * u.arcsec, 400 * u.arcsec)
else:
subrmap1.plot(axes=ax5, cmap=cm.jet, title='')
subrmap1.draw_limb()
subrmap1.draw_grid()
subrmap2.plot(axes=ax7, cmap=cm.RdBu, title='')
subrmap2.draw_limb()
subrmap2.draw_grid() # ax5.contour(subrmapx.value, subrmapy.value, subrmap1.data, # levels=clevels1 * np.nanmax(subrmap1.data), cmap=cm.gray) # ax7.contour(subrmapx.value, subrmapy.value, subrmap2.data, # levels=clevels2 * np.nanmax(subrmap2.data), cmap=cm.gray) # subrmap1.draw_rectangle((fov[0][0], fov[1][0]) * u.arcsec, 400 * u.arcsec, 400 * u.arcsec) # subrmap2.draw_rectangle((fov[0][0], fov[1][0]) * u.arcsec, 400 * u.arcsec, 400 * u.arcsec)
ax5.set_xlim(xyrange[0])
ax5.set_ylim(xyrange[1])
ax5.text(0.02, 0.02, observatory + ' ' + rmap.date.strftime('%H:%M:%S.%f')[:-3], verticalalignment='bottom',
horizontalalignment='left', transform=ax5.transAxes, color='k', fontsize=10)
ax7.set_xlim(xyrange[0])
ax7.set_ylim(xyrange[1])
ax7.text(0.02, 0.02, observatory + ' ' + rmap.date.strftime('%H:%M:%S.%f')[:-3], verticalalignment='bottom',
horizontalalignment='left', transform=ax7.transAxes, color='k', fontsize=10)
fig.show()
def importeovsa(idbfiles,
timebin=None,
width=None,
visprefix=None,
nocreatms=False,
doconcat=False,
modelms=''):
casalog.origin('importeovsa')
# # Initialize the helper class
# pdh = ParallelDataHelper("importeovsa", locals())
#
# # Validate input and output parameters
# try:
# pdh.setupIO()
# except Exception, instance:
# casalog.post('%s' % instance, 'ERROR')
# return False
if type(idbfiles) == Time:
filelist = ri.get_trange_files(idbfiles)
else:
# If input type is not Time, assume that it is the list of files to read
filelist = idbfiles
if type(filelist) == str:
filelist = [filelist]
for f in filelist:
if not os.path.exists(f):
casalog.post("Some files in filelist are invalid. Aborting...")
return False
if not visprefix:
visprefix = './'
if not timebin:
timebin = '0s'
if not width:
width = 1
if not modelms:
if nocreatms:
filename = filelist[0]
modelms = ipe.creatms(filename, visprefix)
else:
if not os.path.exists(modelms):
if nocreatms:
filename = filelist[0]
modelms = ipe.creatms(filename, visprefix)
msfile = []
time_concat = []
for filename in filelist:
uv = aipy.miriad.UV(filename)
# if filename.split('/')[-1][0:3] == 'UDB':
# uv_str = uv_hex_rm(uv)
# else:
# uv_str = uv
uv.select('antennae', 0, 1, include=True)
uv.select('polarization', -5, -5, include=True)
times = []
uv.rewind()
for preamble, data in uv.all():
uvw, t, (i, j) = preamble
times.append(t)
uv.select('clear', -1, -1, include=True)
times = ipe.jd2mjds(np.asarray(times))
inttime = np.median((times - np.roll(times, 1))[1:]) / 60
time_steps = len(times)
time_concat.append(int((times[-1] - times[0]) / 60 + inttime))
time0 = time.time()
if 'antlist' in uv.vartable:
ants = uv['antlist'].replace('\x00', '')
antlist = map(int, ants.split())
else:
antlist = [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16]
good_idx = np.where(uv['sfreq'] > 0)[0]
nf = len(good_idx)
npol = uv['npol']
nants = uv['nants']
source_id = uv['source'].replace('\x00', '')
sfreq = uv['sfreq'][good_idx]
sdf = uv['sdf'][good_idx]
ra, dec = uv['ra'], uv['dec']
nbl = nants * (nants - 1) / 2
bl2ord = ipe.bl_list2(nants)
npairs = nbl + nants
flag = np.ones((npol, nf, time_steps, npairs), dtype=bool)
out = np.zeros((npol, nf, time_steps, npairs),
dtype=np.complex64) # Cross-correlations
uvwarray = np.zeros((3, time_steps, npairs), dtype=np.float)
chan_band = ipe.get_band(sfreq=sfreq, sdf=sdf)
nband = len(chan_band)
uv.rewind()
l = -1
for preamble, data in uv.all():
uvw, t, (i0, j0) = preamble
i = antlist.index(i0 + 1)
j = antlist.index(j0 + 1)
if i > j:
# Reverse order of indices
j = antlist.index(i0 + 1)
i = antlist.index(j0 + 1)
# Assumes uv['pol'] is one of -5, -6, -7, -8
k = -5 - uv['pol']
l += 1
out[k, :, l / (npairs * npol), bl2ord[i0, j0]] = data.data
flag[k, :, l / (npairs * npol), bl2ord[i0, j0]] = data.mask
# if i != j:
if k == 3:
uvwarray[:, l / (npairs * npol),
bl2ord[i0,
j0]] = -uvw * constants.speed_of_light / 1e9
nrows = time_steps * npairs
out = out.reshape(npol, nf, nrows)
flag = flag.reshape(npol, nf, nrows)
uvwarray = uvwarray.reshape(3, nrows)
uvwarray = np.tile(uvwarray, (1, nband))
sigma = np.ones((4, nrows), dtype=np.float) + 1
sigma = np.tile(sigma, (1, nband))
casalog.post(
'IDB File {0} is readed in --- {1:10.2f} seconds ---'.format(
filename, (time.time() - time0)))
msname = list(filename.split('/')[-1])
msname = visprefix + ''.join(msname) + '.ms'
if not nocreatms:
modelms = ipe.creatms(filename, visprefix)
os.system('mv {} {}'.format(modelms, msname))
else:
casalog.post('----------------------------------------')
casalog.post('copying standard MS to {0}'.format(
msname, (time.time() - time0)))
casalog.post('----------------------------------------')
os.system("rm -fr %s" % msname)
os.system("cp -r " + " %s" % modelms + " %s" % msname)
casalog.post(
'Standard MS is copied to {0} in --- {1:10.2f} seconds ---'.
format(msname, (time.time() - time0)))
tb.open(msname, nomodify=False)
casalog.post('----------------------------------------')
casalog.post("Updating the main table of" '%s' % msname)
casalog.post('----------------------------------------')
for l, cband in enumerate(chan_band):
time1 = time.time()
nchannels = len(cband['cidx'])
for row in range(nrows):
tb.putcell('DATA', (row + l * nrows),
out[:, cband['cidx'][0]:cband['cidx'][-1] + 1, row])
tb.putcell(
'FLAG', (row + l * nrows),
flag[:, cband['cidx'][0]:cband['cidx'][-1] + 1, row])
casalog.post(
'---spw {0:02d} is updated in --- {1:10.2f} seconds ---'.
format((l + 1),
time.time() - time1))
tb.putcol('UVW', uvwarray)
tb.putcol('SIGMA', sigma)
tb.putcol('WEIGHT', 1.0 / sigma**2)
timearr = times
timearr = timearr.reshape(1, time_steps, 1)
timearr = np.tile(timearr, (nband, 1, npairs))
timearr = timearr.reshape(nband * npairs * time_steps)
tb.putcol('TIME', timearr)
tb.putcol('TIME_CENTROID', timearr)
scan_id = tb.getcol('SCAN_NUMBER')
scan_id *= 0
tb.putcol('SCAN_NUMBER', scan_id)
colnames = tb.colnames()
cols2rm = ["MODEL_DATA", "CORRECTED_DATA"]
for l in range(len(cols2rm)):
if cols2rm[l] in colnames:
tb.removecols(cols2rm[l])
tb.close()
casalog.post('----------------------------------------')
casalog.post("Updating the OBSERVATION table of" '%s' % msname)
casalog.post('----------------------------------------')
tb.open(msname + '/OBSERVATION', nomodify=False)
tb.putcol(
'TIME_RANGE',
np.asarray([times[0] - 0.5 * inttime,
times[-1] + 0.5 * inttime]).reshape(2, 1))
tb.putcol('OBSERVER', ['EOVSA team'])
tb.close()
casalog.post('----------------------------------------')
casalog.post("Updating the POINTING table of" '%s' % msname)
casalog.post('----------------------------------------')
tb.open(msname + '/POINTING', nomodify=False)
timearr = times.reshape(1, time_steps, 1)
timearr = np.tile(timearr, (nband, 1, nants))
timearr = timearr.reshape(nband * time_steps * nants)
tb.putcol('TIME', timearr)
tb.putcol('TIME_ORIGIN', timearr) # - 0.5 * delta_time)
direction = tb.getcol('DIRECTION')
direction[0, 0, :] = ra
direction[1, 0, :] = dec
tb.putcol('DIRECTION', direction)
target = tb.getcol('TARGET')
target[0, 0, :] = ra
target[1, 0, :] = dec
tb.putcol('TARGET', target)
tb.close()
casalog.post('----------------------------------------')
casalog.post("Updating the SOURCE table of" '%s' % msname)
casalog.post('----------------------------------------')
tb.open(msname + '/SOURCE', nomodify=False)
radec = tb.getcol('DIRECTION')
radec[0], radec[1] = ra, dec
tb.putcol('DIRECTION', radec)
name = np.array([source_id], dtype='|S{0}'.format(len(source_id) + 1))
tb.putcol('NAME', name)
tb.close()
casalog.post('----------------------------------------')
casalog.post("Updating the DATA_DESCRIPTION table of" '%s' % msname)
casalog.post('----------------------------------------')
tb.open(msname + '/DATA_DESCRIPTION/', nomodify=False)
pol_id = tb.getcol('POLARIZATION_ID')
pol_id *= 0
tb.putcol('POLARIZATION_ID', pol_id)
# spw_id = tb.getcol('SPECTRAL_WINDOW_ID')
# spw_id *= 0
# tb.putcol('SPECTRAL_WINDOW_ID', spw_id)
tb.close()
# pdb.set_trace()
casalog.post('----------------------------------------')
casalog.post("Updating the POLARIZATION table of" '%s' % msname)
casalog.post('----------------------------------------')
tb.open(msname + '/POLARIZATION/', nomodify=False)
print tb.nrows()
tb.removerows(rownrs=np.arange(1, nband, dtype=int))
tb.close()
casalog.post('----------------------------------------')
casalog.post("Updating the FIELD table of" '%s' % msname)
casalog.post('----------------------------------------')
tb.open(msname + '/FIELD/', nomodify=False)
delay_dir = tb.getcol('DELAY_DIR')
delay_dir[0], delay_dir[1] = ra, dec
tb.putcol('DELAY_DIR', delay_dir)
phase_dir = tb.getcol('PHASE_DIR')
phase_dir[0], phase_dir[1] = ra, dec
tb.putcol('PHASE_DIR', phase_dir)
reference_dir = tb.getcol('REFERENCE_DIR')
reference_dir[0], reference_dir[1] = ra, dec
tb.putcol('REFERENCE_DIR', reference_dir)
name = np.array([source_id], dtype='|S{0}'.format(len(source_id) + 1))
tb.putcol('NAME', name)
tb.close()
# FIELD: DELAY_DIR, PHASE_DIR, REFERENCE_DIR, NAME
del out, flag, uvwarray, uv, timearr, sigma
gc.collect() #
if not (timebin == '0s' and width == 1):
split(vis=msname,
outputvis=msname + '.split',
datacolumn='data',
timebin=timebin,
width=width,
keepflags=False)
os.system('rm -rf {}'.format(msname))
msfile.append(msname + '.split')
else:
msfile.append(msname)
casalog.post("finished in --- %s seconds ---" % (time.time() - time0))
if doconcat:
msname = list(filelist[0].split('/')[-1])
concatvis = visprefix + ''.join(msname) + '-{:d}m.ms'.format(
int(sum(time_concat)))
concat(vis=msfile, concatvis=concatvis, timesort=True)
# Change all observation ids to be the same (zero)
tb.open(concatvis + '/OBSERVATION', nomodify=False)
nobs = tb.nrows()
tb.removerows([i + 1 for i in range(nobs - 1)])
tb.close()
tb.open(concatvis, nomodify=False)
obsid = tb.getcol('OBSERVATION_ID')
newobsid = np.zeros(len(obsid), dtype='int')
tb.putcol('OBSERVATION_ID', newobsid)
tb.close()
for ll in msfile:
os.system('rm -rf {}'.format(ll))
return True
def compTables(referencetab,
testtab,
excludecols,
tolerance=0.001,
mode="percentage",
startrow=0,
nrow=-1,
rowincr=1):
"""
compTables - compare two CASA tables
referencetab - the table which is assumed to be correct
testtab - the table which is to be compared to referencetab
excludecols - list of column names which are to be ignored
tolerance - permitted fractional difference (default 0.001 = 0.1 percent)
mode - comparison is made as "percentage", "absolute", "phaseabsdeg" (for complex numbers = difference of the phases in degrees)
"""
rval = True
tb2 = casac.table()
tb.open(referencetab)
cnames = tb.colnames()
tb2.open(testtab)
try:
for c in cnames:
if c in excludecols:
continue
print "\nTesting column " + c
a = 0
try:
a = tb.getcol(c, startrow=startrow, nrow=nrow, rowincr=rowincr)
except:
rval = False
print 'Error accessing column ', c, ' in table ', referencetab
print sys.exc_info()[0]
break
b = 0
try:
b = tb2.getcol(c,
startrow=startrow,
nrow=nrow,
rowincr=rowincr)
except:
rval = False
print 'Error accessing column ', c, ' in table ', testtab
print sys.exc_info()[0]
break
if not (len(a) == len(b)):
print 'Column ', c, ' has different length in tables ', referencetab, ' and ', testtab
print a
print b
rval = False
break
else:
differs = False
if not (a == b).all():
for i in range(0, len(a)):
if (isinstance(a[i], float)):
if ((mode == "percentage") and
(abs(a[i] - b[i]) > tolerance * abs(a[i]))
) or ((mode == "absolute") and
(abs(a[i] - b[i]) > tolerance)):
print "Column " + c + " differs"
print "Row=" + str(i)
print "Reference file value: " + str(a[i])
print "Input file value: " + str(b[i])
if (mode == "percentage"):
print "Tolerance is {0}%; observed difference was {1} %".format(
tolerance * 100,
100 * abs(a[i] - b[i]) / abs(a[i]))
else:
print "Absolute tolerance is {0}; observed difference: {1}".format(
tolerance, (abs(a[i] - b[i])))
differs = True
rval = False
break
elif (isinstance(a[i], int)
or isinstance(a[i], np.int32)):
if (abs(a[i] - b[i]) > 0):
print "Column " + c + " differs"
print "Row=" + str(i)
print "Reference file value: " + str(a[i])
print "Input file value: " + str(b[i])
if (mode == "percentage"):
print "tolerance in % should be " + str(
100 * abs(a[i] - b[i]) / abs(a[i]))
else:
print "absolute tolerance should be " + str(
abs(a[i] - b[i]))
differs = True
rval = False
break
elif (isinstance(a[i], str)
or isinstance(a[i], np.bool_)):
if not (a[i] == b[i]):
print "Column " + c + " differs"
print "Row=" + str(i)
print "Reference file value: " + str(a[i])
print "Input file value: " + str(b[i])
if (mode == "percentage"):
print "tolerance in % should be " + str(
100 * abs(a[i] - b[i]) / abs(a[i]))
else:
print "absolute tolerance should be " + str(
abs(a[i] - b[i]))
differs = True
rval = False
break
elif (isinstance(a[i], list)) or (isinstance(
a[i], np.ndarray)):
for j in range(0, len(a[i])):
if differs: break
if ((isinstance(a[i][j], float))
or (isinstance(a[i][j], int))):
if ((mode == "percentage") and
(abs(a[i][j] - b[i][j]) >
tolerance * abs(a[i][j]))) or (
(mode == "absolute") and
(abs(a[i][j] - b[i][j]) >
tolerance)):
print "Column " + c + " differs"
print "(Row,Element)=(" + str(
j) + "," + str(i) + ")"
print "Reference file value: " + str(
a[i][j])
print "Input file value: " + str(
b[i][j])
if (mode == "percentage"):
print "Tolerance in % should be " + str(
100 * abs(a[i][j] - b[i][j]) /
abs(a[i][j]))
else:
print "Absolute tolerance should be " + str(
abs(a[i][j] - b[i][j]))
differs = True
rval = False
break
elif (isinstance(a[i][j],
list)) or (isinstance(
a[i][j], np.ndarray)):
it = xrange(0, len(a[i][j]))
if mode == "percentage":
diff = np.abs(
np.subtract(a[i][j], b[i][j])
) > tolerance * np.abs(a[i][j])
it = np.where(diff)[0]
elif (mode == "absolute"):
diff = np.abs(
np.subtract(a[i][j],
b[i][j])) > tolerance
it = np.where(diff)[0]
for k in it:
if differs: break
if ( ((mode=="percentage") and (abs(a[i][j][k]-b[i][j][k]) > tolerance*abs(a[i][j][k]))) \
or ((mode=="absolute") and (abs(a[i][j][k]-b[i][j][k]) > tolerance)) \
or ((mode=="phaseabsdeg") and (phasediffabsdeg(a[i][j][k],b[i][j][k])>tolerance)) \
):
print "Column " + c + " differs"
print "(Row,Channel,Corr)=(" + str(
k) + "," + str(j) + "," + str(
i) + ")"
print "Reference file value: " + str(
a[i][j][k])
print "Input file value: " + str(
b[i][j][k])
if (mode == "percentage"):
print "Tolerance in % should be " + str(
100 * abs(a[i][j][k] -
b[i][j][k]) /
abs(a[i][j][k]))
elif (mode == "absolute"):
print "Absolute tolerance should be " + str(
abs(a[i][j][k] -
b[i][j][k]))
elif (mode == "phaseabsdeg"):
print "Phase tolerance in degrees should be " + str(
phasediffabsdeg(
a[i][j][k],
b[i][j][k]))
else:
print "Unknown comparison mode: ", mode
differs = True
rval = False
break
else:
print "Unknown data type: ", type(a[i])
differs = True
rval = False
break
if not differs: print "Column " + c + " PASSED"
finally:
tb.close()
tb2.close()
return rval
def uvload(visfile):
checker = visfile.find('uvfits')
if checker == -1:
uvfits = False
else:
uvfits = True
if uvfits:
visdata = fits.open(visfile)
visibilities = visdata[0].data
visheader = visdata[0].header
if visheader['NAXIS'] == 7:
# identify the channel frequency(ies):
visfreq = visdata[1].data
freq0 = visheader['CRVAL4']
dfreq = visheader['CDELT4']
cfreq = visheader['CRPIX4']
nvis = visibilities['DATA'][:, 0, 0, 0, 0, 0, 0].size
nspw = visibilities['DATA'][0, 0, 0, :, 0, 0, 0].size
nfreq = visibilities['DATA'][0, 0, 0, 0, :, 0, 0].size
npol = visibilities['DATA'][0, 0, 0, 0, 0, :, 0].size
if nfreq > 1:
uu = numpy.zeros([nvis, nspw, nfreq, npol])
vv = numpy.zeros([nvis, nspw, nfreq, npol])
ww = numpy.zeros([nvis, nspw, nfreq, npol])
else:
uu = numpy.zeros([nvis, nspw, npol])
vv = numpy.zeros([nvis, nspw, npol])
ww = numpy.zeros([nvis, nspw, npol])
#wgt = numpy.zeros([nvis, nspw, nfreq, npol])
for ispw in range(nspw):
if nspw > 1:
freqif = freq0 + visfreq['IF FREQ'][0][ispw]
else:
try:
freqif = freq0 + visfreq['IF FREQ'][0]
except:
freqif = freq0
#uu[:, ispw] = freqif * visibilities['UU']
#vv[:, ispw] = freqif * visibilities['VV']
for ipol in range(npol):
# then compute the spatial frequencies:
if nfreq > 1:
freq = (numpy.arange(nfreq) - cfreq + 1) * dfreq + freqif
freqvis = numpy.meshgrid(freq, visibilities['UU'])
uu[:, ispw, :, ipol] = freqvis[0] * freqvis[1]
freqvis = numpy.meshgrid(freq, visibilities['VV'])
vv[:, ispw, :, ipol] = freqvis[0] * freqvis[1]
freqvis = numpy.meshgrid(freq, visibilities['WW'])
ww[:, ispw, :, ipol] = freqvis[0] * freqvis[1]
else:
uu[:, ispw, ipol] = freqif * visibilities['UU']
vv[:, ispw, ipol] = freqif * visibilities['VV']
ww[:, ispw, ipol] = freqif * visibilities['WW']
if visheader['NAXIS'] == 6:
# identify the channel frequency(ies):
freq0 = visheader['CRVAL4']
dfreq = visheader['CDELT4']
cfreq = visheader['CRPIX4']
nvis = visibilities['DATA'][:, 0, 0, 0, 0, 0].size
nfreq = visibilities['DATA'][0, 0, 0, :, 0, 0].size
npol = visibilities['DATA'][0, 0, 0, 0, :, 0].size
if nfreq > 1:
uu = numpy.zeros([nvis, nfreq, npol])
vv = numpy.zeros([nvis, nfreq, npol])
ww = numpy.zeros([nvis, nfreq, npol])
else:
uu = numpy.zeros([nvis, npol])
vv = numpy.zeros([nvis, npol])
ww = numpy.zeros([nvis, npol])
#wgt = numpy.zeros([nvis, nspw, nfreq, npol])
freqif = freq0
#uu[:, ispw] = freqif * visibilities['UU']
#vv[:, ispw] = freqif * visibilities['VV']
for ipol in range(npol):
# then compute the spatial frequencies:
if nfreq > 1:
freq = (numpy.arange(nfreq) - cfreq + 1) * dfreq + freqif
freqvis = numpy.meshgrid(freq, visibilities['UU'])
uu[:, 0, :, ipol] = freqvis[0] * freqvis[1]
freqvis = numpy.meshgrid(freq, visibilities['VV'])
vv[:, 0, :, ipol] = freqvis[0] * freqvis[1]
freqvis = numpy.meshgrid(freq, visibilities['WW'])
ww[:, 0, :, ipol] = freqvis[0] * freqvis[1]
else:
uu[:, ipol] = freqif * visibilities['UU']
vv[:, ipol] = freqif * visibilities['VV']
ww[:, ipol] = freqif * visibilities['WW']
else:
from taskinit import tb
# read in the uvfits data
tb.open(visfile)
uvw = tb.getcol('UVW')
uvspw = tb.getcol('DATA_DESC_ID')
tb.close()
tb.open(visfile + '/SPECTRAL_WINDOW')
freq = tb.getcol('CHAN_FREQ')
tb.close()
tb.open(visfile + '/POLARIZATION')
polinfo = tb.getcol('NUM_CORR')
tb.close()
npol = polinfo[0]
nspw = len(freq[0])
for ispw in range(nspw):
ilam = 3e8 / freq[0][ispw]
indx_spw = uvspw == ispw
uvw[:, indx_spw] /= ilam
uu = []
vv = []
ww = []
for ipol in range(npol):
uu.append(uvw[0, :])
vv.append(uvw[1, :])
ww.append(uvw[2, :])
uu = numpy.array(uu)
vv = numpy.array(vv)
ww = numpy.array(ww)
if uu[:, 0].size == 1:
uu = uu.flatten()
vv = vv.flatten()
ww = ww.flatten()
return uu, vv, ww
def compTables(referencetab, testtab, excludecols, tolerance=0.001, mode="percentage", startrow = 0, nrow = -1, rowincr = 1):
"""
compTables - compare two CASA tables
referencetab - the table which is assumed to be correct
testtab - the table which is to be compared to referencetab
excludecols - list of column names which are to be ignored
tolerance - permitted fractional difference (default 0.001 = 0.1 percent)
mode - comparison is made as "percentage", "absolute", "phaseabsdeg" (for complex numbers = difference of the phases in degrees)
"""
rval = True
tb2 = casac.table()
tb.open(referencetab)
cnames = tb.colnames()
tb2.open(testtab)
try:
for c in cnames:
if c in excludecols:
continue
print "\nTesting column " + c
a = 0
try:
a = tb.getcol(c,startrow=startrow,nrow=nrow,rowincr=rowincr)
except:
rval = False
print 'Error accessing column ', c, ' in table ', referencetab
print sys.exc_info()[0]
break
b = 0
try:
b = tb2.getcol(c,startrow=startrow,nrow=nrow,rowincr=rowincr)
except:
rval = False
print 'Error accessing column ', c, ' in table ', testtab
print sys.exc_info()[0]
break
if not (len(a)==len(b)):
print 'Column ',c,' has different length in tables ', referencetab, ' and ', testtab
print a
print b
rval = False
break
else:
differs = False
if not (a==b).all():
for i in range(0,len(a)):
if (isinstance(a[i],float)):
if ((mode=="percentage") and (abs(a[i]-b[i]) > tolerance*abs(a[i]))) or ((mode=="absolute") and (abs(a[i]-b[i]) > tolerance)):
print "Column " + c + " differs"
print "Row=" + str(i)
print "Reference file value: " + str(a[i])
print "Input file value: " + str(b[i])
if (mode=="percentage"):
print "Tolerance is {0}%; observed difference was {1} %".format (tolerance * 100, 100*abs(a[i]-b[i])/abs(a[i]))
else:
print "Absolute tolerance is {0}; observed difference: {1}".format (tolerance, (abs(a[i]-b[i])))
differs = True
rval = False
break
elif (isinstance(a[i],int) or isinstance(a[i],np.int32)):
if (abs(a[i]-b[i]) > 0):
print "Column " + c + " differs"
print "Row=" + str(i)
print "Reference file value: " + str(a[i])
print "Input file value: " + str(b[i])
if (mode=="percentage"):
print "tolerance in % should be " + str(100*abs(a[i]-b[i])/abs(a[i]))
else:
print "absolute tolerance should be " + str(abs(a[i]-b[i]))
differs = True
rval = False
break
elif (isinstance(a[i],str) or isinstance(a[i],np.bool_)):
if not (a[i]==b[i]):
print "Column " + c + " differs"
print "Row=" + str(i)
print "Reference file value: " + str(a[i])
print "Input file value: " + str(b[i])
if (mode=="percentage"):
print "tolerance in % should be " + str(100*abs(a[i]-b[i])/abs(a[i]))
else:
print "absolute tolerance should be " + str(abs(a[i]-b[i]))
differs = True
rval = False
break
elif (isinstance(a[i],list)) or (isinstance(a[i],np.ndarray)):
for j in range(0,len(a[i])):
if differs: break
if ((isinstance(a[i][j],float)) or (isinstance(a[i][j],int))):
if ((mode=="percentage") and (abs(a[i][j]-b[i][j]) > tolerance*abs(a[i][j]))) or ((mode=="absolute") and (abs(a[i][j]-b[i][j]) > tolerance)):
print "Column " + c + " differs"
print "(Row,Element)=(" + str(j) + "," + str(i) + ")"
print "Reference file value: " + str(a[i][j])
print "Input file value: " + str(b[i][j])
if (mode=="percentage"):
print "Tolerance in % should be " + str(100*abs(a[i][j]-b[i][j])/abs(a[i][j]))
else:
print "Absolute tolerance should be " + str(abs(a[i][j]-b[i][j]))
differs = True
rval = False
break
elif (isinstance(a[i][j],list)) or (isinstance(a[i][j],np.ndarray)):
for k in range(0,len(a[i][j])):
if differs: break
if ( ((mode=="percentage") and (abs(a[i][j][k]-b[i][j][k]) > tolerance*abs(a[i][j][k]))) \
or ((mode=="absolute") and (abs(a[i][j][k]-b[i][j][k]) > tolerance)) \
or ((mode=="phaseabsdeg") and (phasediffabsdeg(a[i][j][k],b[i][j][k])>tolerance)) \
):
print "Column " + c + " differs"
print "(Row,Channel,Corr)=(" + str(k) + "," + str(j) + "," + str(i) + ")"
print "Reference file value: " + str(a[i][j][k])
print "Input file value: " + str(b[i][j][k])
if (mode=="percentage"):
print "Tolerance in % should be " + str(100*abs(a[i][j][k]-b[i][j][k])/abs(a[i][j][k]))
elif (mode=="absolute"):
print "Absolute tolerance should be " + str(abs(a[i][j][k]-b[i][j][k]))
elif (mode=="phaseabsdeg"):
print "Phase tolerance in degrees should be " + str(phasediffabsdeg(a[i][j][k],b[i][j][k]))
else:
print "Unknown comparison mode: ",mode
differs = True
rval = False
break
else:
print "Unknown data type: ",type(a[i])
differs = True
rval = False
break
if not differs: print "Column " + c + " PASSED"
finally:
tb.close()
tb2.close()
return rval
def concat(tb_in=[], tb_out=None):
if not tb_in:
print('tb_in not provided. Abort...')
if os.path.exists(tb_out):
os.system('rm -r ' + tb_out)
# os.system('cp -r '+tb_in[0]+' '+tb_out)
os.system('cp -r ' + tb_in[0] + ' ' + tb_out)
tb.open(tb_out + '/SPECTRAL_WINDOW', nomodify=True)
nspw = tb.nrows()
tb.close()
tim = []
fld = []
spw = []
ant1 = []
ant2 = []
intv = []
scan = []
obid = []
cpar = []
para = []
flag = []
snr = []
# wght=[]
for ctb in tb_in:
tb.open(ctb, nomodify=True)
cols = tb.colnames()
tim0 = tb.getcol(cols[0])
if len(tim0) == 0:
continue
else:
tim.append(tb.getcol(cols[0]))
fld.append(tb.getcol(cols[1]))
spw.append(tb.getcol(cols[2]))
ant1.append(tb.getcol(cols[3]))
ant2.append(tb.getcol(cols[4]))
intv.append(tb.getcol(cols[5]))
scan.append(tb.getcol(cols[6]))
obid.append(tb.getcol(cols[7]))
cpar.append(tb.getcol(cols[8]))
para.append(tb.getcol(cols[9]))
flag.append(tb.getcol(cols[10]))
snr.append(tb.getcol(cols[11]))
# wght.append(tb.getcol(cols[12]))
tb.close()
if len(tim) == 0:
print('tables have no data. Return')
return -1
else:
tim = np.concatenate(tim)
fld = np.concatenate(fld)
spw = np.concatenate(spw)
ant1 = np.concatenate(ant1)
ant2 = np.concatenate(ant2)
intv = np.concatenate(intv)
scan = np.concatenate(scan)
obid = np.concatenate(obid)
cpar = np.concatenate(cpar, axis=2)
para = np.concatenate(para, axis=2)
flag = np.concatenate(flag, axis=2)
snr = np.concatenate(snr, axis=2)
# wght=np.concatenate(wght)
tb.open(tb_out, nomodify=False)
nrows = tb.nrows()
nrows_new = len(tim)
tb.addrows(nrows_new - nrows)
tb.putcol(cols[0], tim)
tb.putcol(cols[1], fld)
tb.putcol(cols[2], spw)
tb.putcol(cols[3], ant1)
tb.putcol(cols[4], ant2)
tb.putcol(cols[5], intv)
tb.putcol(cols[6], scan)
tb.putcol(cols[7], obid)
tb.putcol(cols[8], cpar)
tb.putcol(cols[9], para)
tb.putcol(cols[10], flag)
tb.putcol(cols[11], snr)
tb.close()
return tb_out
def WEI_plot(dofile=True,
vis=None,
timerange=None,
spw='',
aiafits='',
imagehead='',
workdir='',
spwCol=3,
phasecenter='J2000 11h00m48 06d14m60'):
if vis[-1] == '/':
vis = vis[:-1]
ms.open(vis)
spwInfo = ms.getspectralwindowinfo()
ms.close()
tb.open(vis)
starttim = Time(tb.getcell('TIME', 0) / 24. / 3600., format='mjd')
endtim = Time(tb.getcell('TIME',
tb.nrows() - 1) / 24. / 3600.,
format='mjd')
tb.close()
tb.open(vis + '/SPECTRAL_WINDOW')
reffreqs = tb.getcol('REF_FREQUENCY')
bdwds = tb.getcol('TOTAL_BANDWIDTH')
cfreqs = reffreqs + bdwds / 2.
tb.close()
sbeam = 35.
#get timerange from vis file
if not timerange:
timerange = '{0}~{1}'.format(
starttim.iso.replace('-', '/').replace(' ', '/'),
endtim.iso.replace('-', '/').replace(' ', '/'))
nspw = len(spwInfo)
#draw plot aia
fig = plt.figure(figsize=(12, 7), dpi=100)
gs1 = gridspec.GridSpec(4, 3)
gs1.update(left=0.08, right=0.32, wspace=0.05)
ax1 = plt.subplot(gs1[11])
aiamap = smap.Map(aiafits)
aiamap.plot(axes=ax1)
#do clean spwCol by spwCol
for cur_spwCol in range(0, np.floor_divide(nspw, spwCol)):
if ((cur_spwCol + 1) * spwCol) < nspw:
cur_spwRange = str(cur_spwCol * spwCol + 1) + '~' + str(
(cur_spwCol + 1) * spwCol)
else:
cur_spwRange = str(cur_spwCol * spwCol + 1) + '~' + '31'
imagename = imagehead + cur_spwRange + 'SPWs'
cur_eovsaFits = imagename + '.fits'
if cur_spwCol < 6:
cur_mask = '/srg/ywei/data/eovsa/mask/sep_6mask_' + str(
cur_spwCol + 1) + '.rgn'
else:
cur_mask = '/srg/ywei/data/eovsa/mask/sep_6/mask_6.rgn'
if dofile:
#clean(vis=vis, spw=cur_spwRange, timerange=timerange, imagename=imagename, imsize=[256,256], niter=100, cell=['2arcsec'] )
#clean(vis=vis, spw=cur_spwRange, timerange=timerange, imagename=imagename, imsize=[512,512], niter=1000, cell=['1arcsec'],stokes='XX', gain=0.05,weighting='briggs', mode='mfs',imagermode='csclean',psfmode='clark',robust=0.0,restoringbeam = ['10.0arcsec'], mask=cur_mask,pbcor=True)
clean(vis=vis,
spw=cur_spwRange,
timerange=timerange,
imagename=imagename,
imsize=[512, 512],
niter=1000,
cell=['1arcsec'],
stokes='XX',
gain=0.05,
weighting='briggs',
mode='mfs',
imagermode='csclean',
psfmode='clark',
robust=0.0,
restoringbeam=['10.0arcsec'],
mask='',
pbcor=True)
print 'fits name =' + str(cur_eovsaFits)
hf.imreg(vis=vis,
imagefile=imagename + '.image',
fitsfile=imagename + '.fits',
timerange=timerange)
#plot eovsa
cur_emap = smap.Map(cur_eovsaFits)
cur_axname = plt.subplot(gs1[cur_spwCol + 1])
(npol, nf, nx, ny) = cur_emap.data.shape
print 'shape = ' + str(cur_emap.data.shape)
if npol != 1:
print 'To be determined'
else:
cur_emap.data = cur_emap.data.reshape((512, 512))
cur_emap.plot_settings['cmap'] = plt.get_cmap('jet')
cur_emap.plot(axes=cur_axname)
def calc_phasecenter_from_solxy(vis,
timerange='',
xycen=None,
usemsphacenter=True):
'''
return the phase center in RA and DEC of a given solar coordinates
:param vis: input measurement sets file
:param timerange: can be a string or astropy.time.core.Time object, or a 2-element list of string or Time object
:param xycen: solar x-pos and y-pos in arcsec
:param usemsphacenter:
:return:
phasecenter
midtim: mid time of the given timerange
'''
tb.open(vis + '/POINTING')
tst = Time(tb.getcell('TIME_ORIGIN', 0) / 24. / 3600., format='mjd')
ted = Time(tb.getcell('TIME_ORIGIN',
tb.nrows() - 1) / 24. / 3600.,
format='mjd')
tb.close()
datstr = tst.iso[:10]
if isinstance(timerange, Time):
try:
(sttim, edtim) = timerange
except:
sttim = timerange
edtim = sttim
else:
if timerange == '':
sttim = tst
edtim = ted
else:
try:
(tstart, tend) = timerange.split('~')
if tstart[2] == ':':
sttim = Time(datstr + 'T' + tstart)
edtim = Time(datstr + 'T' + tend)
# timerange = '{0}/{1}~{0}/{2}'.format(datstr.replace('-', '/'), tstart, tend)
else:
sttim = Time(qa.quantity(tstart, 'd')['value'],
format='mjd')
edtim = Time(qa.quantity(tend, 'd')['value'], format='mjd')
except:
try:
if timerange[2] == ':':
sttim = Time(datstr + 'T' + timerange)
edtim = sttim
else:
sttim = Time(qa.quantity(timerange, 'd')['value'],
format='mjd')
edtim = sttim
except ValueError:
print("keyword 'timerange' in wrong format")
ms.open(vis)
metadata = ms.metadata()
observatory = metadata.observatorynames()[0]
ms.close()
midtim_mjd = (sttim.mjd + edtim.mjd) / 2.
midtim = Time(midtim_mjd, format='mjd')
eph = read_horizons(t0=midtim)
if observatory == 'EOVSA' or (not usemsphacenter):
print('This is EOVSA data')
# use RA and DEC from FIELD ID 0
tb.open(vis + '/FIELD')
phadir = tb.getcol('PHASE_DIR').flatten()
tb.close()
ra0 = phadir[0]
dec0 = phadir[1]
else:
ra0 = eph['ra'][0]
dec0 = eph['dec'][0]
if not xycen:
# use solar disk center as default
phasecenter = 'J2000 ' + str(ra0) + 'rad ' + str(dec0) + 'rad'
else:
x0 = np.radians(xycen[0] / 3600.)
y0 = np.radians(xycen[1] / 3600.)
p0 = np.radians(eph['p0'][0]) # p angle in radians
raoff = -((x0) * np.cos(p0) - y0 * np.sin(p0)) / np.cos(eph['dec'][0])
decoff = (x0) * np.sin(p0) + y0 * np.cos(p0)
newra = ra0 + raoff
newdec = dec0 + decoff
phasecenter = 'J2000 ' + str(newra) + 'rad ' + str(newdec) + 'rad'
return phasecenter, midtim
def uvload(visfile):
"""load in visibilities from a uv-file
Parameters
----------
visfile: string
visibility data filename, can be model or data
Returns
-------
uu: numpy.array
u visibilities
vv: numpy.array
v visibilities
ww: numpy.array
Note
----
08-14-2015:
Although a better solution to fix the array size mismatch problem that arises when calling `checkvis.uvmcmcfitVis` maybe something similar to the to-be-implemented function: uvmodel.add
which looks for nspw to shape model_complex
"""
checker = visfile.find('uvfits')
if checker == -1:
uvfits = False
else:
uvfits = True
if uvfits:
visdata = fits.open(visfile)
visibilities = visdata[0].data
visheader = visdata[0].header
if visheader['NAXIS'] == 7:
# identify the channel frequency(ies):
visfreq = visdata[1].data
freq0 = visheader['CRVAL4']
dfreq = visheader['CDELT4']
cfreq = visheader['CRPIX4']
nvis = visibilities['DATA'][:, 0, 0, 0, 0, 0, 0].size
nspw = visibilities['DATA'][0, 0, 0, :, 0, 0, 0].size
nfreq = visibilities['DATA'][0, 0, 0, 0, :, 0, 0].size
npol = visibilities['DATA'][0, 0, 0, 0, 0, :, 0].size
if nfreq > 1:
uu = numpy.zeros([nvis, nspw, nfreq, npol])
vv = numpy.zeros([nvis, nspw, nfreq, npol])
ww = numpy.zeros([nvis, nspw, nfreq, npol])
else:
# if miriad is True:
# uu = numpy.zeros([nvis, nspw, nfreq, npol])
# vv = numpy.zeros([nvis, nspw, nfreq, npol])
# ww = numpy.zeros([nvis, nspw, nfreq, npol])
# else:
# uu = numpy.zeros([nvis, nspw, npol])
# vv = numpy.zeros([nvis, nspw, npol])
# ww = numpy.zeros([nvis, nspw, npol])
uu = numpy.zeros([nvis, nspw, npol])
vv = numpy.zeros([nvis, nspw, npol])
ww = numpy.zeros([nvis, nspw, npol])
#wgt = numpy.zeros([nvis, nspw, nfreq, npol])
# get spw frequencies
# reference freq + offset
for ispw in range(nspw):
if nspw > 1:
freqif = freq0 + visfreq['IF FREQ'][0][ispw]
else:
try:
freqif = freq0 + visfreq['IF FREQ'][0]
except:
freqif = freq0
#uu[:, ispw] = freqif * visibilities['UU']
#vv[:, ispw] = freqif * visibilities['VV']
for ipol in range(npol):
# then compute the spatial frequencies:
if nfreq > 1:
freq = (numpy.arange(nfreq) - cfreq + 1) * \
dfreq + freqif
freqvis = numpy.meshgrid(freq, visibilities['UU'])
uu[:, ispw, :, ipol] = freqvis[0] * freqvis[1]
freqvis = numpy.meshgrid(freq, visibilities['VV'])
vv[:, ispw, :, ipol] = freqvis[0] * freqvis[1]
freqvis = numpy.meshgrid(freq, visibilities['WW'])
ww[:, ispw, :, ipol] = freqvis[0] * freqvis[1]
else:
# if miriad is True:
# freq = (numpy.arange(nfreq) - cfreq + 1) * dfreq + freqif
# freqvis = numpy.meshgrid(freq, visibilities['UU'])
# uu[:, ispw, :, ipol] = freqvis[0] * freqvis[1]
# freqvis = numpy.meshgrid(freq, visibilities['VV'])
# vv[:, ispw, :, ipol] = freqvis[0] * freqvis[1]
# freqvis = numpy.meshgrid(freq, visibilities['WW'])
# ww[:, ispw, :, ipol] = freqvis[0] * freqvis[1]
# else:
# uu[:, ispw, ipol] = freqif * visibilities['UU']
# vv[:, ispw, ipol] = freqif * visibilities['VV']
# ww[:, ispw, ipol] = freqif * visibilities['WW']
uu[:, ispw, ipol] = freqif * visibilities['UU']
vv[:, ispw, ipol] = freqif * visibilities['VV']
ww[:, ispw, ipol] = freqif * visibilities['WW']
if visheader['NAXIS'] == 6:
# identify the channel frequency(ies):
freq0 = visheader['CRVAL4']
dfreq = visheader['CDELT4']
cfreq = visheader['CRPIX4']
nvis = visibilities['DATA'][:, 0, 0, 0, 0, 0].size
nfreq = visibilities['DATA'][0, 0, 0, :, 0, 0].size
npol = visibilities['DATA'][0, 0, 0, 0, :, 0].size
if nfreq > 1:
uu = numpy.zeros([nvis, nfreq, npol])
vv = numpy.zeros([nvis, nfreq, npol])
ww = numpy.zeros([nvis, nfreq, npol])
else:
uu = numpy.zeros([nvis, npol])
vv = numpy.zeros([nvis, npol])
ww = numpy.zeros([nvis, npol])
#wgt = numpy.zeros([nvis, nspw, nfreq, npol])
freqif = freq0
#uu[:, ispw] = freqif * visibilities['UU']
#vv[:, ispw] = freqif * visibilities['VV']
for ipol in range(npol):
# then compute the spatial frequencies:
if nfreq > 1:
freq = (numpy.arange(nfreq) - cfreq + 1) * dfreq + freqif
freqvis = numpy.meshgrid(freq, visibilities['UU'])
uu[:, 0, :, ipol] = freqvis[0] * freqvis[1]
freqvis = numpy.meshgrid(freq, visibilities['VV'])
vv[:, 0, :, ipol] = freqvis[0] * freqvis[1]
freqvis = numpy.meshgrid(freq, visibilities['WW'])
ww[:, 0, :, ipol] = freqvis[0] * freqvis[1]
else:
uu[:, ipol] = freqif * visibilities['UU']
vv[:, ipol] = freqif * visibilities['VV']
ww[:, ipol] = freqif * visibilities['WW']
else:
from taskinit import tb
# read in the uvfits data
tb.open(visfile)
uvw = tb.getcol('UVW')
uvspw = tb.getcol('DATA_DESC_ID')
tb.close()
tb.open(visfile + '/SPECTRAL_WINDOW')
freq = tb.getcol('CHAN_FREQ')
tb.close()
tb.open(visfile + '/POLARIZATION')
polinfo = tb.getcol('NUM_CORR')
tb.close()
npol = polinfo[0]
nspw = len(freq[0])
for ispw in range(nspw):
ilam = 3e8 / freq[0][ispw]
indx_spw = uvspw == ispw
uvw[:, indx_spw] /= ilam
uu = []
vv = []
ww = []
for ipol in range(npol):
uu.append(uvw[0, :])
vv.append(uvw[1, :])
ww.append(uvw[2, :])
uu = numpy.array(uu)
vv = numpy.array(vv)
ww = numpy.array(ww)
if uu[:, 0].size == 1:
uu = uu.flatten()
vv = vv.flatten()
ww = ww.flatten()
return uu, vv, ww
def read_msinfo(vis=None, msinfofile=None, use_scan_time=True):
import glob
# read MS information #
msinfo = dict.fromkeys([
'vis', 'scans', 'fieldids', 'btimes', 'btimestr', 'inttimes', 'ras',
'decs', 'observatory'
])
ms.open(vis)
metadata = ms.metadata()
observatory = metadata.observatorynames()[0]
scans = ms.getscansummary()
scanids = sorted(scans.keys(), key=lambda x: int(x))
nscanid = len(scanids)
btimes = []
btimestr = []
etimes = []
fieldids = []
inttimes = []
dirs = []
ras = []
decs = []
ephem_file = glob.glob(vis + '/FIELD/EPHEM*SUN.tab')
if ephem_file:
print('Loading ephemeris info from {}'.format(ephem_file[0]))
tb.open(ephem_file[0])
col_ra = tb.getcol('RA')
col_dec = tb.getcol('DEC')
col_mjd = tb.getcol('MJD')
if use_scan_time:
from scipy.interpolate import interp1d
f_ra = interp1d(col_mjd, col_ra)
f_dec = interp1d(col_mjd, col_dec)
for idx, scanid in enumerate(scanids):
btimes.append(scans[scanid]['0']['BeginTime'])
etimes.append(scans[scanid]['0']['EndTime'])
fieldid = scans[scanid]['0']['FieldId']
fieldids.append(fieldid)
inttimes.append(scans[scanid]['0']['IntegrationTime'])
ras = f_ra(np.array(btimes))
decs = f_dec(np.array(btimes))
ras = qa.convert(qa.quantity(ras, 'deg'), 'rad')
decs = qa.convert(qa.quantity(decs, 'deg'), 'rad')
else:
ras = qa.convert(qa.quantity(col_ra, 'deg'), 'rad')
decs = qa.convert(qa.quantity(col_dec, 'deg'), 'rad')
else:
for idx, scanid in enumerate(scanids):
btimes.append(scans[scanid]['0']['BeginTime'])
etimes.append(scans[scanid]['0']['EndTime'])
fieldid = scans[scanid]['0']['FieldId']
fieldids.append(fieldid)
inttimes.append(scans[scanid]['0']['IntegrationTime'])
dir = ms.getfielddirmeas('PHASE_DIR', fieldid)
dirs.append(dir)
ras.append(dir['m0'])
decs.append(dir['m1'])
ms.close()
btimestr = [
qa.time(qa.quantity(btimes[idx], 'd'), form='fits', prec=10)[0]
for idx in range(nscanid)
]
msinfo['vis'] = vis
msinfo['scans'] = scans
msinfo['fieldids'] = fieldids
msinfo['btimes'] = btimes
msinfo['btimestr'] = btimestr
msinfo['inttimes'] = inttimes
msinfo['ras'] = ras
msinfo['decs'] = decs
msinfo['observatory'] = observatory
if msinfofile:
np.savez(msinfofile,
vis=vis,
scans=scans,
fieldids=fieldids,
btimes=btimes,
btimestr=btimestr,
inttimes=inttimes,
ras=ras,
decs=decs,
observatory=observatory)
return msinfo
from copy import copy
from taskinit import tb
from tasks import split
'''
Identify the continuum and line SPWs and split into separate MSs and
directories
'''
logprint("Starting EVLA_pipe_mixed_setup_split.py",
logfileout='logs/mixed_setup_split.log')
# Figure out which are the lines and which are the continuum SPWs.
tb.open(ms_active + '/SPECTRAL_WINDOW')
bandwidths = tb.getcol('TOTAL_BANDWIDTH')
tb.close()
tb.open(ms_active + '/FIELD')
fields = tb.getcol('NAME')
tb.close()
# Define a threshold between expected bandwidths
# Going with 10 MHz
thresh_bw = 1.0e7
spws = np.arange(0, len(bandwidths))
line_spws = [str(i) for i in spws[np.where(bandwidths < thresh_bw)]]
cont_spws = [str(i) for i in spws[np.where(bandwidths > thresh_bw)]]
