def get_array_config(vis):
msmd.open(vis)
obstime = Time(msmd.timerangeforobs(0)['begin']['m0']['value'],
format='mjd')
antennadiameter = msmd.antennadiameter(0)['value']
msmd.close()
if antennadiameter == 12:
if os.path.exists(vis + "/ASDM_EXECBLOCK"):
tb.open(vis + "/ASDM_EXECBLOCK")
else:
asdm_execblock = vis.replace(
"calibrated", "calibrated_pipeline") + "/ASDM_EXECBLOCK"
if os.path.exists(asdm_execblock):
tb.open(asdm_execblock)
else:
raise IOError("No ASDM_EXECBLOCK found for vis " + vis)
mous = tb.getcol('sessionReference')[0].split('"')[1].split("/")[-1]
configname = str(tb.getcol('configName')[0])
tb.close()
else:
configname = '7M'
mous = '7M'
return obstime.datetime, configname
def getAntennaPosition(vis):
tb.open(vis + '/ANTENNA')
position = tb.getcol('POSITION')
diameter = tb.getcol('DISH_DIAMETER')
antenna = tb.getcol('NAME')
tb.close()
return position, diameter, antenna
def oldMSpcd(msFile):
"""
get phase center
works for most data without uv-regrid
Parameters
----------
msFile: string
measurement set filename
Returns
-------
pcd:
phase center
Note
----
by Shane
works for data without uv-regrid using CASA's fixvis()
the following will give the old phase center otherwise
"""
from taskinit import tb
tb.open(msFile + '/SOURCE')
pcd_ra = tb.getcol('DIRECTION')[0][0] * 180 / numpy.pi
if pcd_ra < 0:
pcd_ra += 360
pcd_dec = tb.getcol('DIRECTION')[1][0] * 180 / numpy.pi
tb.close()
pcd = [pcd_ra, pcd_dec]
return pcd
def msclearhistory(msfile):
from taskinit import tb
tb.open(msfile + '/HISTORY', nomodify=False)
nrows = tb.nrows()
if nrows > 0:
tb.removerows(range(nrows))
tb.close()
def time2filename(msfile, timerange='', spw=''):
from astropy.time import Time
tb.open(msfile)
starttim = Time(tb.getcell('TIME', 0) / 24. / 3600., format='mjd')
endtim = Time(tb.getcell('TIME', tb.nrows() - 1) / 24. / 3600., format='mjd')
tb.close()
datstr = starttim.iso[:10]
ms.open(msfile)
metadata = ms.metadata()
observatory = metadata.observatorynames()[0]
ms.close()
if timerange is None or timerange == '':
starttim1 = starttim
endtim1 = endtim
else:
(tstart, tend) = timerange.split('~')
if tstart[2] == ':':
starttim1 = Time(datstr + 'T' + tstart)
endtim1 = Time(datstr + 'T' + tend)
else:
starttim1 = Time(qa.quantity(tstart, 'd')['value'], format='mjd')
endtim1 = Time(qa.quantity(tend, 'd')['value'], format='mjd')
midtime = Time((starttim1.mjd + endtim1.mjd) / 2., format='mjd')
tstr = midtime.to_datetime().strftime('{}_%Y%m%dT%H%M%S'.format(observatory))
if spw:
spstr = 'spw{}'.format(spw.replace('~', '-'))
filename = '.'.join([tstr, spstr])
else:
filename = tstr
return filename
def get_trange(msfile):
from astropy.time import Time
tb.open(msfile)
tr = np.array([tb.getcell('TIME', 0),
tb.getcell('TIME',
tb.nrows() - 1)]) / 24. / 3600.
tb.close()
return Time(tr, format='mjd')
def concateovsa(msname,
msfiles,
visprefix='./',
doclearcal=True,
keep_orig_ms=False,
cols2rm=["MODEL_DATA", "CORRECTED_DATA"]):
concatvis = visprefix + msname
msfiles_ = []
for idx, ll in enumerate(msfiles):
if str(ll).endswith('/'):
msfiles[idx] = str(ll)[:-1]
if doclearcal:
print 'Warning: Corrected column in the input ms file will be cleared!!!'
for ll in msfiles:
clearcal(vis=str(ll), addmodel=True)
else:
try:
tmpdir = visprefix + '/tmp_ms/'
if not os.path.exists(tmpdir):
os.makedirs(tmpdir)
for ll in msfiles:
msfile_ = tmpdir + os.path.basename(str(ll))
msfiles_.append(msfile_)
split(vis=str(ll), outputvis=msfile_, datacolumn='corrected')
clearcal(vis=msfile_, addmodel=True)
except:
print 'Warning: Corrected column not found in the input ms file.'
msfiles_ = msfiles
if msfiles_:
concat(vis=msfiles_, concatvis=concatvis, timesort=True)
else:
concat(vis=msfiles, concatvis=concatvis, timesort=True)
# Change all observation ids to be the same (zero)
tb.open(concatvis + '/OBSERVATION', nomodify=False)
nobs = tb.nrows()
tim0 = tb.getcell('TIME_RANGE', 0)[0]
tim1 = tb.getcell('TIME_RANGE', nobs - 1)[1]
tb.removerows([i + 1 for i in range(nobs - 1)])
tb.putcell('TIME_RANGE', 0, [tim0, tim1])
tb.close()
tb.open(concatvis, nomodify=False)
obsid = tb.getcol('OBSERVATION_ID')
newobsid = np.zeros(len(obsid), dtype='int')
tb.putcol('OBSERVATION_ID', newobsid)
colnames = tb.colnames()
for l in range(len(cols2rm)):
if cols2rm[l] in colnames:
try:
tb.removecols(cols2rm[l])
except:
pass
tb.close()
if msfiles_ != [] and msfiles_ != msfiles:
for ll in msfiles_:
os.system('rm -rf {}'.format(ll))
if not keep_orig_ms:
for ll in msfiles:
os.system('rm -rf {}'.format(ll))
def ms_clearhistory(msfile):
from taskinit import tb
tb_history = msfile + '/HISTORY'
os.system('cp -r {0} {0}_bk'.format(tb_history))
tb.open(tb_history, nomodify=False)
nrows = tb.nrows()
if nrows > 0:
tb.removerows(range(nrows))
tb.close()
def verifyMS(msname,
expnumspws,
expnumchan,
inspw,
expchanfreqs=[],
ignoreflags=False):
'''Function to verify spw and channels information in an MS
msname --> name of MS to verify
expnumspws --> expected number of SPWs in the MS
expnumchan --> expected number of channels in spw
inspw --> SPW ID
expchanfreqs --> numpy array with expected channel frequencies
ignoreflags --> do not check the FLAG column
Returns a list with True or False and a state message'''
msg = ''
tb.open(msname + '/SPECTRAL_WINDOW')
nc = tb.getcell("NUM_CHAN", inspw)
nr = tb.nrows()
cf = tb.getcell("CHAN_FREQ", inspw)
tb.close()
# After channel selection/average, need to know the exact row number to check,
# ignore this check in these cases.
if not ignoreflags:
tb.open(msname)
dimdata = tb.getcell("FLAG", 0)[0].size
tb.close()
if not (nr == expnumspws):
msg = "Found " + str(nr) + ", expected " + str(
expnumspws) + " spectral windows in " + msname
return [False, msg]
if not (nc == expnumchan):
msg = "Found " + str(nc) + ", expected " + str(
expnumchan) + " channels in spw " + str(inspw) + " in " + msname
return [False, msg]
if not ignoreflags and (dimdata != expnumchan):
msg = "Found " + str(dimdata) + ", expected " + str(
expnumchan) + " channels in FLAG column in " + msname
return [False, msg]
if not (expchanfreqs == []):
print "Testing channel frequencies ..."
# print cf
# print expchanfreqs
if not (expchanfreqs.size == expnumchan):
msg = "Internal error: array of expected channel freqs should have dimension ", expnumchan
return [False, msg]
df = (cf - expchanfreqs) / expchanfreqs
if not (abs(df) < 1E-8).all:
msg = "channel frequencies in spw " + str(
inspw
) + " differ from expected values by (relative error) " + str(df)
return [False, msg]
return [True, msg]
def splitX(vis, datacolumn2='MODEL_DATA', **kwargs):
import os
from clearcal_cli import clearcal_cli as clearcal
from split_cli import split_cli as split
'''
:param msfile:
:param datacolumn:
:param datacolumn2:
:return:
'''
kwargs2 = kwargs.copy()
datacolumn2 = datacolumn2.upper()
outmsfile = kwargs['outputvis']
if outmsfile.endswith('/'):
outmsfile = outmsfile[:-1]
if os.path.exists(outmsfile):
os.system('rm -rf {}'.format(outmsfile))
if os.path.exists('{}.flagversions'.format(outmsfile)):
os.system('rm -rf {}.flagversions'.format(outmsfile))
split(vis, **kwargs)
for k in ['datacolumn', 'outputvis']:
if k in kwargs2:
kwargs2.pop(k)
kwargs2['outputvis'] = 'tmpms.ms'
kwargs2['datacolumn'] = datacolumn2.replace('_DATA', '')
if os.path.exists('tmpms.ms'):
os.system('rm -rf tmpms.ms')
split(vis, **kwargs2)
tb.open('tmpms.ms')
nrows = tb.nrows()
data = []
for row in tqdm(range(nrows),
desc='getting {} column'.format(datacolumn2),
ascii=True):
data.append(tb.getcell('DATA', row))
tb.close()
clearcal(outmsfile, addmodel=True)
tb.open(outmsfile, nomodify=False)
for row in tqdm(range(nrows),
desc='writing {} column'.format(datacolumn2),
ascii=True):
tb.putcell(datacolumn2, row, data[row])
tb.close()
os.system('rm -rf tmpms.ms')
return outmsfile
def getspwfreq(vis):
'''
:param vis:
:return: mid frequencies in GHz of each spw in the vis
'''
tb.open(vis + '/SPECTRAL_WINDOW')
reffreqs = tb.getcol('REF_FREQUENCY')
bdwds = tb.getcol('TOTAL_BANDWIDTH')
cfreqs = reffreqs + bdwds / 2.
tb.close()
cfreqs = cfreqs / 1.0e9
return cfreqs
def checkwithtaql(taqlstring):
os.system('rm -rf mynewtable.tab')
tb.create('mynewtable.tab')
tb.open('mynewtable.tab',nomodify=False)
rval = tb.taql(taqlstring)
tb.close()
therval = rval.nrows()
tmpname = rval.name()
rval.close()
os.system('rm -rf mynewtable.tab')
os.system('rm -rf '+tmpname)
print "Found ", therval, " rows in selection."
return therval
def checkwithtaql(taqlstring):
os.system('rm -rf mynewtable.tab')
tb.create('mynewtable.tab')
tb.open('mynewtable.tab', nomodify=False)
rval = tb.taql(taqlstring)
tb.close()
therval = rval.nrows()
tmpname = rval.name()
rval.close()
os.system('rm -rf mynewtable.tab')
os.system('rm -rf ' + tmpname)
print "Found ", therval, " rows in selection."
return therval
def find_expected_beams(vis, spw, baseline_percentile=95):
'''
Return the expected synthesized beam (approximately) and the primary
beam size based on the baselines.
Parameters
----------
vis : str
Name of MS.
spw : int
Which SPW in the MS to consider.
baseline_percentile : int or float between 0 and 100
The percentile of the longest baseline to estimate the synthesized
beam with.
Returns
-------
syn_beam : float
Approximate Synthesized beam in arcseconds
prim_beam : float
Primary beam size in arcseconds.
'''
# Get percentile of max baseline and dish size
bline_max = getBaselinePercentile(vis, baseline_percentile)
tb.open(os.path.join(vis, 'ANTENNA'))
dishs = tb.getcol('DISH_DIAMETER')
dish_min = min(dishs)
tb.close()
tb.open(os.path.join(vis, 'SPECTRAL_WINDOW'))
ref_freqs = tb.getcol('REF_FREQUENCY')
try:
freq = ref_freqs[spw]
except IndexError:
raise IndexError("Given SPW ({0}) is not within the range of SPWs"
"found ({1})".format(spw, len(ref_freqs)))
# Find the beam
# XXX
# When astropy is easier to install (CASA 4.7??), switch to using the
# defined constants.
centre_lambda = 299792458.0 / freq
syn_beam = (centre_lambda / bline_max) * 180 / np.pi * 3600
prim_beam = (centre_lambda / dish_min) * 180 / np.pi * 3600
return syn_beam, prim_beam
def msclearhistory(msfile):
'''Clears history in the a measurement sets file
:param msfile: string
The name of a measurement sets file
:return:
'''
tb.open(msfile + '/HISTORY', nomodify=False)
nrows = tb.nrows()
if nrows > 0:
tb.removerows(range(nrows))
tb.close()
def getObservatoryName(ms):
"""
Returns the observatory name in the specified ms, using the tb tool.
-- Todd Hunter
"""
antTable = ms + '/OBSERVATION'
try:
tb.open(antTable)
myName = tb.getcell('TELESCOPE_NAME')
tb.close()
except:
print("Could not open OBSERVATION table to get the telescope name: {}".
format(antTable))
myName = ''
return (myName)
def get_freq_wlcm_bm(vis=None):
tb.open(vis + '/SPECTRAL_WINDOW')
tb.open(vis + '/SPECTRAL_WINDOW')
reffreqs = tb.getcol('REF_FREQUENCY')
bdwds = tb.getcol('TOTAL_BANDWIDTH')
cfreqs = reffreqs + bdwds / 2.
tb.close()
sbeam = 40.
bm = np.zeros(30)
wlcm = np.zeros(30)
for sp in range(30):
cfreq = cfreqs[sp]
wlcm[sp] = 30000000000 / cfreq
bm[sp] = max(sbeam * cfreqs[1] / cfreq, 10.)
return (cfreqs, wlcm, bm)
def insertdiskmodel(vis, sizescale=1.0, fdens=None, dsize=None, xmlfile='SOLDISK.xml', writediskinfoonly=False,
active=False, overwrite=True):
# Apply size scale adjustment (default is no adjustment)
for i in range(len(dsize)):
num, unit = dsize[i].split('arc')
dsize[i] = str(float(num) * sizescale)[:6] + 'arc' + unit
msfile = vis
ms.open(msfile)
spwinfo = ms.getspectralwindowinfo()
nspw = len(spwinfo.keys())
ms.close()
diskimdir = 'diskim/'
if not os.path.exists(diskimdir):
os.makedirs(diskimdir)
frq = []
spws = range(nspw)
for sp in spws:
spw = spwinfo[str(sp)]
frq.append('{:.4f}GHz'.format((spw['RefFreq'] + spw['TotalWidth'] / 2.0) / 1e9))
frq = np.array(frq)
writediskxml(dsize, fdens, frq, xmlfile=xmlfile)
if not writediskinfoonly:
tb.open(msfile + '/FIELD')
phadir = tb.getcol('PHASE_DIR').flatten()
tb.close()
ra = phadir[0]
dec = phadir[1]
direction = 'J2000 ' + str(ra) + 'rad ' + str(dec) + 'rad'
diskim = []
for sp in tqdm(spws, desc='Generating {} disk models'.format(nspw), ascii=True):
diskim.append(
mk_diskmodel(outname=diskimdir + 'disk{:02d}_'.format(sp), bdwidth=spwinfo[str(sp)],
direction=direction,
reffreq=frq[sp],
flux=fdens[sp], eqradius=dsize[sp], polradius=dsize[sp], overwrite=overwrite))
if not active:
delmod(msfile, otf=True, scr=True)
for sp in tqdm(spws, desc='Inserting disk model', ascii=True):
ft(vis=msfile, spw=str(sp), field='', model=str(diskim[sp]), nterms=1,
reffreq="", complist="", incremental=True, usescratch=True)
return msfile, diskim
def pcdload(visfile):
checker = visfile.find('uvfits')
if checker == -1:
uvfits = False
else:
uvfits = True
if uvfits:
# uv fits format
visdata = fits.open(visfile)
visheader = visdata[0].header
if visheader['NAXIS'] == 7:
# identify the phase center
try:
pcd_ra = visdata['AIPS SU '].data['RAEPO'][0]
pcd_dec = visdata['AIPS SU '].data['DECEPO'][0]
except:
pcd_ra = visheader['CRVAL6'] #RP mod was 5
pcd_dec = visheader['CRVAL7'] #RP mod was 6
if pcd_ra < 0:
pcd_ra += 360
pcd = [pcd_ra, pcd_dec]
return pcd
if visheader['NAXIS'] == 6:
# identify the channel frequency(ies):
pcd_ra = visdata[0].header['CRVAL5']
pcd_dec = visdata[0].header['CRVAL6']
if pcd_ra < 0:
pcd_ra += 360
pcd = [pcd_ra, pcd_dec]
return pcd
else:
# CASA MS
from taskinit import tb
tb.open(visfile + '/SOURCE')
pcd_ra = tb.getcol('DIRECTION')[0][0] * 180 / numpy.pi
if pcd_ra < 0:
pcd_ra += 360
pcd_dec = tb.getcol('DIRECTION')[1][0] * 180 / numpy.pi
tb.close()
pcd = [pcd_ra, pcd_dec]
return pcd
def pcdload(visfile):
checker = visfile.find('uvfits')
if checker == -1:
uvfits = False
else:
uvfits = True
if uvfits:
# uv fits format
visdata = fits.open(visfile)
visheader = visdata[0].header
if visheader['NAXIS'] == 7:
# identify the phase center
try:
pcd_ra = visdata['AIPS SU '].data['RAEPO'][0]
pcd_dec = visdata['AIPS SU '].data['DECEPO'][0]
except:
pcd_ra = visheader['CRVAL5']
pcd_dec = visheader['CRVAL6']
if pcd_ra < 0:
pcd_ra += 360
pcd = [pcd_ra, pcd_dec]
return pcd
if visheader['NAXIS'] == 6:
# identify the channel frequency(ies):
pcd_ra = visdata[0].header['CRVAL5']
pcd_dec = visdata[0].header['CRVAL6']
if pcd_ra < 0:
pcd_ra += 360
pcd = [pcd_ra, pcd_dec]
return pcd
else:
# CASA MS
from taskinit import tb
tb.open(visfile + '/SOURCE')
pcd_ra = tb.getcol('DIRECTION')[0][0] * 180 / numpy.pi
if pcd_ra < 0:
pcd_ra += 360
pcd_dec = tb.getcol('DIRECTION')[1][0] * 180 / numpy.pi
tb.close()
pcd = [pcd_ra, pcd_dec]
return pcd
def verifyMS(msname, expnumspws, expnumchan, inspw, expchanfreqs=[], ignoreflags=False):
'''Function to verify spw and channels information in an MS
msname --> name of MS to verify
expnumspws --> expected number of SPWs in the MS
expnumchan --> expected number of channels in spw
inspw --> SPW ID
expchanfreqs --> numpy array with expected channel frequencies
ignoreflags --> do not check the FLAG column
Returns a list with True or False and a state message'''
msg = ''
tb.open(msname+'/SPECTRAL_WINDOW')
nc = tb.getcell("NUM_CHAN", inspw)
nr = tb.nrows()
cf = tb.getcell("CHAN_FREQ", inspw)
tb.close()
# After channel selection/average, need to know the exact row number to check,
# ignore this check in these cases.
if not ignoreflags:
tb.open(msname)
dimdata = tb.getcell("FLAG", 0)[0].size
tb.close()
if not (nr==expnumspws):
msg = "Found "+str(nr)+", expected "+str(expnumspws)+" spectral windows in "+msname
return [False,msg]
if not (nc == expnumchan):
msg = "Found "+ str(nc) +", expected "+str(expnumchan)+" channels in spw "+str(inspw)+" in "+msname
return [False,msg]
if not ignoreflags and (dimdata != expnumchan):
msg = "Found "+ str(dimdata) +", expected "+str(expnumchan)+" channels in FLAG column in "+msname
return [False,msg]
if not (expchanfreqs==[]):
print "Testing channel frequencies ..."
# print cf
# print expchanfreqs
if not (expchanfreqs.size == expnumchan):
msg = "Internal error: array of expected channel freqs should have dimension ", expnumchan
return [False,msg]
df = (cf - expchanfreqs)/expchanfreqs
if not (abs(df) < 1E-8).all:
msg = "channel frequencies in spw "+str(inspw)+" differ from expected values by (relative error) "+str(df)
return [False,msg]
return [True,msg]
def getVarCol(table, colname):
'''Return the requested variable column
table --> name of table or MS
colname --> column name
Return the column as a dictionary'''
col = {}
try:
try:
tb.open(table)
col = tb.getvarcol(colname)
except:
print 'Cannot open table ' + table
finally:
tb.close()
return col
def cpxx2yy(tb_in=[]):
if not tb_in:
print('tb_in not provided. Abort...')
if type(tb_in) is str:
tb_in = [tb_in]
tb.open(tb_in[0] + '/SPECTRAL_WINDOW', nomodify=False)
nspw = tb.nrows()
tb.close()
for ctb in tb_in:
tb.open(ctb, nomodify=False)
for s in range(nspw):
subt = tb.query("DATA_DESC_ID==" + str(s))
model_d = subt.getcol('MODEL_DATA')
# cp xx to yy
model_d[1] = model_d[0]
subt.putcol('MODEL_DATA', model_d)
subt.close()
tb.close()
def getVarCol(table, colname):
'''Return the requested variable column
table --> name of table or MS
colname --> column name
Return the column as a dictionary'''
col = {}
try:
try:
tb.open(table)
col = tb.getvarcol(colname)
except:
print 'Cannot open table '+table
finally:
tb.close()
return col
def getBaselinePercentile(msFile, percentile):
"""
Based on getBaselineExtrema from analysisUtils
"""
tb.open(msFile + '/ANTENNA')
positions = np.transpose(tb.getcol('POSITION'))
tb.close()
all_lengths = []
for i in range(len(positions)):
for j in range(i + 1, len(positions)):
length = au.computeBaselineLength(positions[i],
positions[j])
if length != 0.0:
all_lengths.append(length)
all_lengths = np.array(all_lengths)
return np.percentile(all_lengths, percentile)
def getColDesc(table, colname):
'''Get the description of a column in a table
table --> name of table or MS
colname --> column name
Return a dictionary with the column description'''
coldesc = {}
try:
try:
tb.open(table)
tcols = tb.colnames()
if tcols.__contains__(colname):
coldesc = tb.getcoldesc(colname)
except:
pass
finally:
tb.close()
return coldesc
def getColDesc(table, colname):
'''Get the description of a column in a table
table --> name of table or MS
colname --> column name
Return a dictionary with the column description'''
coldesc = {}
try:
try:
tb.open(table)
tcols = tb.colnames()
if tcols.__contains__(colname):
coldesc = tb.getcoldesc(colname)
except:
pass
finally:
tb.close()
return coldesc
def set_imagermode(vis, source):
tb.open(os.path.join(vis, 'FIELD'))
names = tb.getcol('NAME')
tb.close()
moscount = 0
for name in names:
chsrc = name.find(source)
if chsrc != -1:
moscount = moscount + 1
if moscount > 1:
imagermode = "mosaic"
else:
imagermode = "csclean"
return imagermode
def writeVis(vis_complex, visdataloc, modelvisloc, miriad=False):
if miriad:
os.system('rm -rf ' + modelvisloc)
cmd = 'cp ' + visdataloc + ' ' + modelvisloc
os.system(cmd)
# get the real and imaginary components
real = numpy.real(vis_complex)
imag = numpy.imag(vis_complex)
# replace data visibilities with model visibilities
visfile = fits.open(modelvisloc, mode='update')
visibilities = visfile[0].data
visheader = visfile[0].header
if visheader['NAXIS'] == 7:
visibilities['DATA'][:, 0, 0, :, :, :, 0] = real
visibilities['DATA'][:, 0, 0, :, :, :, 1] = imag
elif visheader['NAXIS'] == 6:
visibilities['DATA'][:, 0, 0, :, :, 0] = real
visibilities['DATA'][:, 0, 0, :, :, 1] = imag
else:
print("Visibility dataset has >7 or <6 axes. I can't read this.")
#visibilities['DATA'][:, 0, 0, :, :, 2] = wgt
# replace the data visibilities with the model visibilities
visfile[0].data = visibilities
visfile.flush()
else:
from taskinit import tb
print("Writing visibility data to " + modelvisloc)
os.system('rm -rf ' + modelvisloc)
tb.open(visdataloc)
tb.copy(modelvisloc)
tb.close()
tb.open(modelvisloc, nomodify=False)
datashape = tb.getcol('DATA').shape
vis_complex = vis_complex.reshape(datashape)
tb.putcol('DATA', vis_complex)
tb.close()
def writeVis(vis_complex, visdataloc, modelvisloc, miriad=False):
if miriad:
os.system('rm -rf ' + modelvisloc)
cmd = 'cp ' + visdataloc + ' ' + modelvisloc
os.system(cmd)
# get the real and imaginary components
real = numpy.real(vis_complex)
imag = numpy.imag(vis_complex)
# replace data visibilities with model visibilities
visfile = fits.open(modelvisloc, mode='update')
visibilities = visfile[0].data
visheader = visfile[0].header
if visheader['NAXIS'] == 7:
visibilities['DATA'][:, 0, 0, :, :, :, 0] = real
visibilities['DATA'][:, 0, 0, :, :, :, 1] = imag
elif visheader['NAXIS'] == 6:
visibilities['DATA'][:, 0, 0, :, :, 0] = real
visibilities['DATA'][:, 0, 0, :, :, 1] = imag
else:
print("Visibility dataset has >7 or <6 axes. I can't read this.")
#visibilities['DATA'][:, 0, 0, :, :, 2] = wgt
# replace the data visibilities with the model visibilities
visfile[0].data = visibilities
visfile.flush()
else:
from taskinit import tb
print("Writing visibility data to " + modelvisloc)
os.system('rm -rf ' + modelvisloc)
tb.open(visdataloc)
tb.copy(modelvisloc)
tb.close()
tb.open(modelvisloc, nomodify=False)
datashape = tb.getcol('DATA').shape
vis_complex = vis_complex.reshape(datashape)
tb.putcol('DATA', vis_complex)
tb.close()
def has_field(vis, source):
'''
Check if source is contained in at least one of the field names.
'''
tb.open(os.path.join(vis, 'FIELD'))
names = tb.getcol('NAME')
tb.close()
moscount = 0
for name in names:
chsrc = name.find(source)
if chsrc != -1:
moscount = moscount + 1
if moscount == 0:
return False
return True
def getFreqfromtb(msFile):
"""get a list of IF from the measurement set table
same as ('CHAN_FREQ') if
Parameters
----------
msFile: string
CASA measurement set
Returns
-------
freq0: numpy.ndarray
reference frequency in Hz
"""
from taskinit import tb
tb.open(msFile+'/SPECTRAL_WINDOW')
freq0 = tb.getcol('REF_FREQUENCY')
nchan = tb.getcol('NUM_CHAN')
tb.close()
return freq0
def MSpcd(msFile):
"""
Explore different ways to get phase center consistent with data that has been regridded from one epoch to another
Parameters
----------
msFile: string
measurement set filename
Returns
-------
pcd: list
phase center
Note
----
which method gives the phase center that correspond to that used in CASA imaging needs further investigation
"""
from taskinit import tb
tb.open(msFile + '/FIELD')
old = tb.getcol('DELAY_DIR') #, fieldid
new = tb.getcol('PHASE_DIR')
def shiftRA(ra):
if ra < 0.: ra += numpy.pi * 2
return ra
old[0] = shiftRA(old[0])
new[0] = shiftRA(new[0])
# old phase center
# _pcd_ra, _pcd_dec = old[0] * 180. / numpy.pi, old[1] * 180. / numpy.pi
# new phase center
pcd_ra, pcd_dec = new[0] * 180. / numpy.pi, new[1] * 180. / numpy.pi
tb.close()
pcd = pcd_ra[0][0], pcd_dec[0][0]
return list(pcd)
def get_channel_freqs_widths(msname, spwid):
'''
Get frequencies and widths of all the channels for an spw ID
msname --> name of MS
spwid --> spw ID
Return two numpy arrays (frequencies, widths), each of the same length as the number of
channels'''
try:
spw_table = os.path.join(msname, 'SPECTRAL_WINDOW')
try:
tb.open(spw_table)
except RuntimeError:
print 'Cannot open table: {0}'.format(spw_table)
freqs = tb.getcell("CHAN_FREQ", spwid)
widths = tb.getcell("CHAN_WIDTH", spwid)
finally:
tb.close()
return freqs, widths
def getChannels(msname, spwid, chanlist):
'''From a list of channel indices, return their frequencies
msname --> name of MS
spwid --> spw ID
chanlist --> list of channel indices
Return a numpy array, the same size of chanlist, with the frequencies'''
try:
try:
tb.open(msname+'/SPECTRAL_WINDOW')
except:
print 'Cannot open table '+msname+'SPECTRAL_WINDOW'
cf = tb.getcell("CHAN_FREQ", spwid)
# Get only the requested channels
b = [cf[i] for i in chanlist]
selchans = np.array(b)
finally:
tb.close()
return selchans
def getChannels(msname, spwid, chanlist):
'''From a list of channel indices, return their frequencies
msname --> name of MS
spwid --> spw ID
chanlist --> list of channel indices
Return a numpy array, the same size of chanlist, with the frequencies'''
try:
try:
tb.open(msname + '/SPECTRAL_WINDOW')
except:
print 'Cannot open table ' + msname + 'SPECTRAL_WINDOW'
cf = tb.getcell("CHAN_FREQ", spwid)
# Get only the requested channels
b = [cf[i] for i in chanlist]
selchans = np.array(b)
finally:
tb.close()
return selchans
def clearflagrow(msfile, mode='clear'):
'''
:param msfile:
:param mode: FLAG_ROW operation
default: 'clear': (default) clear the FLAG_ROW
'list': to list existing FLAG_ROW
:return:
'''
if mode == 'list':
tb.open(msfile, nomodify=True)
a = tb.getcol('FLAG_ROW')
nfrows = np.sum(a)
nrows = float(len(a))
print('{:0d} out of {:.0f} ({:.0f}%) rows are flagged in {}'.format(nfrows, nrows, nfrows / nrows * 100,
os.path.basename(msfile)))
elif mode == 'clear':
tb.open(msfile, nomodify=False)
a = tb.getcol('FLAG_ROW')
a[:] = False
tb.putcol('FLAG_ROW', a)
print('reset successfully')
tb.close()
def concat_slftb(tb_in=[], tb_out=None):
if not tb_in:
print('tb_in not provided. Abort...')
if os.path.exists(tb_out):
os.system('rm -r {}'.format(tb_out))
os.system('cp -r {} {}'.format(tb_in[0], tb_out))
tbdata = {}
tb.open(tb_out)
cols = tb.colnames()
tb.close()
cols.remove('WEIGHT')
for col in cols:
tbdata[col] = []
for tbidx, ctb in enumerate(tb_in):
tb.open(ctb, nomodify=True)
tim0 = tb.getcol(cols[0])
if len(tim0) == 0:
continue
else:
for col in cols:
if tbidx == 1 and col in ['CPARAM', 'PARAMERR', 'FLAG', 'SNR']:
tbdata[col].append(tb.getcol(col)[::-1,...])
else:
tbdata[col].append(tb.getcol(col))
tb.close()
if len(tbdata[cols[0]]) == 0:
print('tables have no data. Return')
return -1
else:
for col in cols:
if col in ['CPARAM', 'PARAMERR', 'FLAG', 'SNR']:
tbdata[col] = np.concatenate(tbdata[col], axis=2)
else:
tbdata[col] = np.concatenate(tbdata[col])
tb.open(tb_out, nomodify=False)
nrows = tb.nrows()
nrows_new = len(tbdata[cols[0]])
tb.addrows(nrows_new - nrows)
for col in cols:
tb.putcol(col, tbdata[col])
tb.close()
return tb_out
def compTables(referencetab,
testtab,
excludecols,
tolerance=0.001,
mode="percentage",
startrow=0,
nrow=-1,
rowincr=1):
"""
compTables - compare two CASA tables
referencetab - the table which is assumed to be correct
testtab - the table which is to be compared to referencetab
excludecols - list of column names which are to be ignored
tolerance - permitted fractional difference (default 0.001 = 0.1 percent)
mode - comparison is made as "percentage", "absolute", "phaseabsdeg" (for complex numbers = difference of the phases in degrees)
"""
rval = True
tb2 = casac.table()
tb.open(referencetab)
cnames = tb.colnames()
tb2.open(testtab)
try:
for c in cnames:
if c in excludecols:
continue
print "\nTesting column " + c
a = 0
try:
a = tb.getcol(c, startrow=startrow, nrow=nrow, rowincr=rowincr)
except:
rval = False
print 'Error accessing column ', c, ' in table ', referencetab
print sys.exc_info()[0]
break
b = 0
try:
b = tb2.getcol(c,
startrow=startrow,
nrow=nrow,
rowincr=rowincr)
except:
rval = False
print 'Error accessing column ', c, ' in table ', testtab
print sys.exc_info()[0]
break
if not (len(a) == len(b)):
print 'Column ', c, ' has different length in tables ', referencetab, ' and ', testtab
print a
print b
rval = False
break
else:
differs = False
if not (a == b).all():
for i in range(0, len(a)):
if (isinstance(a[i], float)):
if ((mode == "percentage") and
(abs(a[i] - b[i]) > tolerance * abs(a[i]))
) or ((mode == "absolute") and
(abs(a[i] - b[i]) > tolerance)):
print "Column " + c + " differs"
print "Row=" + str(i)
print "Reference file value: " + str(a[i])
print "Input file value: " + str(b[i])
if (mode == "percentage"):
print "Tolerance is {0}%; observed difference was {1} %".format(
tolerance * 100,
100 * abs(a[i] - b[i]) / abs(a[i]))
else:
print "Absolute tolerance is {0}; observed difference: {1}".format(
tolerance, (abs(a[i] - b[i])))
differs = True
rval = False
break
elif (isinstance(a[i], int)
or isinstance(a[i], np.int32)):
if (abs(a[i] - b[i]) > 0):
print "Column " + c + " differs"
print "Row=" + str(i)
print "Reference file value: " + str(a[i])
print "Input file value: " + str(b[i])
if (mode == "percentage"):
print "tolerance in % should be " + str(
100 * abs(a[i] - b[i]) / abs(a[i]))
else:
print "absolute tolerance should be " + str(
abs(a[i] - b[i]))
differs = True
rval = False
break
elif (isinstance(a[i], str)
or isinstance(a[i], np.bool_)):
if not (a[i] == b[i]):
print "Column " + c + " differs"
print "Row=" + str(i)
print "Reference file value: " + str(a[i])
print "Input file value: " + str(b[i])
if (mode == "percentage"):
print "tolerance in % should be " + str(
100 * abs(a[i] - b[i]) / abs(a[i]))
else:
print "absolute tolerance should be " + str(
abs(a[i] - b[i]))
differs = True
rval = False
break
elif (isinstance(a[i], list)) or (isinstance(
a[i], np.ndarray)):
for j in range(0, len(a[i])):
if differs: break
if ((isinstance(a[i][j], float))
or (isinstance(a[i][j], int))):
if ((mode == "percentage") and
(abs(a[i][j] - b[i][j]) >
tolerance * abs(a[i][j]))) or (
(mode == "absolute") and
(abs(a[i][j] - b[i][j]) >
tolerance)):
print "Column " + c + " differs"
print "(Row,Element)=(" + str(
j) + "," + str(i) + ")"
print "Reference file value: " + str(
a[i][j])
print "Input file value: " + str(
b[i][j])
if (mode == "percentage"):
print "Tolerance in % should be " + str(
100 * abs(a[i][j] - b[i][j]) /
abs(a[i][j]))
else:
print "Absolute tolerance should be " + str(
abs(a[i][j] - b[i][j]))
differs = True
rval = False
break
elif (isinstance(a[i][j],
list)) or (isinstance(
a[i][j], np.ndarray)):
it = xrange(0, len(a[i][j]))
if mode == "percentage":
diff = np.abs(
np.subtract(a[i][j], b[i][j])
) > tolerance * np.abs(a[i][j])
it = np.where(diff)[0]
elif (mode == "absolute"):
diff = np.abs(
np.subtract(a[i][j],
b[i][j])) > tolerance
it = np.where(diff)[0]
for k in it:
if differs: break
if ( ((mode=="percentage") and (abs(a[i][j][k]-b[i][j][k]) > tolerance*abs(a[i][j][k]))) \
or ((mode=="absolute") and (abs(a[i][j][k]-b[i][j][k]) > tolerance)) \
or ((mode=="phaseabsdeg") and (phasediffabsdeg(a[i][j][k],b[i][j][k])>tolerance)) \
):
print "Column " + c + " differs"
print "(Row,Channel,Corr)=(" + str(
k) + "," + str(j) + "," + str(
i) + ")"
print "Reference file value: " + str(
a[i][j][k])
print "Input file value: " + str(
b[i][j][k])
if (mode == "percentage"):
print "Tolerance in % should be " + str(
100 * abs(a[i][j][k] -
b[i][j][k]) /
abs(a[i][j][k]))
elif (mode == "absolute"):
print "Absolute tolerance should be " + str(
abs(a[i][j][k] -
b[i][j][k]))
elif (mode == "phaseabsdeg"):
print "Phase tolerance in degrees should be " + str(
phasediffabsdeg(
a[i][j][k],
b[i][j][k]))
else:
print "Unknown comparison mode: ", mode
differs = True
rval = False
break
else:
print "Unknown data type: ", type(a[i])
differs = True
rval = False
break
if not differs: print "Column " + c + " PASSED"
finally:
tb.close()
tb2.close()
return rval
def compVarColTables(referencetab, testtab, varcol, tolerance=0.):
'''Compare a variable column of two tables.
referencetab --> a reference table
testtab --> a table to verify
varcol --> the name of a variable column (str)
Returns True or False.
'''
retval = True
tb2 = casac.table()
tb.open(referencetab)
cnames = tb.colnames()
tb2.open(testtab)
col = varcol
if tb.isvarcol(col) and tb2.isvarcol(col):
try:
# First check
if tb.nrows() != tb2.nrows():
print 'Length of %s differ from %s, %s!=%s' % (
referencetab, testtab, len(rk), len(tk))
retval = False
else:
for therow in xrange(tb.nrows()):
rdata = tb.getcell(col, therow)
tdata = tb2.getcell(col, therow)
# if not (rdata==tdata).all():
if not rdata.all() == tdata.all():
if (tolerance > 0.):
differs = False
for j in range(0, len(rdata)):
### if (type(rdata[j])==float or type(rdata[j])==int):
if ((isinstance(rdata[j], float))
or (isinstance(rdata[j], int))):
if (abs(rdata[j] - tdata[j]) > tolerance *
abs(rdata[j] + tdata[j])):
# print 'Column ', col,' differs in tables ', referencetab, ' and ', testtab
# print therow, j
# print rdata[j]
# print tdata[j]
differs = True
### elif (type(rdata[j])==list or type(rdata[j])==np.ndarray):
elif (isinstance(rdata[j],
list)) or (isinstance(
rdata[j], np.ndarray)):
for k in range(0, len(rdata[j])):
if (abs(rdata[j][k] - tdata[j][k]) >
tolerance * abs(rdata[j][k] +
tdata[j][k])):
# print 'Column ', col,' differs in tables ', referencetab, ' and ', testtab
# print therow, j, k
# print rdata[j][k]
# print tdata[j][k]
differs = True
if differs:
print 'ERROR: Column %s of %s and %s do not agree within tolerance %s' % (
col, referencetab, testtab, tolerance)
retval = False
break
else:
print 'ERROR: Column %s of %s and %s do not agree.' % (
col, referencetab, testtab)
print 'ERROR: First row to differ is row=%s' % therow
retval = False
break
finally:
tb.close()
tb2.close()
else:
print 'Columns are not varcolumns.'
retval = False
if retval:
print 'Column %s of %s and %s agree' % (col, referencetab, testtab)
return retval
def compVarColTables(referencetab, testtab, varcol, tolerance=0.):
'''Compare a variable column of two tables.
referencetab --> a reference table
testtab --> a table to verify
varcol --> the name of a variable column (str)
Returns True or False.
'''
retval = True
tb2 = casac.table()
tb.open(referencetab)
cnames = tb.colnames()
tb2.open(testtab)
col = varcol
if tb.isvarcol(col) and tb2.isvarcol(col):
try:
# First check
if tb.nrows() != tb2.nrows():
print 'Length of %s differ from %s, %s!=%s'%(referencetab,testtab,len(rk),len(tk))
retval = False
else:
for therow in xrange(tb.nrows()):
rdata = tb.getcell(col,therow)
tdata = tb2.getcell(col,therow)
# if not (rdata==tdata).all():
if not rdata.all()==tdata.all():
if (tolerance>0.):
differs=False
for j in range(0,len(rdata)):
### if (type(rdata[j])==float or type(rdata[j])==int):
if ((isinstance(rdata[j],float)) or (isinstance(rdata[j],int))):
if (abs(rdata[j]-tdata[j]) > tolerance*abs(rdata[j]+tdata[j])):
# print 'Column ', col,' differs in tables ', referencetab, ' and ', testtab
# print therow, j
# print rdata[j]
# print tdata[j]
differs = True
### elif (type(rdata[j])==list or type(rdata[j])==np.ndarray):
elif (isinstance(rdata[j],list)) or (isinstance(rdata[j],np.ndarray)):
for k in range(0,len(rdata[j])):
if (abs(rdata[j][k]-tdata[j][k]) > tolerance*abs(rdata[j][k]+tdata[j][k])):
# print 'Column ', col,' differs in tables ', referencetab, ' and ', testtab
# print therow, j, k
# print rdata[j][k]
# print tdata[j][k]
differs = True
if differs:
print 'ERROR: Column %s of %s and %s do not agree within tolerance %s'%(col,referencetab, testtab, tolerance)
retval = False
break
else:
print 'ERROR: Column %s of %s and %s do not agree.'%(col,referencetab, testtab)
print 'ERROR: First row to differ is row=%s'%therow
retval = False
break
finally:
tb.close()
tb2.close()
else:
print 'Columns are not varcolumns.'
retval = False
if retval:
print 'Column %s of %s and %s agree'%(col,referencetab, testtab)
return retval
def mk_qlook_image(trange,
doimport=False,
docalib=False,
ncpu=10,
twidth=12,
stokes=None,
antenna='0~12',
lowcutoff_freq=3.7,
imagedir=None,
spws=['1~5', '6~10', '11~15', '16~25'],
toTb=True,
overwrite=True,
doslfcal=False,
verbose=False):
'''
trange: can be 1) a single Time() object: use the entire day
2) a range of Time(), e.g., Time(['2017-08-01 00:00','2017-08-01 23:00'])
3) a single or a list of UDBms file(s)
4) None -- use current date Time.now()
'''
antenna0 = antenna
if type(trange) == Time:
mslist = trange2ms(trange=trange, doimport=doimport)
vis = mslist['ms']
tsts = [l.to_datetime() for l in mslist['tstlist']]
if type(trange) == str:
try:
date = Time(trange)
mslist = trange2ms(trange=trange, doimport=doimport)
vis = mslist['ms']
tsts = [l.to_datetime() for l in mslist['tstlist']]
except:
vis = [trange]
tsts = []
for v in vis:
tb.open(v + '/OBSERVATION')
tsts.append(
Time(tb.getcell('TIME_RANGE')[0] / 24 / 3600,
format='mjd').datetime)
tb.close()
subdir = [tst.strftime("%Y/%m/%d/") for tst in tsts]
for idx, f in enumerate(vis):
if f[-1] == '/':
vis[idx] = f[:-1]
if not stokes:
stokes = 'XX'
if not imagedir:
imagedir = './'
imres = {
'Succeeded': [],
'BeginTime': [],
'EndTime': [],
'ImageName': [],
'Spw': [],
'Vis': [],
'Synoptic': {
'Succeeded': [],
'BeginTime': [],
'EndTime': [],
'ImageName': [],
'Spw': [],
'Vis': []
}
}
for n, msfile in enumerate(vis):
msfilebs = os.path.basename(msfile)
imdir = imagedir + subdir[n]
if not os.path.exists(imdir):
os.makedirs(imdir)
if doslfcal:
slfcalms = './' + msfilebs + '.xx'
split(msfile,
outputvis=slfcalms,
datacolumn='corrected',
correlation='XX')
cfreqs = getspwfreq(msfile)
for spw in spws:
antenna = antenna0
if spw == '':
continue
spwran = [s.zfill(2) for s in spw.split('~')]
freqran = [cfreqs[int(s)] for s in spw.split('~')]
cfreq = np.mean(freqran)
bmsz = max(150. / cfreq, 20.)
uvrange = '<10klambda'
if doslfcal:
slfcal_img = './' + msfilebs + '.slf.spw' + spw.replace(
'~', '-') + '.slfimg'
slfcal_tb = './' + msfilebs + '.slf.spw' + spw.replace(
'~', '-') + '.slftb'
try:
clean(vis=slfcalms,
antenna=antenna,
imagename=slfcal_img,
spw=spw,
mode='mfs',
timerange='',
imagermode='csclean',
psfmode='clark',
imsize=[512, 512],
cell=['5arcsec'],
niter=100,
gain=0.05,
stokes='I',
weighting='natural',
restoringbeam=[str(bmsz) + 'arcsec'],
pbcor=False,
interactive=False,
usescratch=True)
except:
print('error in cleaning spw: ' + spw)
break
gaincal(vis=slfcalms,
refant='0',
antenna=antenna,
caltable=slfcal_tb,
spw=spw,
uvrange='',
gaintable=[],
selectdata=True,
timerange='',
solint='600s',
gaintype='G',
calmode='p',
combine='',
minblperant=3,
minsnr=2,
append=False)
if not os.path.exists(slfcal_tb):
print('No solution found in spw: ' + spw)
break
else:
clearcal(slfcalms)
delmod(slfcalms)
applycal(vis=slfcalms,
gaintable=[slfcal_tb],
spw=spw,
selectdata=True,
antenna=antenna,
interp='nearest',
flagbackup=False,
applymode='calonly',
calwt=False)
msfile = slfcalms
imsize = 512
cell = ['5arcsec']
if len(spwran) == 2:
spwstr = spwran[0] + '~' + spwran[1]
else:
spwstr = spwran[0]
restoringbeam = ['{0:.1f}arcsec'.format(bmsz)]
imagesuffix = '.spw' + spwstr.replace('~', '-')
if cfreq > 10.:
antenna = antenna + ';!0&1;!0&2' # deselect the shortest baselines
# else:
# antenna = antenna + ';!0&1' # deselect the shortest baselines
res = ptclean3(vis=msfile,
imageprefix=imdir,
imagesuffix=imagesuffix,
twidth=twidth,
uvrange=uvrange,
spw=spw,
ncpu=ncpu,
niter=1000,
gain=0.05,
antenna=antenna,
imsize=imsize,
cell=cell,
stokes=stokes,
doreg=True,
usephacenter=False,
overwrite=overwrite,
toTb=toTb,
restoringbeam=restoringbeam,
specmode="mfs",
deconvolver="hogbom",
datacolumn='data',
pbcor=True)
if res:
imres['Succeeded'] += res['Succeeded']
imres['BeginTime'] += res['BeginTime']
imres['EndTime'] += res['EndTime']
imres['ImageName'] += res['ImageName']
imres['Spw'] += [spwstr] * len(res['ImageName'])
imres['Vis'] += [msfile] * len(res['ImageName'])
else:
continue
if len(vis) == 1:
# produce the band-by-band whole-day images
ms.open(msfile)
ms.selectinit()
timfreq = ms.getdata(['time', 'axis_info'], ifraxis=True)
tim = timfreq['time']
ms.close()
cfreqs = getspwfreq(msfile)
imdir = imagedir + subdir[0]
if not os.path.exists(imdir):
os.makedirs(imdir)
for spw in spws:
antenna = antenna0
if spw == '':
spw = '{:d}~{:d}'.format(
next(x[0] for x in enumerate(cfreqs)
if x[1] > lowcutoff_freq),
len(cfreqs) - 1)
spwran = [s.zfill(2) for s in spw.split('~')]
freqran = [cfreqs[int(s)] for s in spw.split('~')]
cfreq = np.mean(freqran)
bmsz = max(150. / cfreq, 20.)
uvrange = ''
imsize = 512
cell = ['5arcsec']
if len(spwran) == 2:
spwstr = spwran[0] + '~' + spwran[1]
else:
spwstr = spwran[0]
restoringbeam = ['{0:.1f}arcsec'.format(bmsz)]
imagesuffix = '.synoptic.spw' + spwstr.replace('~', '-')
antenna = antenna + ';!0&1' # deselect the shortest baselines
res = ptclean3(vis=msfile,
imageprefix=imdir,
imagesuffix=imagesuffix,
twidth=len(tim),
uvrange=uvrange,
spw=spw,
ncpu=1,
niter=0,
gain=0.05,
antenna=antenna,
imsize=imsize,
cell=cell,
stokes=stokes,
doreg=True,
usephacenter=False,
overwrite=overwrite,
toTb=toTb,
restoringbeam=restoringbeam,
specmode="mfs",
deconvolver="hogbom",
datacolumn='data',
pbcor=True)
if res:
imres['Synoptic']['Succeeded'] += res['Succeeded']
imres['Synoptic']['BeginTime'] += res['BeginTime']
imres['Synoptic']['EndTime'] += res['EndTime']
imres['Synoptic']['ImageName'] += res['ImageName']
imres['Synoptic']['Spw'] += [spwstr] * len(res['ImageName'])
imres['Synoptic']['Vis'] += [msfile] * len(res['ImageName'])
else:
continue
# save it for debugging purposes
np.savez('imres.npz', imres=imres)
return imres
def writeVis(vis_complex, visdataloc, modelvisloc, miriad=False):
"""
Parameters
----------
vis_complex: numpy.array
visdataloc: string
uv-data filename
miriad: Boolean
True: not using CASA, just python data manipulation, hence can run in ipython,
False: use CASA taskinit module, must run inside CASA
Returns
-------
modelvisloc: string
uv-model filename
"""
if miriad:
os.system('rm -rf ' + modelvisloc)
cmd = 'cp ' + visdataloc + ' ' + modelvisloc
os.system(cmd)
# get the real and imaginary components
real = numpy.real(vis_complex)
imag = numpy.imag(vis_complex)
# replace data visibilities with model visibilities
visfile = fits.open(modelvisloc, mode='update')
visibilities = visfile[0].data
visheader = visfile[0].header
if visheader['NAXIS'] == 7:
# expect uu.dim = 4, hence real.ndim also 4
nfreq = visibilities['DATA'][0, 0, 0, 0, :, 0, 0].size
# to match uvutil.uvload() instead of fixing array size mismatch with miriad= in the function
if nfreq > 1:
visibilities['DATA'][:, 0, 0, :, :, :, 0] = real
visibilities['DATA'][:, 0, 0, :, :, :, 1] = imag
else:
try:
visibilities['DATA'][:, 0, 0, :, :, :, 0] = real
visibilities['DATA'][:, 0, 0, :, :, :, 1] = imag
except ValueError:
# mistmatach arise when uu.ndim is 3
# which would be the case if nfreq = 0
# vis, 0, 0, spw, chan(freq), pol, (real, imag, weights)
visibilities['DATA'][:, 0, 0, :, 0, :, 0] = real
visibilities['DATA'][:, 0, 0, :, 0, :, 1] = imag
elif visheader['NAXIS'] == 6:
visibilities['DATA'][:, 0, 0, :, :, 0] = real
visibilities['DATA'][:, 0, 0, :, :, 1] = imag
else:
print("Visibility dataset has >7 or <6 axes. I can't read this.")
#visibilities['DATA'][:, 0, 0, :, :, 2] = wgt
# replace the data visibilities with the model visibilities
visfile[0].data = visibilities
visfile.flush()
else:
from taskinit import tb
print("Writing visibility data to " + modelvisloc)
os.system('rm -rf ' + modelvisloc)
tb.open(visdataloc)
tb.copy(modelvisloc)
tb.close()
tb.open(modelvisloc, nomodify=False)
datashape = tb.getcol('DATA').shape
vis_complex = vis_complex.reshape(datashape)
tb.putcol('DATA', vis_complex)
tb.close()
def subtract_outliers(vis, outlier_coords, field='M33*', split_fields=True,
stokes='I', interactive=True, weighting='natural',
threshold='5mJy/beam', cell='3arcsec', cleanup=False,
datacolumn="CORRECTED", imsize=64, save_space=False,
masks=None, multiscale=[]):
'''
Subtract an outlier at the given coordinates. Splits out each field,
tries to image at that coordinate, then subracts of the model in the UV
plane. The fields are concatenated into a final ms.
'''
vis = vis.rstrip("/")
# Get some info from the image
tb.open(os.path.join(vis, 'FIELD'))
all_fields = tb.getcol('NAME')
tb.close()
regex = re.compile(field)
fields = [f for f in all_fields if re.match(regex, f)]
# If only one position given, convert to list for iteration.
if isinstance(outlier_coords, six.string_types):
outlier_coords = [outlier_coords]
if masks is not None:
if len(masks) != len(outlier_coords):
raise Warning("The number of specified masks must match the"
" number of coordinates.")
try:
os.mkdir('temp_files')
except OSError:
warnings.warn("temp_files already exists. "
"Going to remove all ms files from it.")
catch_fail(rmtables, tablenames='temp_files/*')
# Loop through the fields
for f in fields:
fieldvis = os.path.join('temp_files', f+".ms")
fieldimg = os.path.join('temp_files', f)
# Split the field off
catch_fail(split, vis=vis, outputvis=fieldvis, field=f,
datacolumn=datacolumn)
# Now image the data, keeping the phasecenter at the outlier
# Image each outlier at its phasecenter, then uvsub
for i, coord in enumerate(outlier_coords):
outfield_img = fieldimg + "_" + str(i)
if masks is not None:
mask = masks[i]
else:
mask = None
catch_fail(clean, vis=fieldvis, imagename=outfield_img, mode='mfs',
phasecenter=coord, niter=10000, usescratch=True,
interactive=interactive, cell='3arcsec',
imsize=imsize, threshold=threshold, weighting=weighting,
minpb=0.0, mask=mask, multiscale=multiscale)
# Subtract out the model from the imaging
catch_fail(uvsub, vis=fieldvis)
# Remove the individual images
if cleanup:
catch_fail(rmtables, tablenames=outfield_img+"*")
if save_space:
fieldvis_corronly = \
os.path.join('temp_files', f+"_corrected.ms")
catch_fail(split, vis=fieldvis, outputvis=fieldvis_corronly,
field=f, datacolumn="CORRECTED")
catch_fail(rmtables, tablenames=fieldvis)
# Now append the uvsub fields back together
individ_ms = glob.glob("temp_files/*.ms")
catch_fail(concat, vis=individ_ms,
concatvis=vis.rstrip(".ms")+"_outsub.ms",
respectname=True)
if cleanup:
catch_fail(rmtables, tablenames='temp_files/*')
def importeovsa_iter(filelist, timebin, width, visprefix, nocreatms, modelms,
doscaling, keep_nsclms, fileidx):
from taskinit import tb, casalog
filename = filelist[fileidx]
uv = aipy.miriad.UV(filename)
# try:
msname0 = list(filename.split('/')[-1])
msname = visprefix + ''.join(msname0) + '.ms'
uv.select('antennae', 0, 1, include=True)
uv.select('polarization', -5, -5, include=True)
times = []
uv.rewind()
for preamble, data in uv.all():
uvw, t, (i, j) = preamble
times.append(t)
uv.select('clear', -1, -1, include=True)
times = ipe.jd2mjds(np.asarray(times))
inttime = np.median((times - np.roll(times, 1))[1:]) / 60
time_steps = len(times)
durtim = int((times[-1] - times[0]) / 60 + inttime)
time0 = time.time()
if 'antlist' in uv.vartable:
ants = uv['antlist'].replace('\x00', '')
antlist = map(int, ants.split())
else:
antlist = [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16]
good_idx = np.where(uv['sfreq'] > 0)[0]
nf = len(good_idx)
npol = uv['npol']
nants = uv['nants']
source_id = uv['source'].replace('\x00', '')
sfreq = uv['sfreq'][good_idx]
sdf = uv['sdf'][good_idx]
ra, dec = uv['ra'], uv['dec']
nbl = nants * (nants - 1) / 2
bl2ord = ipe.bl_list2(nants)
npairs = nbl + nants
flag = np.ones((npol, nf, time_steps, npairs), dtype=bool)
out = np.zeros((npol, nf, time_steps, npairs),
dtype=np.complex64) # Cross-correlations
uvwarray = np.zeros((3, time_steps, npairs), dtype=np.float)
chan_band = ipe.get_band(sfreq=sfreq, sdf=sdf)
nband = len(chan_band)
uv.rewind()
l = -1
for preamble, data in uv.all():
uvw, t, (i0, j0) = preamble
i = antlist.index(i0 + 1)
j = antlist.index(j0 + 1)
if i > j:
# Reverse order of indices
j = antlist.index(i0 + 1)
i = antlist.index(j0 + 1)
# Assumes uv['pol'] is one of -5, -6, -7, -8
k = -5 - uv['pol']
l += 1
data = ma.masked_array(ma.masked_invalid(data), fill_value=0.0)
out[k, :, l / (npairs * npol), bl2ord[i0, j0]] = data.data
flag[k, :, l / (npairs * npol), bl2ord[i0, j0]] = data.mask
# if i != j:
if k == 3:
uvwarray[:, l / (npairs * npol),
bl2ord[i0, j0]] = -uvw * constants.speed_of_light / 1e9
nrows = time_steps * npairs
if doscaling:
out2 = out.copy()
for i0 in antlist:
for j0 in antlist:
if i0 < j0:
i, j = i0 - 1, j0 - 1
out2[:, :, :,
bl2ord[i, j]] = out[:, :, :, bl2ord[i, j]] / np.sqrt(
np.abs(out[:, :, :, bl2ord[i, i]]) *
np.abs(out[:, :, :, bl2ord[j, j]]))
out2 = out2.reshape(npol, nf, nrows)
out2[np.isnan(out2)] = 0
out2[np.isinf(out2)] = 0
# out2 = ma.masked_array(ma.masked_invalid(out2), fill_value=0.0)
out = out.reshape(npol, nf, nrows) * 1e4
flag = flag.reshape(npol, nf, nrows)
uvwarray = uvwarray.reshape(3, nrows)
uvwarray = np.tile(uvwarray, (1, nband))
sigma = np.ones((4, nrows), dtype=np.float) + 1
sigma = np.tile(sigma, (1, nband))
casalog.post('IDB File {0} is readed in --- {1:10.2f} seconds ---'.format(
filename, (time.time() - time0)))
if not nocreatms:
modelms = ipe.creatms(filename, visprefix)
os.system('mv {} {}'.format(modelms, msname))
else:
casalog.post('----------------------------------------')
casalog.post('copying standard MS to {0}'.format(
msname, (time.time() - time0)))
casalog.post('----------------------------------------')
os.system("rm -fr %s" % msname)
os.system("cp -r " + " %s" % modelms + " %s" % msname)
casalog.post(
'Standard MS is copied to {0} in --- {1:10.2f} seconds ---'.format(
msname, (time.time() - time0)))
tb.open(msname, nomodify=False)
casalog.post('----------------------------------------')
casalog.post("Updating the main table of" '%s' % msname)
casalog.post('----------------------------------------')
for l, cband in enumerate(chan_band):
time1 = time.time()
nchannels = len(cband['cidx'])
for row in range(nrows):
if not doscaling or keep_nsclms:
tb.putcell('DATA', (row + l * nrows),
out[:, cband['cidx'][0]:cband['cidx'][-1] + 1, row])
tb.putcell('FLAG', (row + l * nrows),
flag[:, cband['cidx'][0]:cband['cidx'][-1] + 1, row])
casalog.post(
'---spw {0:02d} is updated in --- {1:10.2f} seconds ---'.format(
(l + 1),
time.time() - time1))
tb.putcol('UVW', uvwarray)
tb.putcol('SIGMA', sigma)
tb.putcol('WEIGHT', 1.0 / sigma**2)
timearr = times
timearr = timearr.reshape(1, time_steps, 1)
timearr = np.tile(timearr, (nband, 1, npairs))
timearr = timearr.reshape(nband * npairs * time_steps)
tb.putcol('TIME', timearr)
tb.putcol('TIME_CENTROID', timearr)
scan_id = tb.getcol('SCAN_NUMBER')
scan_id *= 0
tb.putcol('SCAN_NUMBER', scan_id)
colnames = tb.colnames()
cols2rm = ["MODEL_DATA", "CORRECTED_DATA"]
for l in range(len(cols2rm)):
if cols2rm[l] in colnames:
tb.removecols(cols2rm[l])
tb.close()
casalog.post('----------------------------------------')
casalog.post("Updating the OBSERVATION table of" '%s' % msname)
casalog.post('----------------------------------------')
tb.open(msname + '/OBSERVATION', nomodify=False)
tb.putcol(
'TIME_RANGE',
np.asarray([times[0] - 0.5 * inttime,
times[-1] + 0.5 * inttime]).reshape(2, 1))
tb.putcol('OBSERVER', ['EOVSA team'])
tb.close()
casalog.post('----------------------------------------')
casalog.post("Updating the POINTING table of" '%s' % msname)
casalog.post('----------------------------------------')
tb.open(msname + '/POINTING', nomodify=False)
timearr = times.reshape(1, time_steps, 1)
timearr = np.tile(timearr, (nband, 1, nants))
timearr = timearr.reshape(nband * time_steps * nants)
tb.putcol('TIME', timearr)
tb.putcol('TIME_ORIGIN', timearr) # - 0.5 * delta_time)
direction = tb.getcol('DIRECTION')
direction[0, 0, :] = ra
direction[1, 0, :] = dec
tb.putcol('DIRECTION', direction)
target = tb.getcol('TARGET')
target[0, 0, :] = ra
target[1, 0, :] = dec
tb.putcol('TARGET', target)
tb.close()
casalog.post('----------------------------------------')
casalog.post("Updating the SOURCE table of" '%s' % msname)
casalog.post('----------------------------------------')
tb.open(msname + '/SOURCE', nomodify=False)
radec = tb.getcol('DIRECTION')
radec[0], radec[1] = ra, dec
tb.putcol('DIRECTION', radec)
name = np.array([source_id], dtype='|S{0}'.format(len(source_id) + 1))
tb.putcol('NAME', name)
tb.close()
casalog.post('----------------------------------------')
casalog.post("Updating the DATA_DESCRIPTION table of" '%s' % msname)
casalog.post('----------------------------------------')
tb.open(msname + '/DATA_DESCRIPTION/', nomodify=False)
pol_id = tb.getcol('POLARIZATION_ID')
pol_id *= 0
tb.putcol('POLARIZATION_ID', pol_id)
# spw_id = tb.getcol('SPECTRAL_WINDOW_ID')
# spw_id *= 0
# tb.putcol('SPECTRAL_WINDOW_ID', spw_id)
tb.close()
# casalog.post('----------------------------------------')
# casalog.post("Updating the POLARIZATION table of" '%s' % msname)
# casalog.post('----------------------------------------')
# tb.open(msname + '/POLARIZATION/', nomodify=False)
# tb.removerows(rownrs=np.arange(1, nband, dtype=int))
# tb.close()
casalog.post('----------------------------------------')
casalog.post("Updating the FIELD table of" '%s' % msname)
casalog.post('----------------------------------------')
tb.open(msname + '/FIELD/', nomodify=False)
delay_dir = tb.getcol('DELAY_DIR')
delay_dir[0], delay_dir[1] = ra, dec
tb.putcol('DELAY_DIR', delay_dir)
phase_dir = tb.getcol('PHASE_DIR')
phase_dir[0], phase_dir[1] = ra, dec
tb.putcol('PHASE_DIR', phase_dir)
reference_dir = tb.getcol('REFERENCE_DIR')
reference_dir[0], reference_dir[1] = ra, dec
tb.putcol('REFERENCE_DIR', reference_dir)
name = np.array([source_id], dtype='|S{0}'.format(len(source_id) + 1))
tb.putcol('NAME', name)
tb.close()
# FIELD: DELAY_DIR, PHASE_DIR, REFERENCE_DIR, NAME
# del out, flag, uvwarray, uv, timearr, sigma
# gc.collect() #
if doscaling:
if keep_nsclms:
msname_scl = visprefix + ''.join(msname0) + '_scl.ms'
os.system('cp -r {} {}'.format(msname, msname_scl))
else:
msname_scl = msname
tb.open(msname_scl, nomodify=False)
casalog.post('----------------------------------------')
casalog.post("Updating the main table of" '%s' % msname_scl)
casalog.post('----------------------------------------')
for l, cband in enumerate(chan_band):
time1 = time.time()
for row in range(nrows):
tb.putcell(
'DATA', (row + l * nrows),
out2[:, cband['cidx'][0]:cband['cidx'][-1] + 1, row])
casalog.post(
'---spw {0:02d} is updated in --- {1:10.2f} seconds ---'.
format((l + 1),
time.time() - time1))
tb.close()
if not (timebin == '0s' and width == 1):
msfile = msname + '.split'
if doscaling:
split(vis=msname_scl,
outputvis=msname_scl + '.split',
datacolumn='data',
timebin=timebin,
width=width,
keepflags=False)
os.system('rm -rf {}'.format(msname_scl))
msfile_scl = msname_scl + '.split'
if not (doscaling and not keep_nsclms):
split(vis=msname,
outputvis=msname + '.split',
datacolumn='data',
timebin=timebin,
width=width,
keepflags=False)
os.system('rm -rf {}'.format(msname))
else:
msfile = msname
if doscaling:
msfile_scl = msname_scl
casalog.post("finished in --- %s seconds ---" % (time.time() - time0))
if doscaling:
return [True, msfile, msfile_scl, durtim]
else:
return [True, msfile, durtim]
def uvload(visfile):
checker = visfile.find('uvfits')
if checker == -1:
uvfits = False
else:
uvfits = True
if uvfits:
visdata = fits.open(visfile)
visibilities = visdata[0].data
visheader = visdata[0].header
if visheader['NAXIS'] == 7:
# identify the channel frequency(ies):
visfreq = visdata[1].data
freq0 = visheader['CRVAL4']
dfreq = visheader['CDELT4']
cfreq = visheader['CRPIX4']
nvis = visibilities['DATA'][:, 0, 0, 0, 0, 0, 0].size
nspw = visibilities['DATA'][0, 0, 0, :, 0, 0, 0].size
nfreq = visibilities['DATA'][0, 0, 0, 0, :, 0, 0].size
npol = visibilities['DATA'][0, 0, 0, 0, 0, :, 0].size
if nfreq > 1:
uu = numpy.zeros([nvis, nspw, nfreq, npol])
vv = numpy.zeros([nvis, nspw, nfreq, npol])
ww = numpy.zeros([nvis, nspw, nfreq, npol])
else:
uu = numpy.zeros([nvis, nspw, npol])
vv = numpy.zeros([nvis, nspw, npol])
ww = numpy.zeros([nvis, nspw, npol])
#wgt = numpy.zeros([nvis, nspw, nfreq, npol])
for ispw in range(nspw):
if nspw > 1:
freqif = freq0 + visfreq['IF FREQ'][0][ispw]
else:
try:
freqif = freq0 + visfreq['IF FREQ'][0]
except:
freqif = freq0
#uu[:, ispw] = freqif * visibilities['UU']
#vv[:, ispw] = freqif * visibilities['VV']
for ipol in range(npol):
# then compute the spatial frequencies:
if nfreq > 1:
freq = (numpy.arange(nfreq) - cfreq + 1) * dfreq + freqif
freqvis = numpy.meshgrid(freq, visibilities['UU'])
uu[:, ispw, :, ipol] = freqvis[0] * freqvis[1]
freqvis = numpy.meshgrid(freq, visibilities['VV'])
vv[:, ispw, :, ipol] = freqvis[0] * freqvis[1]
freqvis = numpy.meshgrid(freq, visibilities['WW'])
ww[:, ispw, :, ipol] = freqvis[0] * freqvis[1]
else:
uu[:, ispw, ipol] = freqif * visibilities['UU']
vv[:, ispw, ipol] = freqif * visibilities['VV']
ww[:, ispw, ipol] = freqif * visibilities['WW']
if visheader['NAXIS'] == 6:
# identify the channel frequency(ies):
freq0 = visheader['CRVAL4']
dfreq = visheader['CDELT4']
cfreq = visheader['CRPIX4']
nvis = visibilities['DATA'][:, 0, 0, 0, 0, 0].size
nfreq = visibilities['DATA'][0, 0, 0, :, 0, 0].size
npol = visibilities['DATA'][0, 0, 0, 0, :, 0].size
if nfreq > 1:
uu = numpy.zeros([nvis, nfreq, npol])
vv = numpy.zeros([nvis, nfreq, npol])
ww = numpy.zeros([nvis, nfreq, npol])
else:
uu = numpy.zeros([nvis, npol])
vv = numpy.zeros([nvis, npol])
ww = numpy.zeros([nvis, npol])
#wgt = numpy.zeros([nvis, nspw, nfreq, npol])
freqif = freq0
#uu[:, ispw] = freqif * visibilities['UU']
#vv[:, ispw] = freqif * visibilities['VV']
for ipol in range(npol):
# then compute the spatial frequencies:
if nfreq > 1:
freq = (numpy.arange(nfreq) - cfreq + 1) * dfreq + freqif
freqvis = numpy.meshgrid(freq, visibilities['UU'])
uu[:, 0, :, ipol] = freqvis[0] * freqvis[1]
freqvis = numpy.meshgrid(freq, visibilities['VV'])
vv[:, 0, :, ipol] = freqvis[0] * freqvis[1]
freqvis = numpy.meshgrid(freq, visibilities['WW'])
ww[:, 0, :, ipol] = freqvis[0] * freqvis[1]
else:
uu[:, ipol] = freqif * visibilities['UU']
vv[:, ipol] = freqif * visibilities['VV']
ww[:, ipol] = freqif * visibilities['WW']
else:
from taskinit import tb
# read in the uvfits data
tb.open(visfile)
uvw = tb.getcol('UVW')
uvspw = tb.getcol('DATA_DESC_ID')
tb.close()
tb.open(visfile + '/SPECTRAL_WINDOW')
freq = tb.getcol('CHAN_FREQ')
tb.close()
tb.open(visfile + '/POLARIZATION')
polinfo = tb.getcol('NUM_CORR')
tb.close()
npol = polinfo[0]
nspw = len(freq[0])
for ispw in range(nspw):
ilam = 3e8 / freq[0][ispw]
indx_spw = uvspw == ispw
uvw[:, indx_spw] /= ilam
uu = []
vv = []
ww = []
for ipol in range(npol):
uu.append(uvw[0, :])
vv.append(uvw[1, :])
ww.append(uvw[2, :])
uu = numpy.array(uu)
vv = numpy.array(vv)
ww = numpy.array(ww)
if uu[:, 0].size == 1:
uu = uu.flatten()
vv = vv.flatten()
ww = ww.flatten()
return uu, vv, ww
def visload(visfile):
checker = visfile.find('uvfits')
if checker == -1:
uvfits = False
else:
uvfits = True
if uvfits:
visdata = fits.open(visfile)
# get the telescope name
visheader = visdata[0].header
#telescop = visheader['TELESCOP']
# if we are dealing with SMA data
if visheader['NAXIS'] == 6:
nfreq = visdata[0].data['DATA'][0, 0, 0, :, 0, 0].size
if nfreq > 1:
data_real = visdata[0].data['DATA'][:,0,0,:,:,0]
data_imag = visdata[0].data['DATA'][:,0,0,:,:,1]
data_wgt = visdata[0].data['DATA'][:,0,0,:,:,2]
else:
data_real = visdata[0].data['DATA'][:,0,0,0,:,0]
data_imag = visdata[0].data['DATA'][:,0,0,0,:,1]
data_wgt = visdata[0].data['DATA'][:,0,0,0,:,2]
# if we are dealing with ALMA or PdBI data
if visheader['NAXIS'] == 7:
nfreq = visdata[0].data['DATA'][0, 0, 0, 0, :, 0, 0].size
if nfreq > 1:
data_real = visdata[0].data['DATA'][:,0,0,:,:,:,0]
data_imag = visdata[0].data['DATA'][:,0,0,:,:,:,1]
data_wgt = visdata[0].data['DATA'][:,0,0,:,:,:,2]
else:
data_real = visdata[0].data['DATA'][:,0,0,:,0,:,0]
data_imag = visdata[0].data['DATA'][:,0,0,:,0,:,1]
data_wgt = visdata[0].data['DATA'][:,0,0,:,0,:,2]
data_complex = numpy.array(data_real) + \
1j * numpy.array(data_imag)
else:
from taskinit import tb
# read in the CASA MS
tb.open(visfile)
vis_complex = tb.getcol('DATA')
vis_weight = tb.getcol('WEIGHT')
tb.close()
#tb.open(visfile + '/POLARIZATION')
#polinfo = tb.getcol('NUM_CORR')
#npol = polinfo[0]
data_complex = vis_complex
data_wgt = vis_weight
wgtshape = data_wgt.shape
if len(wgtshape) == 2:
npol = wgtshape[0]
nrow = wgtshape[1]
wgtshape = (npol, 1, nrow)
data_wgt = data_wgt.reshape(wgtshape)
#data_complex = []
#data_wgt = []
#for ipol in range(npol):
# data_complex.append(vis_complex[ipol, 0, :])
# data_wgt.append(vis_weight[ipol, :])
#data_complex = numpy.array(data_complex)
#data_wgt = numpy.array(data_wgt)
return data_complex, data_wgt
from taskinit import tb
from tasks import split
'''
Identify the continuum and line SPWs and split into separate MSs and
directories
'''
logprint("Starting EVLA_pipe_mixed_setup_split.py",
logfileout='logs/mixed_setup_split.log')
# Figure out which are the lines and which are the continuum SPWs.
tb.open(ms_active + '/SPECTRAL_WINDOW')
bandwidths = tb.getcol('TOTAL_BANDWIDTH')
tb.close()
tb.open(ms_active + '/FIELD')
fields = tb.getcol('NAME')
tb.close()
# Define a threshold between expected bandwidths
# Going with 10 MHz
thresh_bw = 1.0e7
spws = np.arange(0, len(bandwidths))
line_spws = [str(i) for i in spws[np.where(bandwidths < thresh_bw)]]
cont_spws = [str(i) for i in spws[np.where(bandwidths > thresh_bw)]]
logprint("Line SPWS: {}".format(line_spws),
def compTables(referencetab, testtab, excludecols, tolerance=0.001, mode="percentage", startrow = 0, nrow = -1, rowincr = 1):
"""
compTables - compare two CASA tables
referencetab - the table which is assumed to be correct
testtab - the table which is to be compared to referencetab
excludecols - list of column names which are to be ignored
tolerance - permitted fractional difference (default 0.001 = 0.1 percent)
mode - comparison is made as "percentage", "absolute", "phaseabsdeg" (for complex numbers = difference of the phases in degrees)
"""
rval = True
tb2 = casac.table()
tb.open(referencetab)
cnames = tb.colnames()
tb2.open(testtab)
try:
for c in cnames:
if c in excludecols:
continue
print "\nTesting column " + c
a = 0
try:
a = tb.getcol(c,startrow=startrow,nrow=nrow,rowincr=rowincr)
except:
rval = False
print 'Error accessing column ', c, ' in table ', referencetab
print sys.exc_info()[0]
break
b = 0
try:
b = tb2.getcol(c,startrow=startrow,nrow=nrow,rowincr=rowincr)
except:
rval = False
print 'Error accessing column ', c, ' in table ', testtab
print sys.exc_info()[0]
break
if not (len(a)==len(b)):
print 'Column ',c,' has different length in tables ', referencetab, ' and ', testtab
print a
print b
rval = False
break
else:
differs = False
if not (a==b).all():
for i in range(0,len(a)):
if (isinstance(a[i],float)):
if ((mode=="percentage") and (abs(a[i]-b[i]) > tolerance*abs(a[i]))) or ((mode=="absolute") and (abs(a[i]-b[i]) > tolerance)):
print "Column " + c + " differs"
print "Row=" + str(i)
print "Reference file value: " + str(a[i])
print "Input file value: " + str(b[i])
if (mode=="percentage"):
print "Tolerance is {0}%; observed difference was {1} %".format (tolerance * 100, 100*abs(a[i]-b[i])/abs(a[i]))
else:
print "Absolute tolerance is {0}; observed difference: {1}".format (tolerance, (abs(a[i]-b[i])))
differs = True
rval = False
break
elif (isinstance(a[i],int) or isinstance(a[i],np.int32)):
if (abs(a[i]-b[i]) > 0):
print "Column " + c + " differs"
print "Row=" + str(i)
print "Reference file value: " + str(a[i])
print "Input file value: " + str(b[i])
if (mode=="percentage"):
print "tolerance in % should be " + str(100*abs(a[i]-b[i])/abs(a[i]))
else:
print "absolute tolerance should be " + str(abs(a[i]-b[i]))
differs = True
rval = False
break
elif (isinstance(a[i],str) or isinstance(a[i],np.bool_)):
if not (a[i]==b[i]):
print "Column " + c + " differs"
print "Row=" + str(i)
print "Reference file value: " + str(a[i])
print "Input file value: " + str(b[i])
if (mode=="percentage"):
print "tolerance in % should be " + str(100*abs(a[i]-b[i])/abs(a[i]))
else:
print "absolute tolerance should be " + str(abs(a[i]-b[i]))
differs = True
rval = False
break
elif (isinstance(a[i],list)) or (isinstance(a[i],np.ndarray)):
for j in range(0,len(a[i])):
if differs: break
if ((isinstance(a[i][j],float)) or (isinstance(a[i][j],int))):
if ((mode=="percentage") and (abs(a[i][j]-b[i][j]) > tolerance*abs(a[i][j]))) or ((mode=="absolute") and (abs(a[i][j]-b[i][j]) > tolerance)):
print "Column " + c + " differs"
print "(Row,Element)=(" + str(j) + "," + str(i) + ")"
print "Reference file value: " + str(a[i][j])
print "Input file value: " + str(b[i][j])
if (mode=="percentage"):
print "Tolerance in % should be " + str(100*abs(a[i][j]-b[i][j])/abs(a[i][j]))
else:
print "Absolute tolerance should be " + str(abs(a[i][j]-b[i][j]))
differs = True
rval = False
break
elif (isinstance(a[i][j],list)) or (isinstance(a[i][j],np.ndarray)):
for k in range(0,len(a[i][j])):
if differs: break
if ( ((mode=="percentage") and (abs(a[i][j][k]-b[i][j][k]) > tolerance*abs(a[i][j][k]))) \
or ((mode=="absolute") and (abs(a[i][j][k]-b[i][j][k]) > tolerance)) \
or ((mode=="phaseabsdeg") and (phasediffabsdeg(a[i][j][k],b[i][j][k])>tolerance)) \
):
print "Column " + c + " differs"
print "(Row,Channel,Corr)=(" + str(k) + "," + str(j) + "," + str(i) + ")"
print "Reference file value: " + str(a[i][j][k])
print "Input file value: " + str(b[i][j][k])
if (mode=="percentage"):
print "Tolerance in % should be " + str(100*abs(a[i][j][k]-b[i][j][k])/abs(a[i][j][k]))
elif (mode=="absolute"):
print "Absolute tolerance should be " + str(abs(a[i][j][k]-b[i][j][k]))
elif (mode=="phaseabsdeg"):
print "Phase tolerance in degrees should be " + str(phasediffabsdeg(a[i][j][k],b[i][j][k]))
else:
print "Unknown comparison mode: ",mode
differs = True
rval = False
break
else:
print "Unknown data type: ",type(a[i])
differs = True
rval = False
break
if not differs: print "Column " + c + " PASSED"
finally:
tb.close()
tb2.close()
return rval
def makeVis(config, miriad=False, idtag=''):
"""
Make simulated visibilities given a model image and observed visibilities.
Writes the visibilities to uvfits files.
"""
import uvmodel
import os
# get the uvfits files
visfile = config['UVData']
#----------------------------------------------------------------------
# Python version of UVMODEL
# "Observe" the lensed emission with the SMA and write to a new file
#----------------------------------------------------------------------
# Python version of UVMODEL's "replace" subroutine:
# is visfile a list of visibilities files?
if not type(visfile) is list:
visname, visext = os.path.splitext(visfile)
if miriad:
# We use miriad to do the imaging
tag = '.miriad'
DataMiriad = visname + tag
if not os.path.exists(DataMiriad):
print("Creating new miriad data file: " + DataMiriad)
os.system('rm -rf ' + DataMiriad)
command = 'fits op=uvin in=' + visfile + ' out=' + DataMiriad
os.system(command)
else:
# We use CASA to do the imaging
tag = '.ms'
# check to see if the CASA ms exists
try:
from taskinit import tb
tb.open(visname + tag)
tb.close()
print "Found an existing CASA ms file."
except RuntimeError:
print "No CASA ms file found, creating one from " + visname \
+ ".uvfits file."
from casa import importuvfits
infile = visname + '.uvfits'
outfile = visname + '.ms'
importuvfits(fitsfile=infile, vis=outfile)
visfile = visname + tag
SBmapLoc = 'LensedSBmap.fits'
modelvisfile = visname + '_model_' + idtag + tag
os.system('rm -rf ' + modelvisfile)
if miriad:
SBmapMiriad = 'LensedSBmap' + tag
os.system('rm -rf ' + SBmapMiriad)
command = 'fits op=xyin in=' + SBmapLoc + ' out=' \
+ SBmapMiriad
os.system(command)
command = 'uvmodel options=replace vis=' + visfile + \
' model=' + SBmapMiriad + ' out=' + modelvisfile
os.system(command + ' > uvmodeloutput.txt')
#command = 'cp ' + visfile + '/wflags ' + modelvisfile
#os.system(command)
else:
#print(visfile, modelvisfile)
uvmodel.replace(SBmapLoc, visfile, modelvisfile,
miriad=miriad)
#print(visfile, modelvisfile)
# Python version of UVMODEL's "subtract" subroutine:
modelvisfile = visname + '_residual_' + idtag + tag
os.system('rm -rf ' + modelvisfile)
if miriad:
SBmapMiriad = 'LensedSBmap' + tag
os.system('rm -rf ' + SBmapMiriad)
command = 'fits op=xyin in=' + SBmapLoc + ' out=' \
+ SBmapMiriad
os.system(command)
os.system('rm -rf ' + modelvisfile)
command = 'uvmodel options=subtract vis=' + visfile + \
' model=' + SBmapMiriad + ' out=' + modelvisfile
os.system(command)
#command = 'cp ' + visfile + '/wflags ' + modelvisfile
#os.system(command)
else:
#print(visfile, modelvisfile)
uvmodel.subtract(SBmapLoc, visfile, modelvisfile,
miriad=miriad)
else:
for i, ivisfile in enumerate(visfile):
visname, visext = os.path.splitext(ivisfile)
print(visname)
if miriad:
tag = '.uvfits'
else:
tag = '.ms'
# check to see if the CASA ms exists
try:
from taskinit import tb
tb.open(visname + tag)
tb.close()
print "Found an existing CASA ms file."
except RuntimeError:
print "No CASA ms file found, creating one from " + visname \
+ ".uvfits file."
from casa import importuvfits
infile = visname + '.uvfits'
outfile = visname + '.ms'
importuvfits(fitsfile=infile, vis=outfile)
SBmapLoc = 'LensedSBmap.fits'
if miriad:
SBmapMiriad = 'LensedSBmap.miriad'
os.system('rm -rf ' + SBmapMiriad)
command = 'fits op=xyin in=' + SBmapLoc + ' out=' \
+ SBmapMiriad
os.system(command)
ivisfile = visname + '.miriad'
modelivisfile = visname + '_model_' + idtag + '.miriad'
os.system('rm -rf ' + modelivisfile)
command = 'uvmodel options=replace vis=' + ivisfile + \
' model=' + SBmapMiriad + ' out=' + modelivisfile
os.system(command)
#command = 'cp ' + ivisfile + '/wflags ' + modelivisfile
#os.system(command)
else:
ivisfile = visname + tag
modelivisfile = visname + '_model_' + idtag + tag
os.system('rm -rf ' + modelivisfile)
uvmodel.replace(SBmapLoc, ivisfile, modelivisfile,
miriad=miriad)
# Python version of UVMODEL's "subtract" subroutine:
if miriad:
SBmapMiriad = 'LensedSBmap.miriad'
os.system('rm -rf ' + SBmapMiriad)
command = 'fits op=xyin in=' + SBmapLoc + ' out=' \
+ SBmapMiriad
os.system(command)
modelivisfile = visname + '_residual_' + idtag + '.miriad'
os.system('rm -rf ' + modelivisfile)
command = 'uvmodel options=subtract vis=' + ivisfile + \
' model=' + SBmapMiriad + ' out=' + modelivisfile
os.system(command)
#command = 'cp ' + ivisfile + '/wflags ' + modelivisfile
#os.system(command)
else:
modelivisfile = visname + '_residual_' + idtag + tag
os.system('rm -rf ' + modelivisfile)
uvmodel.subtract(SBmapLoc, ivisfile, modelivisfile,
miriad=miriad)
def uvload(visfile):
"""load in visibilities from a uv-file
Parameters
----------
visfile: string
visibility data filename, can be model or data
Returns
-------
uu: numpy.array
u visibilities
vv: numpy.array
v visibilities
ww: numpy.array
Note
----
08-14-2015:
Although a better solution to fix the array size mismatch problem that arises when calling `checkvis.uvmcmcfitVis` maybe something similar to the to-be-implemented function: uvmodel.add
which looks for nspw to shape model_complex
"""
checker = visfile.find('uvfits')
if checker == -1:
uvfits = False
else:
uvfits = True
if uvfits:
visdata = fits.open(visfile)
visibilities = visdata[0].data
visheader = visdata[0].header
if visheader['NAXIS'] == 7:
# identify the channel frequency(ies):
visfreq = visdata[1].data
freq0 = visheader['CRVAL4']
dfreq = visheader['CDELT4']
cfreq = visheader['CRPIX4']
nvis = visibilities['DATA'][:, 0, 0, 0, 0, 0, 0].size
nspw = visibilities['DATA'][0, 0, 0, :, 0, 0, 0].size
nfreq = visibilities['DATA'][0, 0, 0, 0, :, 0, 0].size
npol = visibilities['DATA'][0, 0, 0, 0, 0, :, 0].size
if nfreq > 1:
uu = numpy.zeros([nvis, nspw, nfreq, npol])
vv = numpy.zeros([nvis, nspw, nfreq, npol])
ww = numpy.zeros([nvis, nspw, nfreq, npol])
else:
# if miriad is True:
# uu = numpy.zeros([nvis, nspw, nfreq, npol])
# vv = numpy.zeros([nvis, nspw, nfreq, npol])
# ww = numpy.zeros([nvis, nspw, nfreq, npol])
# else:
# uu = numpy.zeros([nvis, nspw, npol])
# vv = numpy.zeros([nvis, nspw, npol])
# ww = numpy.zeros([nvis, nspw, npol])
uu = numpy.zeros([nvis, nspw, npol])
vv = numpy.zeros([nvis, nspw, npol])
ww = numpy.zeros([nvis, nspw, npol])
#wgt = numpy.zeros([nvis, nspw, nfreq, npol])
# get spw frequencies
# reference freq + offset
for ispw in range(nspw):
if nspw > 1:
freqif = freq0 + visfreq['IF FREQ'][0][ispw]
else:
try:
freqif = freq0 + visfreq['IF FREQ'][0]
except:
freqif = freq0
#uu[:, ispw] = freqif * visibilities['UU']
#vv[:, ispw] = freqif * visibilities['VV']
for ipol in range(npol):
# then compute the spatial frequencies:
if nfreq > 1:
freq = (numpy.arange(nfreq) - cfreq + 1) * \
dfreq + freqif
freqvis = numpy.meshgrid(freq, visibilities['UU'])
uu[:, ispw, :, ipol] = freqvis[0] * freqvis[1]
freqvis = numpy.meshgrid(freq, visibilities['VV'])
vv[:, ispw, :, ipol] = freqvis[0] * freqvis[1]
freqvis = numpy.meshgrid(freq, visibilities['WW'])
ww[:, ispw, :, ipol] = freqvis[0] * freqvis[1]
else:
# if miriad is True:
# freq = (numpy.arange(nfreq) - cfreq + 1) * dfreq + freqif
# freqvis = numpy.meshgrid(freq, visibilities['UU'])
# uu[:, ispw, :, ipol] = freqvis[0] * freqvis[1]
# freqvis = numpy.meshgrid(freq, visibilities['VV'])
# vv[:, ispw, :, ipol] = freqvis[0] * freqvis[1]
# freqvis = numpy.meshgrid(freq, visibilities['WW'])
# ww[:, ispw, :, ipol] = freqvis[0] * freqvis[1]
# else:
# uu[:, ispw, ipol] = freqif * visibilities['UU']
# vv[:, ispw, ipol] = freqif * visibilities['VV']
# ww[:, ispw, ipol] = freqif * visibilities['WW']
uu[:, ispw, ipol] = freqif * visibilities['UU']
vv[:, ispw, ipol] = freqif * visibilities['VV']
ww[:, ispw, ipol] = freqif * visibilities['WW']
if visheader['NAXIS'] == 6:
# identify the channel frequency(ies):
freq0 = visheader['CRVAL4']
dfreq = visheader['CDELT4']
cfreq = visheader['CRPIX4']
nvis = visibilities['DATA'][:, 0, 0, 0, 0, 0].size
nfreq = visibilities['DATA'][0, 0, 0, :, 0, 0].size
npol = visibilities['DATA'][0, 0, 0, 0, :, 0].size
if nfreq > 1:
uu = numpy.zeros([nvis, nfreq, npol])
vv = numpy.zeros([nvis, nfreq, npol])
ww = numpy.zeros([nvis, nfreq, npol])
else:
uu = numpy.zeros([nvis, npol])
vv = numpy.zeros([nvis, npol])
ww = numpy.zeros([nvis, npol])
#wgt = numpy.zeros([nvis, nspw, nfreq, npol])
freqif = freq0
#uu[:, ispw] = freqif * visibilities['UU']
#vv[:, ispw] = freqif * visibilities['VV']
for ipol in range(npol):
# then compute the spatial frequencies:
if nfreq > 1:
freq = (numpy.arange(nfreq) - cfreq + 1) * dfreq + freqif
freqvis = numpy.meshgrid(freq, visibilities['UU'])
uu[:, 0, :, ipol] = freqvis[0] * freqvis[1]
freqvis = numpy.meshgrid(freq, visibilities['VV'])
vv[:, 0, :, ipol] = freqvis[0] * freqvis[1]
freqvis = numpy.meshgrid(freq, visibilities['WW'])
ww[:, 0, :, ipol] = freqvis[0] * freqvis[1]
else:
uu[:, ipol] = freqif * visibilities['UU']
vv[:, ipol] = freqif * visibilities['VV']
ww[:, ipol] = freqif * visibilities['WW']
else:
from taskinit import tb
# read in the uvfits data
tb.open(visfile)
uvw = tb.getcol('UVW')
uvspw = tb.getcol('DATA_DESC_ID')
tb.close()
tb.open(visfile + '/SPECTRAL_WINDOW')
freq = tb.getcol('CHAN_FREQ')
tb.close()
tb.open(visfile + '/POLARIZATION')
polinfo = tb.getcol('NUM_CORR')
tb.close()
npol = polinfo[0]
nspw = len(freq[0])
for ispw in range(nspw):
ilam = 3e8 / freq[0][ispw]
indx_spw = uvspw == ispw
uvw[:, indx_spw] /= ilam
uu = []
vv = []
ww = []
for ipol in range(npol):
uu.append(uvw[0, :])
vv.append(uvw[1, :])
ww.append(uvw[2, :])
uu = numpy.array(uu)
vv = numpy.array(vv)
ww = numpy.array(ww)
if uu[:, 0].size == 1:
uu = uu.flatten()
vv = vv.flatten()
ww = ww.flatten()
return uu, vv, ww
pol = 'XX' # polarization to image, use XX for now
pbcor = True # correct for primary beam response?
grid_spacing = 2. * u.deg # spacing for plotting longitude and latitude grid in degrees
outdir = './image_series/' # Specify where you want to save the output fits files
imresfile = 'imres.npz' # File to write the imageing results summary
outimgpre = 'EO' # Something to add to the image name
#################################################################################
################### (OPTIONAL) DEFINE RESTORING BEAM SIZE ##################
### (Optional) the following is to define beam size ###
# read frequency information from visibility data
tb.open(vis + '/SPECTRAL_WINDOW')
reffreqs = tb.getcol('REF_FREQUENCY')
bdwds = tb.getcol('TOTAL_BANDWIDTH')
cfreqs = reffreqs + bdwds / 2.
tb.close()
spws = [str(s + 1) for s in range(30)]
sbeam = 30.
bmsz = [max(sbeam * cfreqs[1] / f, 6.) for f in cfreqs]
#################################################################################
############# This block checks if the script is being run on Virgo ############
import socket
if socket.gethostname() == 'ip-172-26-5-203.ec2.internal':
print(
'!!!!!!Caution!!!!!!: We detected that you are trying to run this computationally intensive script on Virgo.'
)
print(
'Please do not try to run this script when Virgo is busy (e.g., during the tutorial)'
)
def makeVis(config, miriad=False, idtag=''):
"""
Make simulated visibilities given a model image and observed visibilities.
Writes the visibilities to uvfits files.
"""
import uvmodel
import os
# get the uvfits files
visfile = config['UVData']
#----------------------------------------------------------------------
# Python version of UVMODEL
# "Observe" the lensed emission with the SMA and write to a new file
#----------------------------------------------------------------------
# Python version of UVMODEL's "replace" subroutine:
# is visfile a list of visibilities files?
if not type(visfile) is list:
visname, visext = os.path.splitext(visfile)
if miriad:
# We use miriad to do the imaging
tag = '.miriad'
DataMiriad = visname + tag
if not os.path.exists(DataMiriad):
print("Creating new miriad data file: " + DataMiriad)
os.system('rm -rf ' + DataMiriad)
command = 'fits op=uvin in=' + visfile + ' out=' + DataMiriad
os.system(command)
else:
# We use CASA to do the imaging
tag = '.ms'
# check to see if the CASA ms exists
try:
from taskinit import tb
tb.open(visname + tag)
tb.close()
print "Found an existing CASA ms file."
except RuntimeError:
print "No CASA ms file found, creating one from " + visname \
+ ".uvfits file."
from casa import importuvfits
infile = visname + '.uvfits'
outfile = visname + '.ms'
importuvfits(fitsfile=infile, vis=outfile)
visfile = visname + tag
SBmapLoc = 'LensedSBmap.fits'
modelvisfile = visname + '_model_' + idtag + tag
os.system('rm -rf ' + modelvisfile)
if miriad:
SBmapMiriad = 'LensedSBmap' + tag
os.system('rm -rf ' + SBmapMiriad)
command = 'fits op=xyin in=' + SBmapLoc + ' out=' \
+ SBmapMiriad
os.system(command)
command = 'uvmodel options=replace vis=' + visfile + \
' model=' + SBmapMiriad + ' out=' + modelvisfile
os.system(command + ' > uvmodeloutput.txt')
#command = 'cp ' + visfile + '/wflags ' + modelvisfile
#os.system(command)
else:
#print(visfile, modelvisfile)
uvmodel.replace(SBmapLoc, visfile, modelvisfile, miriad=miriad)
#print(visfile, modelvisfile)
# Python version of UVMODEL's "subtract" subroutine:
modelvisfile = visname + '_residual_' + idtag + tag
os.system('rm -rf ' + modelvisfile)
if miriad:
SBmapMiriad = 'LensedSBmap' + tag
os.system('rm -rf ' + SBmapMiriad)
command = 'fits op=xyin in=' + SBmapLoc + ' out=' \
+ SBmapMiriad
os.system(command)
os.system('rm -rf ' + modelvisfile)
command = 'uvmodel options=subtract vis=' + visfile + \
' model=' + SBmapMiriad + ' out=' + modelvisfile
os.system(command)
#command = 'cp ' + visfile + '/wflags ' + modelvisfile
#os.system(command)
else:
#print(visfile, modelvisfile)
uvmodel.subtract(SBmapLoc, visfile, modelvisfile, miriad=miriad)
else:
for i, ivisfile in enumerate(visfile):
visname, visext = os.path.splitext(ivisfile)
print(visname)
if miriad:
tag = '.uvfits'
else:
tag = '.ms'
# check to see if the CASA ms exists
try:
from taskinit import tb
tb.open(visname + tag)
tb.close()
print "Found an existing CASA ms file."
except RuntimeError:
print "No CASA ms file found, creating one from " + visname \
+ ".uvfits file."
from casa import importuvfits
infile = visname + '.uvfits'
outfile = visname + '.ms'
importuvfits(fitsfile=infile, vis=outfile)
SBmapLoc = 'LensedSBmap.fits'
if miriad:
SBmapMiriad = 'LensedSBmap.miriad'
os.system('rm -rf ' + SBmapMiriad)
command = 'fits op=xyin in=' + SBmapLoc + ' out=' \
+ SBmapMiriad
os.system(command)
ivisfile = visname + '.miriad'
modelivisfile = visname + '_model_' + idtag + '.miriad'
os.system('rm -rf ' + modelivisfile)
command = 'uvmodel options=replace vis=' + ivisfile + \
' model=' + SBmapMiriad + ' out=' + modelivisfile
os.system(command)
#command = 'cp ' + ivisfile + '/wflags ' + modelivisfile
#os.system(command)
else:
ivisfile = visname + tag
modelivisfile = visname + '_model_' + idtag + tag
os.system('rm -rf ' + modelivisfile)
uvmodel.replace(SBmapLoc,
ivisfile,
modelivisfile,
miriad=miriad)
# Python version of UVMODEL's "subtract" subroutine:
if miriad:
SBmapMiriad = 'LensedSBmap.miriad'
os.system('rm -rf ' + SBmapMiriad)
command = 'fits op=xyin in=' + SBmapLoc + ' out=' \
+ SBmapMiriad
os.system(command)
modelivisfile = visname + '_residual_' + idtag + '.miriad'
os.system('rm -rf ' + modelivisfile)
command = 'uvmodel options=subtract vis=' + ivisfile + \
' model=' + SBmapMiriad + ' out=' + modelivisfile
os.system(command)
#command = 'cp ' + ivisfile + '/wflags ' + modelivisfile
#os.system(command)
else:
modelivisfile = visname + '_residual_' + idtag + tag
os.system('rm -rf ' + modelivisfile)
uvmodel.subtract(SBmapLoc,
ivisfile,
modelivisfile,
miriad=miriad)