def concateovsa(msname,
msfiles,
visprefix='./',
doclearcal=True,
keep_orig_ms=False,
cols2rm=["MODEL_DATA", "CORRECTED_DATA"]):
concatvis = visprefix + msname
msfiles_ = []
for idx, ll in enumerate(msfiles):
if str(ll).endswith('/'):
msfiles[idx] = str(ll)[:-1]
if doclearcal:
print 'Warning: Corrected column in the input ms file will be cleared!!!'
for ll in msfiles:
clearcal(vis=str(ll), addmodel=True)
else:
try:
tmpdir = visprefix + '/tmp_ms/'
if not os.path.exists(tmpdir):
os.makedirs(tmpdir)
for ll in msfiles:
msfile_ = tmpdir + os.path.basename(str(ll))
msfiles_.append(msfile_)
split(vis=str(ll), outputvis=msfile_, datacolumn='corrected')
clearcal(vis=msfile_, addmodel=True)
except:
print 'Warning: Corrected column not found in the input ms file.'
msfiles_ = msfiles
if msfiles_:
concat(vis=msfiles_, concatvis=concatvis, timesort=True)
else:
concat(vis=msfiles, concatvis=concatvis, timesort=True)
# Change all observation ids to be the same (zero)
tb.open(concatvis + '/OBSERVATION', nomodify=False)
nobs = tb.nrows()
tim0 = tb.getcell('TIME_RANGE', 0)[0]
tim1 = tb.getcell('TIME_RANGE', nobs - 1)[1]
tb.removerows([i + 1 for i in range(nobs - 1)])
tb.putcell('TIME_RANGE', 0, [tim0, tim1])
tb.close()
tb.open(concatvis, nomodify=False)
obsid = tb.getcol('OBSERVATION_ID')
newobsid = np.zeros(len(obsid), dtype='int')
tb.putcol('OBSERVATION_ID', newobsid)
colnames = tb.colnames()
for l in range(len(cols2rm)):
if cols2rm[l] in colnames:
try:
tb.removecols(cols2rm[l])
except:
pass
tb.close()
if msfiles_ != [] and msfiles_ != msfiles:
for ll in msfiles_:
os.system('rm -rf {}'.format(ll))
if not keep_orig_ms:
for ll in msfiles:
os.system('rm -rf {}'.format(ll))
def concat_slftb(tb_in=[], tb_out=None):
if not tb_in:
print('tb_in not provided. Abort...')
if os.path.exists(tb_out):
os.system('rm -r {}'.format(tb_out))
os.system('cp -r {} {}'.format(tb_in[0], tb_out))
tbdata = {}
tb.open(tb_out)
cols = tb.colnames()
tb.close()
cols.remove('WEIGHT')
for col in cols:
tbdata[col] = []
for tbidx, ctb in enumerate(tb_in):
tb.open(ctb, nomodify=True)
tim0 = tb.getcol(cols[0])
if len(tim0) == 0:
continue
else:
for col in cols:
if tbidx == 1 and col in ['CPARAM', 'PARAMERR', 'FLAG', 'SNR']:
tbdata[col].append(tb.getcol(col)[::-1,...])
else:
tbdata[col].append(tb.getcol(col))
tb.close()
if len(tbdata[cols[0]]) == 0:
print('tables have no data. Return')
return -1
else:
for col in cols:
if col in ['CPARAM', 'PARAMERR', 'FLAG', 'SNR']:
tbdata[col] = np.concatenate(tbdata[col], axis=2)
else:
tbdata[col] = np.concatenate(tbdata[col])
tb.open(tb_out, nomodify=False)
nrows = tb.nrows()
nrows_new = len(tbdata[cols[0]])
tb.addrows(nrows_new - nrows)
for col in cols:
tb.putcol(col, tbdata[col])
tb.close()
return tb_out
def flagcaltboutliers(caltable, limit=[]):
import numpy as np
import numpy.ma as ma
# def removeOutliers(x, outlierConstant):
# a = np.array(x)
# idx, = np.where(np.diff(np.sort(datamag[0, 0, :]))>)
# upper_quartile = np.percentile(a, 80)
# lower_quartile = np.percentile(a, 20)
# IQR = (upper_quartile - lower_quartile) * outlierConstant
# quartileSet = (lower_quartile - IQR, upper_quartile + IQR)
# return ma.masked_outside(x, quartileSet[1], quartileSet[0])
if not os.path.exists(caltable): return 0
if isinstance(limit, list):
if len(limit) == 2:
tb.open(caltable, nomodify=False)
# subt = tb.query("ANTENNA1==1 && SPECTRAL_WINDOW_ID=10")
# data = subt.getcol('CPARAM')
# flag = subt.getcol('FLAG')
# spw = subt.getcol('SPECTRAL_WINDOW_ID')
# datamag = np.abs(data)
# mdatamag = ma.masked_outside(datamag, limit[0], limit[1])
# mask = np.logical_or(mdatamag.mask, flag)
# dataidx1 = datamag<limit[0]
# dataidx2 = datamag>limit[1]
# mdatamag = ma.masked_array(mdatamag, mask)
# mdatamag[0, 0, :] = removeOutliers(mdatamag[0, 0, :], 5)
# mdatamag[1, 0, :] = removeOutliers(mdatamag[1, 0, :], 5)
data = tb.getcol('CPARAM')
flag = tb.getcol('FLAG')
datamag = np.abs(data)
dataidx1 = datamag < limit[0]
dataidx2 = datamag > limit[1]
flag[dataidx1] = True
flag[dataidx2] = True
tb.putcol('FLAG', flag)
return 1
else:
print('limit must have two elements. Aborted!')
return 0
else:
print('limit must be a list. Aborted!')
def writeVis(vis_complex, visdataloc, modelvisloc, miriad=False):
if miriad:
os.system('rm -rf ' + modelvisloc)
cmd = 'cp ' + visdataloc + ' ' + modelvisloc
os.system(cmd)
# get the real and imaginary components
real = numpy.real(vis_complex)
imag = numpy.imag(vis_complex)
# replace data visibilities with model visibilities
visfile = fits.open(modelvisloc, mode='update')
visibilities = visfile[0].data
visheader = visfile[0].header
if visheader['NAXIS'] == 7:
visibilities['DATA'][:, 0, 0, :, :, :, 0] = real
visibilities['DATA'][:, 0, 0, :, :, :, 1] = imag
elif visheader['NAXIS'] == 6:
visibilities['DATA'][:, 0, 0, :, :, 0] = real
visibilities['DATA'][:, 0, 0, :, :, 1] = imag
else:
print("Visibility dataset has >7 or <6 axes. I can't read this.")
#visibilities['DATA'][:, 0, 0, :, :, 2] = wgt
# replace the data visibilities with the model visibilities
visfile[0].data = visibilities
visfile.flush()
else:
from taskinit import tb
print("Writing visibility data to " + modelvisloc)
os.system('rm -rf ' + modelvisloc)
tb.open(visdataloc)
tb.copy(modelvisloc)
tb.close()
tb.open(modelvisloc, nomodify=False)
datashape = tb.getcol('DATA').shape
vis_complex = vis_complex.reshape(datashape)
tb.putcol('DATA', vis_complex)
tb.close()
def writeVis(vis_complex, visdataloc, modelvisloc, miriad=False):
if miriad:
os.system('rm -rf ' + modelvisloc)
cmd = 'cp ' + visdataloc + ' ' + modelvisloc
os.system(cmd)
# get the real and imaginary components
real = numpy.real(vis_complex)
imag = numpy.imag(vis_complex)
# replace data visibilities with model visibilities
visfile = fits.open(modelvisloc, mode='update')
visibilities = visfile[0].data
visheader = visfile[0].header
if visheader['NAXIS'] == 7:
visibilities['DATA'][:, 0, 0, :, :, :, 0] = real
visibilities['DATA'][:, 0, 0, :, :, :, 1] = imag
elif visheader['NAXIS'] == 6:
visibilities['DATA'][:, 0, 0, :, :, 0] = real
visibilities['DATA'][:, 0, 0, :, :, 1] = imag
else:
print("Visibility dataset has >7 or <6 axes. I can't read this.")
#visibilities['DATA'][:, 0, 0, :, :, 2] = wgt
# replace the data visibilities with the model visibilities
visfile[0].data = visibilities
visfile.flush()
else:
from taskinit import tb
print("Writing visibility data to " + modelvisloc)
os.system('rm -rf ' + modelvisloc)
tb.open(visdataloc)
tb.copy(modelvisloc)
tb.close()
tb.open(modelvisloc, nomodify=False)
datashape = tb.getcol('DATA').shape
vis_complex = vis_complex.reshape(datashape)
tb.putcol('DATA', vis_complex)
tb.close()
def clearflagrow(msfile, mode='clear'):
'''
:param msfile:
:param mode: FLAG_ROW operation
default: 'clear': (default) clear the FLAG_ROW
'list': to list existing FLAG_ROW
:return:
'''
if mode == 'list':
tb.open(msfile, nomodify=True)
a = tb.getcol('FLAG_ROW')
nfrows = np.sum(a)
nrows = float(len(a))
print('{:0d} out of {:.0f} ({:.0f}%) rows are flagged in {}'.format(nfrows, nrows, nfrows / nrows * 100,
os.path.basename(msfile)))
elif mode == 'clear':
tb.open(msfile, nomodify=False)
a = tb.getcol('FLAG_ROW')
a[:] = False
tb.putcol('FLAG_ROW', a)
print('reset successfully')
tb.close()
def importeovsa_iter(filelist, timebin, width, visprefix, nocreatms, modelms,
doscaling, keep_nsclms, fileidx):
from taskinit import tb, casalog
filename = filelist[fileidx]
uv = aipy.miriad.UV(filename)
# try:
msname0 = list(filename.split('/')[-1])
msname = visprefix + ''.join(msname0) + '.ms'
uv.select('antennae', 0, 1, include=True)
uv.select('polarization', -5, -5, include=True)
times = []
uv.rewind()
for preamble, data in uv.all():
uvw, t, (i, j) = preamble
times.append(t)
uv.select('clear', -1, -1, include=True)
times = ipe.jd2mjds(np.asarray(times))
inttime = np.median((times - np.roll(times, 1))[1:]) / 60
time_steps = len(times)
durtim = int((times[-1] - times[0]) / 60 + inttime)
time0 = time.time()
if 'antlist' in uv.vartable:
ants = uv['antlist'].replace('\x00', '')
antlist = map(int, ants.split())
else:
antlist = [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16]
good_idx = np.where(uv['sfreq'] > 0)[0]
nf = len(good_idx)
npol = uv['npol']
nants = uv['nants']
source_id = uv['source'].replace('\x00', '')
sfreq = uv['sfreq'][good_idx]
sdf = uv['sdf'][good_idx]
ra, dec = uv['ra'], uv['dec']
nbl = nants * (nants - 1) / 2
bl2ord = ipe.bl_list2(nants)
npairs = nbl + nants
flag = np.ones((npol, nf, time_steps, npairs), dtype=bool)
out = np.zeros((npol, nf, time_steps, npairs),
dtype=np.complex64) # Cross-correlations
uvwarray = np.zeros((3, time_steps, npairs), dtype=np.float)
chan_band = ipe.get_band(sfreq=sfreq, sdf=sdf)
nband = len(chan_band)
uv.rewind()
l = -1
for preamble, data in uv.all():
uvw, t, (i0, j0) = preamble
i = antlist.index(i0 + 1)
j = antlist.index(j0 + 1)
if i > j:
# Reverse order of indices
j = antlist.index(i0 + 1)
i = antlist.index(j0 + 1)
# Assumes uv['pol'] is one of -5, -6, -7, -8
k = -5 - uv['pol']
l += 1
data = ma.masked_array(ma.masked_invalid(data), fill_value=0.0)
out[k, :, l / (npairs * npol), bl2ord[i0, j0]] = data.data
flag[k, :, l / (npairs * npol), bl2ord[i0, j0]] = data.mask
# if i != j:
if k == 3:
uvwarray[:, l / (npairs * npol),
bl2ord[i0, j0]] = -uvw * constants.speed_of_light / 1e9
nrows = time_steps * npairs
if doscaling:
out2 = out.copy()
for i0 in antlist:
for j0 in antlist:
if i0 < j0:
i, j = i0 - 1, j0 - 1
out2[:, :, :,
bl2ord[i, j]] = out[:, :, :, bl2ord[i, j]] / np.sqrt(
np.abs(out[:, :, :, bl2ord[i, i]]) *
np.abs(out[:, :, :, bl2ord[j, j]]))
out2 = out2.reshape(npol, nf, nrows)
out2[np.isnan(out2)] = 0
out2[np.isinf(out2)] = 0
# out2 = ma.masked_array(ma.masked_invalid(out2), fill_value=0.0)
out = out.reshape(npol, nf, nrows) * 1e4
flag = flag.reshape(npol, nf, nrows)
uvwarray = uvwarray.reshape(3, nrows)
uvwarray = np.tile(uvwarray, (1, nband))
sigma = np.ones((4, nrows), dtype=np.float) + 1
sigma = np.tile(sigma, (1, nband))
casalog.post('IDB File {0} is readed in --- {1:10.2f} seconds ---'.format(
filename, (time.time() - time0)))
if not nocreatms:
modelms = ipe.creatms(filename, visprefix)
os.system('mv {} {}'.format(modelms, msname))
else:
casalog.post('----------------------------------------')
casalog.post('copying standard MS to {0}'.format(
msname, (time.time() - time0)))
casalog.post('----------------------------------------')
os.system("rm -fr %s" % msname)
os.system("cp -r " + " %s" % modelms + " %s" % msname)
casalog.post(
'Standard MS is copied to {0} in --- {1:10.2f} seconds ---'.format(
msname, (time.time() - time0)))
tb.open(msname, nomodify=False)
casalog.post('----------------------------------------')
casalog.post("Updating the main table of" '%s' % msname)
casalog.post('----------------------------------------')
for l, cband in enumerate(chan_band):
time1 = time.time()
nchannels = len(cband['cidx'])
for row in range(nrows):
if not doscaling or keep_nsclms:
tb.putcell('DATA', (row + l * nrows),
out[:, cband['cidx'][0]:cband['cidx'][-1] + 1, row])
tb.putcell('FLAG', (row + l * nrows),
flag[:, cband['cidx'][0]:cband['cidx'][-1] + 1, row])
casalog.post(
'---spw {0:02d} is updated in --- {1:10.2f} seconds ---'.format(
(l + 1),
time.time() - time1))
tb.putcol('UVW', uvwarray)
tb.putcol('SIGMA', sigma)
tb.putcol('WEIGHT', 1.0 / sigma**2)
timearr = times
timearr = timearr.reshape(1, time_steps, 1)
timearr = np.tile(timearr, (nband, 1, npairs))
timearr = timearr.reshape(nband * npairs * time_steps)
tb.putcol('TIME', timearr)
tb.putcol('TIME_CENTROID', timearr)
scan_id = tb.getcol('SCAN_NUMBER')
scan_id *= 0
tb.putcol('SCAN_NUMBER', scan_id)
colnames = tb.colnames()
cols2rm = ["MODEL_DATA", "CORRECTED_DATA"]
for l in range(len(cols2rm)):
if cols2rm[l] in colnames:
tb.removecols(cols2rm[l])
tb.close()
casalog.post('----------------------------------------')
casalog.post("Updating the OBSERVATION table of" '%s' % msname)
casalog.post('----------------------------------------')
tb.open(msname + '/OBSERVATION', nomodify=False)
tb.putcol(
'TIME_RANGE',
np.asarray([times[0] - 0.5 * inttime,
times[-1] + 0.5 * inttime]).reshape(2, 1))
tb.putcol('OBSERVER', ['EOVSA team'])
tb.close()
casalog.post('----------------------------------------')
casalog.post("Updating the POINTING table of" '%s' % msname)
casalog.post('----------------------------------------')
tb.open(msname + '/POINTING', nomodify=False)
timearr = times.reshape(1, time_steps, 1)
timearr = np.tile(timearr, (nband, 1, nants))
timearr = timearr.reshape(nband * time_steps * nants)
tb.putcol('TIME', timearr)
tb.putcol('TIME_ORIGIN', timearr) # - 0.5 * delta_time)
direction = tb.getcol('DIRECTION')
direction[0, 0, :] = ra
direction[1, 0, :] = dec
tb.putcol('DIRECTION', direction)
target = tb.getcol('TARGET')
target[0, 0, :] = ra
target[1, 0, :] = dec
tb.putcol('TARGET', target)
tb.close()
casalog.post('----------------------------------------')
casalog.post("Updating the SOURCE table of" '%s' % msname)
casalog.post('----------------------------------------')
tb.open(msname + '/SOURCE', nomodify=False)
radec = tb.getcol('DIRECTION')
radec[0], radec[1] = ra, dec
tb.putcol('DIRECTION', radec)
name = np.array([source_id], dtype='|S{0}'.format(len(source_id) + 1))
tb.putcol('NAME', name)
tb.close()
casalog.post('----------------------------------------')
casalog.post("Updating the DATA_DESCRIPTION table of" '%s' % msname)
casalog.post('----------------------------------------')
tb.open(msname + '/DATA_DESCRIPTION/', nomodify=False)
pol_id = tb.getcol('POLARIZATION_ID')
pol_id *= 0
tb.putcol('POLARIZATION_ID', pol_id)
# spw_id = tb.getcol('SPECTRAL_WINDOW_ID')
# spw_id *= 0
# tb.putcol('SPECTRAL_WINDOW_ID', spw_id)
tb.close()
# casalog.post('----------------------------------------')
# casalog.post("Updating the POLARIZATION table of" '%s' % msname)
# casalog.post('----------------------------------------')
# tb.open(msname + '/POLARIZATION/', nomodify=False)
# tb.removerows(rownrs=np.arange(1, nband, dtype=int))
# tb.close()
casalog.post('----------------------------------------')
casalog.post("Updating the FIELD table of" '%s' % msname)
casalog.post('----------------------------------------')
tb.open(msname + '/FIELD/', nomodify=False)
delay_dir = tb.getcol('DELAY_DIR')
delay_dir[0], delay_dir[1] = ra, dec
tb.putcol('DELAY_DIR', delay_dir)
phase_dir = tb.getcol('PHASE_DIR')
phase_dir[0], phase_dir[1] = ra, dec
tb.putcol('PHASE_DIR', phase_dir)
reference_dir = tb.getcol('REFERENCE_DIR')
reference_dir[0], reference_dir[1] = ra, dec
tb.putcol('REFERENCE_DIR', reference_dir)
name = np.array([source_id], dtype='|S{0}'.format(len(source_id) + 1))
tb.putcol('NAME', name)
tb.close()
# FIELD: DELAY_DIR, PHASE_DIR, REFERENCE_DIR, NAME
# del out, flag, uvwarray, uv, timearr, sigma
# gc.collect() #
if doscaling:
if keep_nsclms:
msname_scl = visprefix + ''.join(msname0) + '_scl.ms'
os.system('cp -r {} {}'.format(msname, msname_scl))
else:
msname_scl = msname
tb.open(msname_scl, nomodify=False)
casalog.post('----------------------------------------')
casalog.post("Updating the main table of" '%s' % msname_scl)
casalog.post('----------------------------------------')
for l, cband in enumerate(chan_band):
time1 = time.time()
for row in range(nrows):
tb.putcell(
'DATA', (row + l * nrows),
out2[:, cband['cidx'][0]:cband['cidx'][-1] + 1, row])
casalog.post(
'---spw {0:02d} is updated in --- {1:10.2f} seconds ---'.
format((l + 1),
time.time() - time1))
tb.close()
if not (timebin == '0s' and width == 1):
msfile = msname + '.split'
if doscaling:
split(vis=msname_scl,
outputvis=msname_scl + '.split',
datacolumn='data',
timebin=timebin,
width=width,
keepflags=False)
os.system('rm -rf {}'.format(msname_scl))
msfile_scl = msname_scl + '.split'
if not (doscaling and not keep_nsclms):
split(vis=msname,
outputvis=msname + '.split',
datacolumn='data',
timebin=timebin,
width=width,
keepflags=False)
os.system('rm -rf {}'.format(msname))
else:
msfile = msname
if doscaling:
msfile_scl = msname_scl
casalog.post("finished in --- %s seconds ---" % (time.time() - time0))
if doscaling:
return [True, msfile, msfile_scl, durtim]
else:
return [True, msfile, durtim]
def concat(tb_in=[], tb_out=None):
if not tb_in:
print('tb_in not provided. Abort...')
if os.path.exists(tb_out):
os.system('rm -r ' + tb_out)
# os.system('cp -r '+tb_in[0]+' '+tb_out)
os.system('cp -r ' + tb_in[0] + ' ' + tb_out)
tb.open(tb_out + '/SPECTRAL_WINDOW', nomodify=True)
nspw = tb.nrows()
tb.close()
tim = []
fld = []
spw = []
ant1 = []
ant2 = []
intv = []
scan = []
obid = []
cpar = []
para = []
flag = []
snr = []
# wght=[]
for ctb in tb_in:
tb.open(ctb, nomodify=True)
cols = tb.colnames()
tim0 = tb.getcol(cols[0])
if len(tim0) == 0:
continue
else:
tim.append(tb.getcol(cols[0]))
fld.append(tb.getcol(cols[1]))
spw.append(tb.getcol(cols[2]))
ant1.append(tb.getcol(cols[3]))
ant2.append(tb.getcol(cols[4]))
intv.append(tb.getcol(cols[5]))
scan.append(tb.getcol(cols[6]))
obid.append(tb.getcol(cols[7]))
cpar.append(tb.getcol(cols[8]))
para.append(tb.getcol(cols[9]))
flag.append(tb.getcol(cols[10]))
snr.append(tb.getcol(cols[11]))
# wght.append(tb.getcol(cols[12]))
tb.close()
if len(tim) == 0:
print('tables have no data. Return')
return -1
else:
tim = np.concatenate(tim)
fld = np.concatenate(fld)
spw = np.concatenate(spw)
ant1 = np.concatenate(ant1)
ant2 = np.concatenate(ant2)
intv = np.concatenate(intv)
scan = np.concatenate(scan)
obid = np.concatenate(obid)
cpar = np.concatenate(cpar, axis=2)
para = np.concatenate(para, axis=2)
flag = np.concatenate(flag, axis=2)
snr = np.concatenate(snr, axis=2)
# wght=np.concatenate(wght)
tb.open(tb_out, nomodify=False)
nrows = tb.nrows()
nrows_new = len(tim)
tb.addrows(nrows_new - nrows)
tb.putcol(cols[0], tim)
tb.putcol(cols[1], fld)
tb.putcol(cols[2], spw)
tb.putcol(cols[3], ant1)
tb.putcol(cols[4], ant2)
tb.putcol(cols[5], intv)
tb.putcol(cols[6], scan)
tb.putcol(cols[7], obid)
tb.putcol(cols[8], cpar)
tb.putcol(cols[9], para)
tb.putcol(cols[10], flag)
tb.putcol(cols[11], snr)
tb.close()
return tb_out
def calibeovsa(vis=None,
caltype=None,
interp=None,
docalib=True,
doflag=True,
flagant=None,
doimage=False,
imagedir=None,
antenna=None,
timerange=None,
spw=None,
stokes=None,
doconcat=False,
msoutdir=None,
keep_orig_ms=True):
'''
:param vis: EOVSA visibility dataset(s) to be calibrated
:param caltype:
:param interp:
:param docalib:
:param qlookimage:
:param flagant:
:param stokes:
:param doconcat:
:return:
'''
if type(vis) == str:
vis = [vis]
for idx, f in enumerate(vis):
if f[-1] == '/':
vis[idx] = f[:-1]
for msfile in vis:
casalog.origin('calibeovsa')
if not caltype:
casalog.post(
"Caltype not provided. Perform reference phase calibration and daily phase calibration."
)
caltype = [
'refpha', 'phacal', 'fluxcal'
] ## use this line after the phacal is applied # caltype = ['refcal']
if not os.path.exists(msfile):
casalog.post("Input visibility does not exist. Aborting...")
continue
if msfile.endswith('/'):
msfile = msfile[:-1]
if not msfile[-3:] in ['.ms', '.MS']:
casalog.post(
"Invalid visibility. Please provide a proper visibility file ending with .ms"
)
# if not caltable:
# caltable=[os.path.basename(vis).replace('.ms','.'+c) for c in caltype]
# get band information
tb.open(msfile + '/SPECTRAL_WINDOW')
nspw = tb.nrows()
bdname = tb.getcol('NAME')
bd_nchan = tb.getcol('NUM_CHAN')
bd = [int(b[4:]) - 1 for b in bdname] # band index from 0 to 33
# nchans = tb.getcol('NUM_CHAN')
# reffreqs = tb.getcol('REF_FREQUENCY')
# cenfreqs = np.zeros((nspw))
tb.close()
tb.open(msfile + '/ANTENNA')
nant = tb.nrows()
antname = tb.getcol('NAME')
antlist = [str(ll) for ll in range(len(antname) - 1)]
antennas = ','.join(antlist)
tb.close()
# get time stamp, use the beginning of the file
tb.open(msfile + '/OBSERVATION')
trs = {'BegTime': [], 'EndTime': []}
for ll in range(tb.nrows()):
tim0, tim1 = Time(tb.getcell('TIME_RANGE', ll) / 24 / 3600,
format='mjd')
trs['BegTime'].append(tim0)
trs['EndTime'].append(tim1)
tb.close()
trs['BegTime'] = Time(trs['BegTime'])
trs['EndTime'] = Time(trs['EndTime'])
btime = np.min(trs['BegTime'])
etime = np.max(trs['EndTime'])
# ms.open(vis)
# summary = ms.summary()
# ms.close()
# btime = Time(summary['BeginTime'], format='mjd')
# etime = Time(summary['EndTime'], format='mjd')
## stop using ms.summary to avoid conflicts with importeovsa
t_mid = Time((btime.mjd + etime.mjd) / 2., format='mjd')
print "This scan observed from {} to {} UTC".format(
btime.iso, etime.iso)
gaintables = []
if ('refpha' in caltype) or ('refamp' in caltype) or ('refcal'
in caltype):
refcal = ra.sql2refcalX(btime)
pha = refcal['pha'] # shape is 15 (nant) x 2 (npol) x 34 (nband)
pha[np.where(refcal['flag'] == 1)] = 0.
amp = refcal['amp']
amp[np.where(refcal['flag'] == 1)] = 1.
t_ref = refcal['timestamp']
# find the start and end time of the local day when refcal is registered
try:
dhr = t_ref.LocalTime.utcoffset().total_seconds() / 60. / 60.
except:
dhr = -7.
bt = Time(np.fix(t_ref.mjd + dhr / 24.) - dhr / 24., format='mjd')
et = Time(bt.mjd + 1., format='mjd')
(yr, mon, day) = (bt.datetime.year, bt.datetime.month,
bt.datetime.day)
dirname = caltbdir + str(yr) + str(mon).zfill(2) + '/'
if not os.path.exists(dirname):
os.mkdir(dirname)
# check if there is any ROACH reboot between the reference calibration found and the current data
t_rbts = db.get_reboot(Time([t_ref, btime]))
if not t_rbts:
casalog.post(
"Reference calibration is derived from observation at " +
t_ref.iso)
print "Reference calibration is derived from observation at " + t_ref.iso
else:
casalog.post(
"Oh crap! Roach reboot detected between the reference calibration time "
+ t_ref.iso + ' and the current observation at ' +
btime.iso)
casalog.post("Aborting...")
print "Oh crap! Roach reboot detected between the reference calibration time " + t_ref.iso + ' and the current observation at ' + btime.iso
print "Aborting..."
para_pha = []
para_amp = []
calpha = np.zeros((nspw, 15, 2))
calamp = np.zeros((nspw, 15, 2))
for s in range(nspw):
for n in range(15):
for p in range(2):
calpha[s, n, p] = pha[n, p, bd[s]]
calamp[s, n, p] = amp[n, p, bd[s]]
para_pha.append(np.degrees(pha[n, p, bd[s]]))
para_amp.append(amp[n, p, bd[s]])
if 'fluxcal' in caltype:
calfac = pc.get_calfac(Time(t_mid.iso.split(' ')[0] + 'T23:59:59'))
t_bp = Time(calfac['timestamp'], format='lv')
if int(t_mid.mjd) == int(t_bp.mjd):
accalfac = calfac['accalfac'] # (ant x pol x freq)
# tpcalfac = calfac['tpcalfac'] # (ant x pol x freq)
caltb_autoamp = dirname + t_bp.isot[:-4].replace(
':', '').replace('-', '') + '.bandpass'
if not os.path.exists(caltb_autoamp):
bandpass(vis=msfile,
caltable=caltb_autoamp,
solint='inf',
refant='eo01',
minblperant=0,
minsnr=0,
bandtype='B',
docallib=False)
tb.open(caltb_autoamp, nomodify=False) # (ant x spw)
bd_chanidx = np.hstack([[0], bd_nchan.cumsum()])
for ll in range(nspw):
antfac = np.sqrt(
accalfac[:, :, bd_chanidx[ll]:bd_chanidx[ll + 1]])
# # antfac *= tpcalfac[:, :,bd_chanidx[ll]:bd_chanidx[ll + 1]]
antfac = np.moveaxis(antfac, 0, 2)
cparam = np.zeros((2, bd_nchan[ll], nant))
cparam[:, :, :-3] = 1.0 / antfac
tb.putcol('CPARAM', cparam + 0j, ll * nant, nant)
paramerr = tb.getcol('PARAMERR', ll * nant, nant)
paramerr = paramerr * 0
tb.putcol('PARAMERR', paramerr, ll * nant, nant)
bpflag = tb.getcol('FLAG', ll * nant, nant)
bpant1 = tb.getcol('ANTENNA1', ll * nant, nant)
bpflagidx, = np.where(bpant1 >= 13)
bpflag[:] = False
bpflag[:, :, bpflagidx] = True
tb.putcol('FLAG', bpflag, ll * nant, nant)
bpsnr = tb.getcol('SNR', ll * nant, nant)
bpsnr[:] = 100.0
bpsnr[:, :, bpflagidx] = 0.0
tb.putcol('SNR', bpsnr, ll * nant, nant)
tb.close()
msg_prompt = "Scaling calibration is derived for {}.".format(
msfile)
casalog.post(msg_prompt)
print msg_prompt
gaintables.append(caltb_autoamp)
else:
msg_prompt = "Caution: No TPCAL is available on {}. No scaling calibration is derived for {}.".format(
t_mid.datetime.strftime('%b %d, %Y'), msfile)
casalog.post(msg_prompt)
print msg_prompt
if ('refpha' in caltype) or ('refcal' in caltype):
# caltb_pha = os.path.basename(vis).replace('.ms', '.refpha')
# check if the calibration table already exists
caltb_pha = dirname + t_ref.isot[:-4].replace(':', '').replace(
'-', '') + '.refpha'
if not os.path.exists(caltb_pha):
gencal(vis=msfile,
caltable=caltb_pha,
caltype='ph',
antenna=antennas,
pol='X,Y',
spw='0~' + str(nspw - 1),
parameter=para_pha)
gaintables.append(caltb_pha)
if ('refamp' in caltype) or ('refcal' in caltype):
# caltb_amp = os.path.basename(vis).replace('.ms', '.refamp')
caltb_amp = dirname + t_ref.isot[:-4].replace(':', '').replace(
'-', '') + '.refamp'
if not os.path.exists(caltb_amp):
gencal(vis=msfile,
caltable=caltb_amp,
caltype='amp',
antenna=antennas,
pol='X,Y',
spw='0~' + str(nspw - 1),
parameter=para_amp)
gaintables.append(caltb_amp)
# calibration for the change of delay center between refcal time and beginning of scan -- hopefully none!
xml, buf = ch.read_calX(4, t=[t_ref, btime], verbose=False)
if buf:
dly_t2 = Time(stf.extract(buf[0], xml['Timestamp']), format='lv')
dlycen_ns2 = stf.extract(buf[0], xml['Delaycen_ns'])[:15]
xml, buf = ch.read_calX(4, t=t_ref)
dly_t1 = Time(stf.extract(buf, xml['Timestamp']), format='lv')
dlycen_ns1 = stf.extract(buf, xml['Delaycen_ns'])[:15]
dlycen_ns_diff = dlycen_ns2 - dlycen_ns1
for n in range(2):
dlycen_ns_diff[:, n] -= dlycen_ns_diff[0, n]
print 'Multi-band delay is derived from delay center difference at {} & {}'.format(
dly_t1.iso, dly_t2.iso)
# print '=====Delays relative to Ant 14====='
# for i, dl in enumerate(dlacen_ns_diff[:, 0] - dlacen_ns_diff[13, 0]):
# ant = antlist[i]
# print 'Ant eo{0:02d}: x {1:.2f} ns & y {2:.2f} ns'.format(int(ant) + 1, dl
# dlacen_ns_diff[i, 1] - dlacen_ns_diff[13, 1])
# caltb_mbd0 = os.path.basename(vis).replace('.ms', '.mbd0')
caltb_dlycen = dirname + dly_t2.isot[:-4].replace(':', '').replace(
'-', '') + '.dlycen'
if not os.path.exists(caltb_dlycen):
gencal(vis=msfile,
caltable=caltb_dlycen,
caltype='mbd',
pol='X,Y',
antenna=antennas,
parameter=dlycen_ns_diff.flatten().tolist())
gaintables.append(caltb_dlycen)
if 'phacal' in caltype:
phacals = np.array(
ra.sql2phacalX([bt, et], neat=True, verbose=False))
if not phacals.any() or len(phacals) == 0:
print "Found no phacal records in SQL database, will skip phase calibration"
else:
# first generate all phacal calibration tables if not already exist
t_phas = Time([phacal['t_pha'] for phacal in phacals])
# sort the array in ascending order by t_pha
sinds = t_phas.mjd.argsort()
t_phas = t_phas[sinds]
phacals = phacals[sinds]
caltbs_phambd = []
for i, phacal in enumerate(phacals):
# filter out phase cals with reference time stamp >30 min away from the provided refcal time
if (phacal['t_ref'].jd -
refcal['timestamp'].jd) > 30. / 1440.:
del phacals[i]
del t_phas[i]
continue
else:
t_pha = phacal['t_pha']
phambd_ns = phacal['pslope']
for n in range(2):
phambd_ns[:, n] -= phambd_ns[0, n]
# set all flagged values to be zero
phambd_ns[np.where(phacal['flag'] == 1)] = 0.
caltb_phambd = dirname + t_pha.isot[:-4].replace(
':', '').replace('-', '') + '.phambd'
caltbs_phambd.append(caltb_phambd)
if not os.path.exists(caltb_phambd):
gencal(vis=msfile,
caltable=caltb_phambd,
caltype='mbd',
pol='X,Y',
antenna=antennas,
parameter=phambd_ns.flatten().tolist())
# now decides which table to apply depending on the interpolation method ("neatest" or "linear")
if interp == 'nearest':
tbind = np.argmin(np.abs(t_phas.mjd - t_mid.mjd))
dt = np.min(np.abs(t_phas.mjd - t_mid.mjd)) * 24.
print "Selected nearest phase calibration table at " + t_phas[
tbind].iso
gaintables.append(caltbs_phambd[tbind])
if interp == 'linear':
# bphacal = ra.sql2phacalX(btime)
# ephacal = ra.sql2phacalX(etime,reverse=True)
bt_ind, = np.where(t_phas.mjd < btime.mjd)
et_ind, = np.where(t_phas.mjd > etime.mjd)
if len(bt_ind) == 0 and len(et_ind) == 0:
print "No phacal found before or after the ms data within the day of observation"
print "Skipping daily phase calibration"
elif len(bt_ind) > 0 and len(et_ind) == 0:
gaintables.append(caltbs_phambd[bt_ind[-1]])
elif len(bt_ind) == 0 and len(et_ind) > 0:
gaintables.append(caltbs_phambd[et_ind[0]])
elif len(bt_ind) > 0 and len(et_ind) > 0:
bphacal = phacals[bt_ind[-1]]
ephacal = phacals[et_ind[0]]
# generate a new table interpolating between two daily phase calibrations
t_pha_mean = Time(np.mean(
[bphacal['t_pha'].mjd, ephacal['t_pha'].mjd]),
format='mjd')
phambd_ns = (bphacal['pslope'] +
ephacal['pslope']) / 2.
for n in range(2):
phambd_ns[:, n] -= phambd_ns[0, n]
# set all flagged values to be zero
phambd_ns[np.where(bphacal['flag'] == 1)] = 0.
phambd_ns[np.where(ephacal['flag'] == 1)] = 0.
caltb_phambd_interp = dirname + t_pha_mean.isot[:-4].replace(
':', '').replace('-', '') + '.phambd'
if not os.path.exists(caltb_phambd_interp):
gencal(vis=msfile,
caltable=caltb_phambd_interp,
caltype='mbd',
pol='X,Y',
antenna=antennas,
parameter=phambd_ns.flatten().tolist())
print "Using phase calibration table interpolated between records at " + bphacal[
't_pha'].iso + ' and ' + ephacal['t_pha'].iso
gaintables.append(caltb_phambd_interp)
if docalib:
clearcal(msfile)
applycal(vis=msfile,
gaintable=gaintables,
applymode='calflag',
calwt=False)
# delete the interpolated phase calibration table
try:
caltb_phambd_interp
except:
pass
else:
if os.path.exists(caltb_phambd_interp):
shutil.rmtree(caltb_phambd_interp)
if doflag:
# flag zeros and NaNs
flagdata(vis=msfile, mode='clip', clipzeros=True)
if flagant:
try:
flagdata(vis=msfile, antenna=flagant)
except:
print "Something wrong with flagant. Abort..."
if doimage:
from matplotlib import pyplot as plt
from suncasa.utils import helioimage2fits as hf
from sunpy import map as smap
if not antenna:
antenna = '0~12'
if not stokes:
stokes = 'XX'
if not timerange:
timerange = ''
if not spw:
spw = '1~3'
if not imagedir:
imagedir = '.'
#(yr, mon, day) = (bt.datetime.year, bt.datetime.month, bt.datetime.day)
#dirname = imagedir + str(yr) + '/' + str(mon).zfill(2) + '/' + str(day).zfill(2) + '/'
#if not os.path.exists(dirname):
# os.makedirs(dirname)
bds = [spw]
nbd = len(bds)
imgs = []
for bd in bds:
if '~' in bd:
bdstr = bd.replace('~', '-')
else:
bdstr = str(bd).zfill(2)
imname = imagedir + '/' + os.path.basename(msfile).replace(
'.ms', '.bd' + bdstr)
print 'Cleaning image: ' + imname
try:
clean(vis=msfile,
imagename=imname,
antenna=antenna,
spw=bd,
timerange=timerange,
imsize=[512],
cell=['5.0arcsec'],
stokes=stokes,
niter=500)
except:
print 'clean not successfull for band ' + str(bd)
else:
imgs.append(imname + '.image')
junks = ['.flux', '.mask', '.model', '.psf', '.residual']
for junk in junks:
if os.path.exists(imname + junk):
shutil.rmtree(imname + junk)
tranges = [btime.iso + '~' + etime.iso] * nbd
fitsfiles = [img.replace('.image', '.fits') for img in imgs]
hf.imreg(vis=msfile,
timerange=tranges,
imagefile=imgs,
fitsfile=fitsfiles,
usephacenter=False)
plt.figure(figsize=(6, 6))
for i, fitsfile in enumerate(fitsfiles):
plt.subplot(1, nbd, i + 1)
eomap = smap.Map(fitsfile)
sz = eomap.data.shape
if len(sz) == 4:
eomap.data = eomap.data.reshape((sz[2], sz[3]))
eomap.plot_settings['cmap'] = plt.get_cmap('jet')
eomap.plot()
eomap.draw_limb()
eomap.draw_grid()
plt.show()
if doconcat:
if len(vis) > 1:
# from suncasa.eovsa import concateovsa as ce
from suncasa.tasks import concateovsa_cli as ce
if msoutdir is None:
msoutdir = './'
concatvis = os.path.basename(vis[0])
concatvis = msoutdir + '/' + concatvis.split('.')[0] + '_concat.ms'
ce.concateovsa(vis,
concatvis,
datacolumn='corrected',
keep_orig_ms=keep_orig_ms,
cols2rm="model,corrected")
return [concatvis]
else:
return vis
def concateovsa(vis, concatvis, datacolumn='corrected', keep_orig_ms=True, cols2rm="model,corrected", freqtol="", dirtol="", respectname=False,
timesort=True, copypointing=True, visweightscale=[], forcesingleephemfield=""):
if concatvis[-1] == os.path.sep:
concatvis = concatvis[:-1]
if os.path.sep not in concatvis:
visprefix = './'
else:
visprefix = os.path.dirname(concatvis) + os.path.sep
msfiles = vis
msfiles_ = []
for idx, ll in enumerate(msfiles):
if str(ll).endswith('/'):
msfiles[idx] = str(ll)[:-1]
datacolumn = datacolumn.lower()
if datacolumn == 'data':
print 'DATA columns will be concatenated.'
for ll in msfiles:
clearcal(vis=str(ll), addmodel=True)
elif datacolumn == 'corrected':
# try:
print 'CORRECTED columns will be concatenated.'
tmpdir = os.path.join(visprefix, 'tmp_ms') + os.path.sep
if not os.path.exists(tmpdir):
os.makedirs(tmpdir)
for ll in msfiles:
msfile_ = os.path.join(tmpdir, os.path.basename(str(ll)))
msfiles_.append(msfile_)
split(vis=str(ll), outputvis=msfile_, datacolumn='corrected')
clearcal(vis=msfile_, addmodel=True)
else:
raise ValueError('Please set datacolumn to be "data" or "corrected"!')
if msfiles_:
concat(vis=msfiles_, concatvis=concatvis, freqtol=freqtol, dirtol=dirtol, respectname=respectname, timesort=timesort,
copypointing=copypointing, visweightscale=visweightscale, forcesingleephemfield=forcesingleephemfield)
os.system('rm -rf {}'.format(tmpdir))
else:
concat(vis=msfiles, concatvis=concatvis, freqtol=freqtol, dirtol=dirtol, respectname=respectname, timesort=timesort,
copypointing=copypointing, visweightscale=visweightscale, forcesingleephemfield=forcesingleephemfield)
# Change all observation ids to be the same (zero)
tb.open(concatvis + '/OBSERVATION', nomodify=False)
nobs = tb.nrows()
tim0 = tb.getcell('TIME_RANGE', 0)[0]
tim1 = tb.getcell('TIME_RANGE', nobs - 1)[1]
tb.removerows([i + 1 for i in range(nobs - 1)])
tb.putcell('TIME_RANGE', 0, [tim0, tim1])
tb.close()
tb.open(concatvis + '/DATA_DESCRIPTION', nomodify=False)
nrows = tb.nrows()
pol_id = tb.getcol('POLARIZATION_ID')
tb.removerows(np.where(pol_id != 0)[0])
tb.close()
tb.open(concatvis, nomodify=False)
dd_id = tb.getcol('DATA_DESC_ID')
idx_dd_id, = np.where(dd_id >= nrows / 2)
dd_id[idx_dd_id] = dd_id[idx_dd_id] - nrows / 2
tb.putcol('DATA_DESC_ID', dd_id)
tb.close()
tb.open(concatvis + '/FIELD', nomodify=False)
nobs = tb.nrows()
tb.removerows([i + 1 for i in range(nobs - 1)])
tb.close()
tb.open(concatvis + '/SOURCE', nomodify=False)
nobs = tb.nrows()
tb.removerows([i + 1 for i in range(nobs - 1)])
tb.close()
tb.open(concatvis, nomodify=False)
obsid = tb.getcol('OBSERVATION_ID')
newobsid = np.zeros(len(obsid), dtype='int')
tb.putcol('OBSERVATION_ID', newobsid)
fldid = tb.getcol('FIELD_ID')
newfldid = np.zeros(len(fldid), dtype='int')
tb.putcol('FIELD_ID', newfldid)
colnames = tb.colnames()
cols2rm = cols2rm.upper()
cols2rm = cols2rm.split(',')
for l in range(len(cols2rm)):
col = cols2rm[l] + '_DATA'
if col in colnames:
try:
tb.removecols(col)
print 'Column {} removed.'.format(col)
except:
pass
tb.close()
if msfiles_ != [] and msfiles_ != msfiles:
for ll in msfiles_:
os.system('rm -rf {}'.format(ll))
if not keep_orig_ms:
for ll in msfiles:
os.system('rm -rf {}'.format(ll))
def pimporteovsa(idbfiles,
ncpu=8,
timebin=None,
width=None,
visprefix=None,
nocreatms=False,
doconcat=False,
modelms=''):
casalog.origin('pimporteovsa')
# # Initialize the helper class
# pdh = ParallelDataHelper("importeovsa", locals())
#
# # Validate input and output parameters
# try:
# pdh.setupIO()
# except Exception, instance:
# casalog.post('%s' % instance, 'ERROR')
# return False
# import glob
# idbfiles = glob.glob('IDB*_unrot')[:4]
# ncpu = 4
# timebin = None
# width = None
# visprefix = None
# nocreatms = False
# doconcat = False
# modelms = ''
# if not (type(ncpu) is int):
# casalog.post('ncpu should be an integer')
# ncpu = 8
if type(idbfiles) == Time:
filelist = ri.get_trange_files(idbfiles)
else:
# If input type is not Time, assume that it is the list of files to read
filelist = idbfiles
if type(filelist) == str:
filelist = [filelist]
for f in filelist:
if not os.path.exists(f):
casalog.post("Some files in filelist are invalid. Aborting...")
# return False
if not visprefix:
visprefix = './'
if not timebin:
timebin = '0s'
if not width:
width = 1
if not modelms:
if nocreatms:
filename = filelist[0]
modelms = ipe.creatms(filename, visprefix)
else:
if not os.path.exists(modelms):
if nocreatms:
filename = filelist[0]
modelms = ipe.creatms(filename, visprefix)
iterable = range(len(filelist))
imppart = partial(importeovsa_iter, filelist, timebin, width, visprefix,
nocreatms, modelms)
t0 = time.time()
casalog.post('Perform importeovsa in parallel ...')
pool = mp.Pool(ncpu)
res = pool.map(imppart, iterable)
pool.close()
pool.join()
t1 = time.time()
timelapse = t1 - t0
print 'It took %f secs to complete' % timelapse
results = pd.DataFrame({'succeeded': [], 'msfile': [], 'durtim': []})
for r in res:
results = results.append(
pd.DataFrame({
'succeeded': [r[0]],
'msfile': [r[1]],
'durtim': [r[2]]
}))
if doconcat:
msname = list(os.path.basename(filelist[0]))
concatvis = visprefix + ''.join(msname) + '-{:d}m.ms'.format(
int(results['durtim'].sum()))
msfile = results.loc[results['succeeded'] == True, :].msfile.tolist()
concat(vis=msfile, concatvis=concatvis, timesort=True)
# Change all observation ids to be the same (zero)
tb.open(concatvis + '/OBSERVATION', nomodify=False)
nobs = tb.nrows()
tb.removerows([i + 1 for i in range(nobs - 1)])
tb.close()
tb.open(concatvis, nomodify=False)
obsid = tb.getcol('OBSERVATION_ID')
newobsid = np.zeros(len(obsid), dtype='int')
tb.putcol('OBSERVATION_ID', newobsid)
tb.close()
for ll in msfile:
os.system('rm -rf {}'.format(ll))
return True
def importeovsa(idbfiles,
timebin=None,
width=None,
visprefix=None,
nocreatms=False,
doconcat=False,
modelms=''):
casalog.origin('importeovsa')
# # Initialize the helper class
# pdh = ParallelDataHelper("importeovsa", locals())
#
# # Validate input and output parameters
# try:
# pdh.setupIO()
# except Exception, instance:
# casalog.post('%s' % instance, 'ERROR')
# return False
if type(idbfiles) == Time:
filelist = ri.get_trange_files(idbfiles)
else:
# If input type is not Time, assume that it is the list of files to read
filelist = idbfiles
if type(filelist) == str:
filelist = [filelist]
for f in filelist:
if not os.path.exists(f):
casalog.post("Some files in filelist are invalid. Aborting...")
return False
if not visprefix:
visprefix = './'
if not timebin:
timebin = '0s'
if not width:
width = 1
if not modelms:
if nocreatms:
filename = filelist[0]
modelms = ipe.creatms(filename, visprefix)
else:
if not os.path.exists(modelms):
if nocreatms:
filename = filelist[0]
modelms = ipe.creatms(filename, visprefix)
msfile = []
time_concat = []
for filename in filelist:
uv = aipy.miriad.UV(filename)
# if filename.split('/')[-1][0:3] == 'UDB':
# uv_str = uv_hex_rm(uv)
# else:
# uv_str = uv
uv.select('antennae', 0, 1, include=True)
uv.select('polarization', -5, -5, include=True)
times = []
uv.rewind()
for preamble, data in uv.all():
uvw, t, (i, j) = preamble
times.append(t)
uv.select('clear', -1, -1, include=True)
times = ipe.jd2mjds(np.asarray(times))
inttime = np.median((times - np.roll(times, 1))[1:]) / 60
time_steps = len(times)
time_concat.append(int((times[-1] - times[0]) / 60 + inttime))
time0 = time.time()
if 'antlist' in uv.vartable:
ants = uv['antlist'].replace('\x00', '')
antlist = map(int, ants.split())
else:
antlist = [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16]
good_idx = np.where(uv['sfreq'] > 0)[0]
nf = len(good_idx)
npol = uv['npol']
nants = uv['nants']
source_id = uv['source'].replace('\x00', '')
sfreq = uv['sfreq'][good_idx]
sdf = uv['sdf'][good_idx]
ra, dec = uv['ra'], uv['dec']
nbl = nants * (nants - 1) / 2
bl2ord = ipe.bl_list2(nants)
npairs = nbl + nants
flag = np.ones((npol, nf, time_steps, npairs), dtype=bool)
out = np.zeros((npol, nf, time_steps, npairs),
dtype=np.complex64) # Cross-correlations
uvwarray = np.zeros((3, time_steps, npairs), dtype=np.float)
chan_band = ipe.get_band(sfreq=sfreq, sdf=sdf)
nband = len(chan_band)
uv.rewind()
l = -1
for preamble, data in uv.all():
uvw, t, (i0, j0) = preamble
i = antlist.index(i0 + 1)
j = antlist.index(j0 + 1)
if i > j:
# Reverse order of indices
j = antlist.index(i0 + 1)
i = antlist.index(j0 + 1)
# Assumes uv['pol'] is one of -5, -6, -7, -8
k = -5 - uv['pol']
l += 1
out[k, :, l / (npairs * npol), bl2ord[i0, j0]] = data.data
flag[k, :, l / (npairs * npol), bl2ord[i0, j0]] = data.mask
# if i != j:
if k == 3:
uvwarray[:, l / (npairs * npol),
bl2ord[i0,
j0]] = -uvw * constants.speed_of_light / 1e9
nrows = time_steps * npairs
out = out.reshape(npol, nf, nrows)
flag = flag.reshape(npol, nf, nrows)
uvwarray = uvwarray.reshape(3, nrows)
uvwarray = np.tile(uvwarray, (1, nband))
sigma = np.ones((4, nrows), dtype=np.float) + 1
sigma = np.tile(sigma, (1, nband))
casalog.post(
'IDB File {0} is readed in --- {1:10.2f} seconds ---'.format(
filename, (time.time() - time0)))
msname = list(filename.split('/')[-1])
msname = visprefix + ''.join(msname) + '.ms'
if not nocreatms:
modelms = ipe.creatms(filename, visprefix)
os.system('mv {} {}'.format(modelms, msname))
else:
casalog.post('----------------------------------------')
casalog.post('copying standard MS to {0}'.format(
msname, (time.time() - time0)))
casalog.post('----------------------------------------')
os.system("rm -fr %s" % msname)
os.system("cp -r " + " %s" % modelms + " %s" % msname)
casalog.post(
'Standard MS is copied to {0} in --- {1:10.2f} seconds ---'.
format(msname, (time.time() - time0)))
tb.open(msname, nomodify=False)
casalog.post('----------------------------------------')
casalog.post("Updating the main table of" '%s' % msname)
casalog.post('----------------------------------------')
for l, cband in enumerate(chan_band):
time1 = time.time()
nchannels = len(cband['cidx'])
for row in range(nrows):
tb.putcell('DATA', (row + l * nrows),
out[:, cband['cidx'][0]:cband['cidx'][-1] + 1, row])
tb.putcell(
'FLAG', (row + l * nrows),
flag[:, cband['cidx'][0]:cband['cidx'][-1] + 1, row])
casalog.post(
'---spw {0:02d} is updated in --- {1:10.2f} seconds ---'.
format((l + 1),
time.time() - time1))
tb.putcol('UVW', uvwarray)
tb.putcol('SIGMA', sigma)
tb.putcol('WEIGHT', 1.0 / sigma**2)
timearr = times
timearr = timearr.reshape(1, time_steps, 1)
timearr = np.tile(timearr, (nband, 1, npairs))
timearr = timearr.reshape(nband * npairs * time_steps)
tb.putcol('TIME', timearr)
tb.putcol('TIME_CENTROID', timearr)
scan_id = tb.getcol('SCAN_NUMBER')
scan_id *= 0
tb.putcol('SCAN_NUMBER', scan_id)
colnames = tb.colnames()
cols2rm = ["MODEL_DATA", "CORRECTED_DATA"]
for l in range(len(cols2rm)):
if cols2rm[l] in colnames:
tb.removecols(cols2rm[l])
tb.close()
casalog.post('----------------------------------------')
casalog.post("Updating the OBSERVATION table of" '%s' % msname)
casalog.post('----------------------------------------')
tb.open(msname + '/OBSERVATION', nomodify=False)
tb.putcol(
'TIME_RANGE',
np.asarray([times[0] - 0.5 * inttime,
times[-1] + 0.5 * inttime]).reshape(2, 1))
tb.putcol('OBSERVER', ['EOVSA team'])
tb.close()
casalog.post('----------------------------------------')
casalog.post("Updating the POINTING table of" '%s' % msname)
casalog.post('----------------------------------------')
tb.open(msname + '/POINTING', nomodify=False)
timearr = times.reshape(1, time_steps, 1)
timearr = np.tile(timearr, (nband, 1, nants))
timearr = timearr.reshape(nband * time_steps * nants)
tb.putcol('TIME', timearr)
tb.putcol('TIME_ORIGIN', timearr) # - 0.5 * delta_time)
direction = tb.getcol('DIRECTION')
direction[0, 0, :] = ra
direction[1, 0, :] = dec
tb.putcol('DIRECTION', direction)
target = tb.getcol('TARGET')
target[0, 0, :] = ra
target[1, 0, :] = dec
tb.putcol('TARGET', target)
tb.close()
casalog.post('----------------------------------------')
casalog.post("Updating the SOURCE table of" '%s' % msname)
casalog.post('----------------------------------------')
tb.open(msname + '/SOURCE', nomodify=False)
radec = tb.getcol('DIRECTION')
radec[0], radec[1] = ra, dec
tb.putcol('DIRECTION', radec)
name = np.array([source_id], dtype='|S{0}'.format(len(source_id) + 1))
tb.putcol('NAME', name)
tb.close()
casalog.post('----------------------------------------')
casalog.post("Updating the DATA_DESCRIPTION table of" '%s' % msname)
casalog.post('----------------------------------------')
tb.open(msname + '/DATA_DESCRIPTION/', nomodify=False)
pol_id = tb.getcol('POLARIZATION_ID')
pol_id *= 0
tb.putcol('POLARIZATION_ID', pol_id)
# spw_id = tb.getcol('SPECTRAL_WINDOW_ID')
# spw_id *= 0
# tb.putcol('SPECTRAL_WINDOW_ID', spw_id)
tb.close()
# pdb.set_trace()
casalog.post('----------------------------------------')
casalog.post("Updating the POLARIZATION table of" '%s' % msname)
casalog.post('----------------------------------------')
tb.open(msname + '/POLARIZATION/', nomodify=False)
print tb.nrows()
tb.removerows(rownrs=np.arange(1, nband, dtype=int))
tb.close()
casalog.post('----------------------------------------')
casalog.post("Updating the FIELD table of" '%s' % msname)
casalog.post('----------------------------------------')
tb.open(msname + '/FIELD/', nomodify=False)
delay_dir = tb.getcol('DELAY_DIR')
delay_dir[0], delay_dir[1] = ra, dec
tb.putcol('DELAY_DIR', delay_dir)
phase_dir = tb.getcol('PHASE_DIR')
phase_dir[0], phase_dir[1] = ra, dec
tb.putcol('PHASE_DIR', phase_dir)
reference_dir = tb.getcol('REFERENCE_DIR')
reference_dir[0], reference_dir[1] = ra, dec
tb.putcol('REFERENCE_DIR', reference_dir)
name = np.array([source_id], dtype='|S{0}'.format(len(source_id) + 1))
tb.putcol('NAME', name)
tb.close()
# FIELD: DELAY_DIR, PHASE_DIR, REFERENCE_DIR, NAME
del out, flag, uvwarray, uv, timearr, sigma
gc.collect() #
if not (timebin == '0s' and width == 1):
split(vis=msname,
outputvis=msname + '.split',
datacolumn='data',
timebin=timebin,
width=width,
keepflags=False)
os.system('rm -rf {}'.format(msname))
msfile.append(msname + '.split')
else:
msfile.append(msname)
casalog.post("finished in --- %s seconds ---" % (time.time() - time0))
if doconcat:
msname = list(filelist[0].split('/')[-1])
concatvis = visprefix + ''.join(msname) + '-{:d}m.ms'.format(
int(sum(time_concat)))
concat(vis=msfile, concatvis=concatvis, timesort=True)
# Change all observation ids to be the same (zero)
tb.open(concatvis + '/OBSERVATION', nomodify=False)
nobs = tb.nrows()
tb.removerows([i + 1 for i in range(nobs - 1)])
tb.close()
tb.open(concatvis, nomodify=False)
obsid = tb.getcol('OBSERVATION_ID')
newobsid = np.zeros(len(obsid), dtype='int')
tb.putcol('OBSERVATION_ID', newobsid)
tb.close()
for ll in msfile:
os.system('rm -rf {}'.format(ll))
return True
def writeVis(vis_complex, visdataloc, modelvisloc, miriad=False):
"""
Parameters
----------
vis_complex: numpy.array
visdataloc: string
uv-data filename
miriad: Boolean
True: not using CASA, just python data manipulation, hence can run in ipython,
False: use CASA taskinit module, must run inside CASA
Returns
-------
modelvisloc: string
uv-model filename
"""
if miriad:
os.system('rm -rf ' + modelvisloc)
cmd = 'cp ' + visdataloc + ' ' + modelvisloc
os.system(cmd)
# get the real and imaginary components
real = numpy.real(vis_complex)
imag = numpy.imag(vis_complex)
# replace data visibilities with model visibilities
visfile = fits.open(modelvisloc, mode='update')
visibilities = visfile[0].data
visheader = visfile[0].header
if visheader['NAXIS'] == 7:
# expect uu.dim = 4, hence real.ndim also 4
nfreq = visibilities['DATA'][0, 0, 0, 0, :, 0, 0].size
# to match uvutil.uvload() instead of fixing array size mismatch with miriad= in the function
if nfreq > 1:
visibilities['DATA'][:, 0, 0, :, :, :, 0] = real
visibilities['DATA'][:, 0, 0, :, :, :, 1] = imag
else:
try:
visibilities['DATA'][:, 0, 0, :, :, :, 0] = real
visibilities['DATA'][:, 0, 0, :, :, :, 1] = imag
except ValueError:
# mistmatach arise when uu.ndim is 3
# which would be the case if nfreq = 0
# vis, 0, 0, spw, chan(freq), pol, (real, imag, weights)
visibilities['DATA'][:, 0, 0, :, 0, :, 0] = real
visibilities['DATA'][:, 0, 0, :, 0, :, 1] = imag
elif visheader['NAXIS'] == 6:
visibilities['DATA'][:, 0, 0, :, :, 0] = real
visibilities['DATA'][:, 0, 0, :, :, 1] = imag
else:
print("Visibility dataset has >7 or <6 axes. I can't read this.")
#visibilities['DATA'][:, 0, 0, :, :, 2] = wgt
# replace the data visibilities with the model visibilities
visfile[0].data = visibilities
visfile.flush()
else:
from taskinit import tb
print("Writing visibility data to " + modelvisloc)
os.system('rm -rf ' + modelvisloc)
tb.open(visdataloc)
tb.copy(modelvisloc)
tb.close()
tb.open(modelvisloc, nomodify=False)
datashape = tb.getcol('DATA').shape
vis_complex = vis_complex.reshape(datashape)
tb.putcol('DATA', vis_complex)
tb.close()
