def concateovsa(msname,
msfiles,
visprefix='./',
doclearcal=True,
keep_orig_ms=False,
cols2rm=["MODEL_DATA", "CORRECTED_DATA"]):
concatvis = visprefix + msname
msfiles_ = []
for idx, ll in enumerate(msfiles):
if str(ll).endswith('/'):
msfiles[idx] = str(ll)[:-1]
if doclearcal:
print 'Warning: Corrected column in the input ms file will be cleared!!!'
for ll in msfiles:
clearcal(vis=str(ll), addmodel=True)
else:
try:
tmpdir = visprefix + '/tmp_ms/'
if not os.path.exists(tmpdir):
os.makedirs(tmpdir)
for ll in msfiles:
msfile_ = tmpdir + os.path.basename(str(ll))
msfiles_.append(msfile_)
split(vis=str(ll), outputvis=msfile_, datacolumn='corrected')
clearcal(vis=msfile_, addmodel=True)
except:
print 'Warning: Corrected column not found in the input ms file.'
msfiles_ = msfiles
if msfiles_:
concat(vis=msfiles_, concatvis=concatvis, timesort=True)
else:
concat(vis=msfiles, concatvis=concatvis, timesort=True)
# Change all observation ids to be the same (zero)
tb.open(concatvis + '/OBSERVATION', nomodify=False)
nobs = tb.nrows()
tim0 = tb.getcell('TIME_RANGE', 0)[0]
tim1 = tb.getcell('TIME_RANGE', nobs - 1)[1]
tb.removerows([i + 1 for i in range(nobs - 1)])
tb.putcell('TIME_RANGE', 0, [tim0, tim1])
tb.close()
tb.open(concatvis, nomodify=False)
obsid = tb.getcol('OBSERVATION_ID')
newobsid = np.zeros(len(obsid), dtype='int')
tb.putcol('OBSERVATION_ID', newobsid)
colnames = tb.colnames()
for l in range(len(cols2rm)):
if cols2rm[l] in colnames:
try:
tb.removecols(cols2rm[l])
except:
pass
tb.close()
if msfiles_ != [] and msfiles_ != msfiles:
for ll in msfiles_:
os.system('rm -rf {}'.format(ll))
if not keep_orig_ms:
for ll in msfiles:
os.system('rm -rf {}'.format(ll))
def getColDesc(table, colname):
'''Get the description of a column in a table
table --> name of table or MS
colname --> column name
Return a dictionary with the column description'''
coldesc = {}
try:
try:
tb.open(table)
tcols = tb.colnames()
if tcols.__contains__(colname):
coldesc = tb.getcoldesc(colname)
except:
pass
finally:
tb.close()
return coldesc
def concat_slftb(tb_in=[], tb_out=None):
if not tb_in:
print('tb_in not provided. Abort...')
if os.path.exists(tb_out):
os.system('rm -r {}'.format(tb_out))
os.system('cp -r {} {}'.format(tb_in[0], tb_out))
tbdata = {}
tb.open(tb_out)
cols = tb.colnames()
tb.close()
cols.remove('WEIGHT')
for col in cols:
tbdata[col] = []
for tbidx, ctb in enumerate(tb_in):
tb.open(ctb, nomodify=True)
tim0 = tb.getcol(cols[0])
if len(tim0) == 0:
continue
else:
for col in cols:
if tbidx == 1 and col in ['CPARAM', 'PARAMERR', 'FLAG', 'SNR']:
tbdata[col].append(tb.getcol(col)[::-1,...])
else:
tbdata[col].append(tb.getcol(col))
tb.close()
if len(tbdata[cols[0]]) == 0:
print('tables have no data. Return')
return -1
else:
for col in cols:
if col in ['CPARAM', 'PARAMERR', 'FLAG', 'SNR']:
tbdata[col] = np.concatenate(tbdata[col], axis=2)
else:
tbdata[col] = np.concatenate(tbdata[col])
tb.open(tb_out, nomodify=False)
nrows = tb.nrows()
nrows_new = len(tbdata[cols[0]])
tb.addrows(nrows_new - nrows)
for col in cols:
tb.putcol(col, tbdata[col])
tb.close()
return tb_out
def getColDesc(table, colname):
'''Get the description of a column in a table
table --> name of table or MS
colname --> column name
Return a dictionary with the column description'''
coldesc = {}
try:
try:
tb.open(table)
tcols = tb.colnames()
if tcols.__contains__(colname):
coldesc = tb.getcoldesc(colname)
except:
pass
finally:
tb.close()
return coldesc
def compTables(referencetab,
testtab,
excludecols,
tolerance=0.001,
mode="percentage",
startrow=0,
nrow=-1,
rowincr=1):
"""
compTables - compare two CASA tables
referencetab - the table which is assumed to be correct
testtab - the table which is to be compared to referencetab
excludecols - list of column names which are to be ignored
tolerance - permitted fractional difference (default 0.001 = 0.1 percent)
mode - comparison is made as "percentage", "absolute", "phaseabsdeg" (for complex numbers = difference of the phases in degrees)
"""
rval = True
tb2 = casac.table()
tb.open(referencetab)
cnames = tb.colnames()
tb2.open(testtab)
try:
for c in cnames:
if c in excludecols:
continue
print "\nTesting column " + c
a = 0
try:
a = tb.getcol(c, startrow=startrow, nrow=nrow, rowincr=rowincr)
except:
rval = False
print 'Error accessing column ', c, ' in table ', referencetab
print sys.exc_info()[0]
break
b = 0
try:
b = tb2.getcol(c,
startrow=startrow,
nrow=nrow,
rowincr=rowincr)
except:
rval = False
print 'Error accessing column ', c, ' in table ', testtab
print sys.exc_info()[0]
break
if not (len(a) == len(b)):
print 'Column ', c, ' has different length in tables ', referencetab, ' and ', testtab
print a
print b
rval = False
break
else:
differs = False
if not (a == b).all():
for i in range(0, len(a)):
if (isinstance(a[i], float)):
if ((mode == "percentage") and
(abs(a[i] - b[i]) > tolerance * abs(a[i]))
) or ((mode == "absolute") and
(abs(a[i] - b[i]) > tolerance)):
print "Column " + c + " differs"
print "Row=" + str(i)
print "Reference file value: " + str(a[i])
print "Input file value: " + str(b[i])
if (mode == "percentage"):
print "Tolerance is {0}%; observed difference was {1} %".format(
tolerance * 100,
100 * abs(a[i] - b[i]) / abs(a[i]))
else:
print "Absolute tolerance is {0}; observed difference: {1}".format(
tolerance, (abs(a[i] - b[i])))
differs = True
rval = False
break
elif (isinstance(a[i], int)
or isinstance(a[i], np.int32)):
if (abs(a[i] - b[i]) > 0):
print "Column " + c + " differs"
print "Row=" + str(i)
print "Reference file value: " + str(a[i])
print "Input file value: " + str(b[i])
if (mode == "percentage"):
print "tolerance in % should be " + str(
100 * abs(a[i] - b[i]) / abs(a[i]))
else:
print "absolute tolerance should be " + str(
abs(a[i] - b[i]))
differs = True
rval = False
break
elif (isinstance(a[i], str)
or isinstance(a[i], np.bool_)):
if not (a[i] == b[i]):
print "Column " + c + " differs"
print "Row=" + str(i)
print "Reference file value: " + str(a[i])
print "Input file value: " + str(b[i])
if (mode == "percentage"):
print "tolerance in % should be " + str(
100 * abs(a[i] - b[i]) / abs(a[i]))
else:
print "absolute tolerance should be " + str(
abs(a[i] - b[i]))
differs = True
rval = False
break
elif (isinstance(a[i], list)) or (isinstance(
a[i], np.ndarray)):
for j in range(0, len(a[i])):
if differs: break
if ((isinstance(a[i][j], float))
or (isinstance(a[i][j], int))):
if ((mode == "percentage") and
(abs(a[i][j] - b[i][j]) >
tolerance * abs(a[i][j]))) or (
(mode == "absolute") and
(abs(a[i][j] - b[i][j]) >
tolerance)):
print "Column " + c + " differs"
print "(Row,Element)=(" + str(
j) + "," + str(i) + ")"
print "Reference file value: " + str(
a[i][j])
print "Input file value: " + str(
b[i][j])
if (mode == "percentage"):
print "Tolerance in % should be " + str(
100 * abs(a[i][j] - b[i][j]) /
abs(a[i][j]))
else:
print "Absolute tolerance should be " + str(
abs(a[i][j] - b[i][j]))
differs = True
rval = False
break
elif (isinstance(a[i][j],
list)) or (isinstance(
a[i][j], np.ndarray)):
it = xrange(0, len(a[i][j]))
if mode == "percentage":
diff = np.abs(
np.subtract(a[i][j], b[i][j])
) > tolerance * np.abs(a[i][j])
it = np.where(diff)[0]
elif (mode == "absolute"):
diff = np.abs(
np.subtract(a[i][j],
b[i][j])) > tolerance
it = np.where(diff)[0]
for k in it:
if differs: break
if ( ((mode=="percentage") and (abs(a[i][j][k]-b[i][j][k]) > tolerance*abs(a[i][j][k]))) \
or ((mode=="absolute") and (abs(a[i][j][k]-b[i][j][k]) > tolerance)) \
or ((mode=="phaseabsdeg") and (phasediffabsdeg(a[i][j][k],b[i][j][k])>tolerance)) \
):
print "Column " + c + " differs"
print "(Row,Channel,Corr)=(" + str(
k) + "," + str(j) + "," + str(
i) + ")"
print "Reference file value: " + str(
a[i][j][k])
print "Input file value: " + str(
b[i][j][k])
if (mode == "percentage"):
print "Tolerance in % should be " + str(
100 * abs(a[i][j][k] -
b[i][j][k]) /
abs(a[i][j][k]))
elif (mode == "absolute"):
print "Absolute tolerance should be " + str(
abs(a[i][j][k] -
b[i][j][k]))
elif (mode == "phaseabsdeg"):
print "Phase tolerance in degrees should be " + str(
phasediffabsdeg(
a[i][j][k],
b[i][j][k]))
else:
print "Unknown comparison mode: ", mode
differs = True
rval = False
break
else:
print "Unknown data type: ", type(a[i])
differs = True
rval = False
break
if not differs: print "Column " + c + " PASSED"
finally:
tb.close()
tb2.close()
return rval
def compVarColTables(referencetab, testtab, varcol, tolerance=0.):
'''Compare a variable column of two tables.
referencetab --> a reference table
testtab --> a table to verify
varcol --> the name of a variable column (str)
Returns True or False.
'''
retval = True
tb2 = casac.table()
tb.open(referencetab)
cnames = tb.colnames()
tb2.open(testtab)
col = varcol
if tb.isvarcol(col) and tb2.isvarcol(col):
try:
# First check
if tb.nrows() != tb2.nrows():
print 'Length of %s differ from %s, %s!=%s' % (
referencetab, testtab, len(rk), len(tk))
retval = False
else:
for therow in xrange(tb.nrows()):
rdata = tb.getcell(col, therow)
tdata = tb2.getcell(col, therow)
# if not (rdata==tdata).all():
if not rdata.all() == tdata.all():
if (tolerance > 0.):
differs = False
for j in range(0, len(rdata)):
### if (type(rdata[j])==float or type(rdata[j])==int):
if ((isinstance(rdata[j], float))
or (isinstance(rdata[j], int))):
if (abs(rdata[j] - tdata[j]) > tolerance *
abs(rdata[j] + tdata[j])):
# print 'Column ', col,' differs in tables ', referencetab, ' and ', testtab
# print therow, j
# print rdata[j]
# print tdata[j]
differs = True
### elif (type(rdata[j])==list or type(rdata[j])==np.ndarray):
elif (isinstance(rdata[j],
list)) or (isinstance(
rdata[j], np.ndarray)):
for k in range(0, len(rdata[j])):
if (abs(rdata[j][k] - tdata[j][k]) >
tolerance * abs(rdata[j][k] +
tdata[j][k])):
# print 'Column ', col,' differs in tables ', referencetab, ' and ', testtab
# print therow, j, k
# print rdata[j][k]
# print tdata[j][k]
differs = True
if differs:
print 'ERROR: Column %s of %s and %s do not agree within tolerance %s' % (
col, referencetab, testtab, tolerance)
retval = False
break
else:
print 'ERROR: Column %s of %s and %s do not agree.' % (
col, referencetab, testtab)
print 'ERROR: First row to differ is row=%s' % therow
retval = False
break
finally:
tb.close()
tb2.close()
else:
print 'Columns are not varcolumns.'
retval = False
if retval:
print 'Column %s of %s and %s agree' % (col, referencetab, testtab)
return retval
def importeovsa_iter(filelist, timebin, width, visprefix, nocreatms, modelms,
doscaling, keep_nsclms, fileidx):
from taskinit import tb, casalog
filename = filelist[fileidx]
uv = aipy.miriad.UV(filename)
# try:
msname0 = list(filename.split('/')[-1])
msname = visprefix + ''.join(msname0) + '.ms'
uv.select('antennae', 0, 1, include=True)
uv.select('polarization', -5, -5, include=True)
times = []
uv.rewind()
for preamble, data in uv.all():
uvw, t, (i, j) = preamble
times.append(t)
uv.select('clear', -1, -1, include=True)
times = ipe.jd2mjds(np.asarray(times))
inttime = np.median((times - np.roll(times, 1))[1:]) / 60
time_steps = len(times)
durtim = int((times[-1] - times[0]) / 60 + inttime)
time0 = time.time()
if 'antlist' in uv.vartable:
ants = uv['antlist'].replace('\x00', '')
antlist = map(int, ants.split())
else:
antlist = [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16]
good_idx = np.where(uv['sfreq'] > 0)[0]
nf = len(good_idx)
npol = uv['npol']
nants = uv['nants']
source_id = uv['source'].replace('\x00', '')
sfreq = uv['sfreq'][good_idx]
sdf = uv['sdf'][good_idx]
ra, dec = uv['ra'], uv['dec']
nbl = nants * (nants - 1) / 2
bl2ord = ipe.bl_list2(nants)
npairs = nbl + nants
flag = np.ones((npol, nf, time_steps, npairs), dtype=bool)
out = np.zeros((npol, nf, time_steps, npairs),
dtype=np.complex64) # Cross-correlations
uvwarray = np.zeros((3, time_steps, npairs), dtype=np.float)
chan_band = ipe.get_band(sfreq=sfreq, sdf=sdf)
nband = len(chan_band)
uv.rewind()
l = -1
for preamble, data in uv.all():
uvw, t, (i0, j0) = preamble
i = antlist.index(i0 + 1)
j = antlist.index(j0 + 1)
if i > j:
# Reverse order of indices
j = antlist.index(i0 + 1)
i = antlist.index(j0 + 1)
# Assumes uv['pol'] is one of -5, -6, -7, -8
k = -5 - uv['pol']
l += 1
data = ma.masked_array(ma.masked_invalid(data), fill_value=0.0)
out[k, :, l / (npairs * npol), bl2ord[i0, j0]] = data.data
flag[k, :, l / (npairs * npol), bl2ord[i0, j0]] = data.mask
# if i != j:
if k == 3:
uvwarray[:, l / (npairs * npol),
bl2ord[i0, j0]] = -uvw * constants.speed_of_light / 1e9
nrows = time_steps * npairs
if doscaling:
out2 = out.copy()
for i0 in antlist:
for j0 in antlist:
if i0 < j0:
i, j = i0 - 1, j0 - 1
out2[:, :, :,
bl2ord[i, j]] = out[:, :, :, bl2ord[i, j]] / np.sqrt(
np.abs(out[:, :, :, bl2ord[i, i]]) *
np.abs(out[:, :, :, bl2ord[j, j]]))
out2 = out2.reshape(npol, nf, nrows)
out2[np.isnan(out2)] = 0
out2[np.isinf(out2)] = 0
# out2 = ma.masked_array(ma.masked_invalid(out2), fill_value=0.0)
out = out.reshape(npol, nf, nrows) * 1e4
flag = flag.reshape(npol, nf, nrows)
uvwarray = uvwarray.reshape(3, nrows)
uvwarray = np.tile(uvwarray, (1, nband))
sigma = np.ones((4, nrows), dtype=np.float) + 1
sigma = np.tile(sigma, (1, nband))
casalog.post('IDB File {0} is readed in --- {1:10.2f} seconds ---'.format(
filename, (time.time() - time0)))
if not nocreatms:
modelms = ipe.creatms(filename, visprefix)
os.system('mv {} {}'.format(modelms, msname))
else:
casalog.post('----------------------------------------')
casalog.post('copying standard MS to {0}'.format(
msname, (time.time() - time0)))
casalog.post('----------------------------------------')
os.system("rm -fr %s" % msname)
os.system("cp -r " + " %s" % modelms + " %s" % msname)
casalog.post(
'Standard MS is copied to {0} in --- {1:10.2f} seconds ---'.format(
msname, (time.time() - time0)))
tb.open(msname, nomodify=False)
casalog.post('----------------------------------------')
casalog.post("Updating the main table of" '%s' % msname)
casalog.post('----------------------------------------')
for l, cband in enumerate(chan_band):
time1 = time.time()
nchannels = len(cband['cidx'])
for row in range(nrows):
if not doscaling or keep_nsclms:
tb.putcell('DATA', (row + l * nrows),
out[:, cband['cidx'][0]:cband['cidx'][-1] + 1, row])
tb.putcell('FLAG', (row + l * nrows),
flag[:, cband['cidx'][0]:cband['cidx'][-1] + 1, row])
casalog.post(
'---spw {0:02d} is updated in --- {1:10.2f} seconds ---'.format(
(l + 1),
time.time() - time1))
tb.putcol('UVW', uvwarray)
tb.putcol('SIGMA', sigma)
tb.putcol('WEIGHT', 1.0 / sigma**2)
timearr = times
timearr = timearr.reshape(1, time_steps, 1)
timearr = np.tile(timearr, (nband, 1, npairs))
timearr = timearr.reshape(nband * npairs * time_steps)
tb.putcol('TIME', timearr)
tb.putcol('TIME_CENTROID', timearr)
scan_id = tb.getcol('SCAN_NUMBER')
scan_id *= 0
tb.putcol('SCAN_NUMBER', scan_id)
colnames = tb.colnames()
cols2rm = ["MODEL_DATA", "CORRECTED_DATA"]
for l in range(len(cols2rm)):
if cols2rm[l] in colnames:
tb.removecols(cols2rm[l])
tb.close()
casalog.post('----------------------------------------')
casalog.post("Updating the OBSERVATION table of" '%s' % msname)
casalog.post('----------------------------------------')
tb.open(msname + '/OBSERVATION', nomodify=False)
tb.putcol(
'TIME_RANGE',
np.asarray([times[0] - 0.5 * inttime,
times[-1] + 0.5 * inttime]).reshape(2, 1))
tb.putcol('OBSERVER', ['EOVSA team'])
tb.close()
casalog.post('----------------------------------------')
casalog.post("Updating the POINTING table of" '%s' % msname)
casalog.post('----------------------------------------')
tb.open(msname + '/POINTING', nomodify=False)
timearr = times.reshape(1, time_steps, 1)
timearr = np.tile(timearr, (nband, 1, nants))
timearr = timearr.reshape(nband * time_steps * nants)
tb.putcol('TIME', timearr)
tb.putcol('TIME_ORIGIN', timearr) # - 0.5 * delta_time)
direction = tb.getcol('DIRECTION')
direction[0, 0, :] = ra
direction[1, 0, :] = dec
tb.putcol('DIRECTION', direction)
target = tb.getcol('TARGET')
target[0, 0, :] = ra
target[1, 0, :] = dec
tb.putcol('TARGET', target)
tb.close()
casalog.post('----------------------------------------')
casalog.post("Updating the SOURCE table of" '%s' % msname)
casalog.post('----------------------------------------')
tb.open(msname + '/SOURCE', nomodify=False)
radec = tb.getcol('DIRECTION')
radec[0], radec[1] = ra, dec
tb.putcol('DIRECTION', radec)
name = np.array([source_id], dtype='|S{0}'.format(len(source_id) + 1))
tb.putcol('NAME', name)
tb.close()
casalog.post('----------------------------------------')
casalog.post("Updating the DATA_DESCRIPTION table of" '%s' % msname)
casalog.post('----------------------------------------')
tb.open(msname + '/DATA_DESCRIPTION/', nomodify=False)
pol_id = tb.getcol('POLARIZATION_ID')
pol_id *= 0
tb.putcol('POLARIZATION_ID', pol_id)
# spw_id = tb.getcol('SPECTRAL_WINDOW_ID')
# spw_id *= 0
# tb.putcol('SPECTRAL_WINDOW_ID', spw_id)
tb.close()
# casalog.post('----------------------------------------')
# casalog.post("Updating the POLARIZATION table of" '%s' % msname)
# casalog.post('----------------------------------------')
# tb.open(msname + '/POLARIZATION/', nomodify=False)
# tb.removerows(rownrs=np.arange(1, nband, dtype=int))
# tb.close()
casalog.post('----------------------------------------')
casalog.post("Updating the FIELD table of" '%s' % msname)
casalog.post('----------------------------------------')
tb.open(msname + '/FIELD/', nomodify=False)
delay_dir = tb.getcol('DELAY_DIR')
delay_dir[0], delay_dir[1] = ra, dec
tb.putcol('DELAY_DIR', delay_dir)
phase_dir = tb.getcol('PHASE_DIR')
phase_dir[0], phase_dir[1] = ra, dec
tb.putcol('PHASE_DIR', phase_dir)
reference_dir = tb.getcol('REFERENCE_DIR')
reference_dir[0], reference_dir[1] = ra, dec
tb.putcol('REFERENCE_DIR', reference_dir)
name = np.array([source_id], dtype='|S{0}'.format(len(source_id) + 1))
tb.putcol('NAME', name)
tb.close()
# FIELD: DELAY_DIR, PHASE_DIR, REFERENCE_DIR, NAME
# del out, flag, uvwarray, uv, timearr, sigma
# gc.collect() #
if doscaling:
if keep_nsclms:
msname_scl = visprefix + ''.join(msname0) + '_scl.ms'
os.system('cp -r {} {}'.format(msname, msname_scl))
else:
msname_scl = msname
tb.open(msname_scl, nomodify=False)
casalog.post('----------------------------------------')
casalog.post("Updating the main table of" '%s' % msname_scl)
casalog.post('----------------------------------------')
for l, cband in enumerate(chan_band):
time1 = time.time()
for row in range(nrows):
tb.putcell(
'DATA', (row + l * nrows),
out2[:, cband['cidx'][0]:cband['cidx'][-1] + 1, row])
casalog.post(
'---spw {0:02d} is updated in --- {1:10.2f} seconds ---'.
format((l + 1),
time.time() - time1))
tb.close()
if not (timebin == '0s' and width == 1):
msfile = msname + '.split'
if doscaling:
split(vis=msname_scl,
outputvis=msname_scl + '.split',
datacolumn='data',
timebin=timebin,
width=width,
keepflags=False)
os.system('rm -rf {}'.format(msname_scl))
msfile_scl = msname_scl + '.split'
if not (doscaling and not keep_nsclms):
split(vis=msname,
outputvis=msname + '.split',
datacolumn='data',
timebin=timebin,
width=width,
keepflags=False)
os.system('rm -rf {}'.format(msname))
else:
msfile = msname
if doscaling:
msfile_scl = msname_scl
casalog.post("finished in --- %s seconds ---" % (time.time() - time0))
if doscaling:
return [True, msfile, msfile_scl, durtim]
else:
return [True, msfile, durtim]
def concat(tb_in=[], tb_out=None):
if not tb_in:
print('tb_in not provided. Abort...')
if os.path.exists(tb_out):
os.system('rm -r ' + tb_out)
# os.system('cp -r '+tb_in[0]+' '+tb_out)
os.system('cp -r ' + tb_in[0] + ' ' + tb_out)
tb.open(tb_out + '/SPECTRAL_WINDOW', nomodify=True)
nspw = tb.nrows()
tb.close()
tim = []
fld = []
spw = []
ant1 = []
ant2 = []
intv = []
scan = []
obid = []
cpar = []
para = []
flag = []
snr = []
# wght=[]
for ctb in tb_in:
tb.open(ctb, nomodify=True)
cols = tb.colnames()
tim0 = tb.getcol(cols[0])
if len(tim0) == 0:
continue
else:
tim.append(tb.getcol(cols[0]))
fld.append(tb.getcol(cols[1]))
spw.append(tb.getcol(cols[2]))
ant1.append(tb.getcol(cols[3]))
ant2.append(tb.getcol(cols[4]))
intv.append(tb.getcol(cols[5]))
scan.append(tb.getcol(cols[6]))
obid.append(tb.getcol(cols[7]))
cpar.append(tb.getcol(cols[8]))
para.append(tb.getcol(cols[9]))
flag.append(tb.getcol(cols[10]))
snr.append(tb.getcol(cols[11]))
# wght.append(tb.getcol(cols[12]))
tb.close()
if len(tim) == 0:
print('tables have no data. Return')
return -1
else:
tim = np.concatenate(tim)
fld = np.concatenate(fld)
spw = np.concatenate(spw)
ant1 = np.concatenate(ant1)
ant2 = np.concatenate(ant2)
intv = np.concatenate(intv)
scan = np.concatenate(scan)
obid = np.concatenate(obid)
cpar = np.concatenate(cpar, axis=2)
para = np.concatenate(para, axis=2)
flag = np.concatenate(flag, axis=2)
snr = np.concatenate(snr, axis=2)
# wght=np.concatenate(wght)
tb.open(tb_out, nomodify=False)
nrows = tb.nrows()
nrows_new = len(tim)
tb.addrows(nrows_new - nrows)
tb.putcol(cols[0], tim)
tb.putcol(cols[1], fld)
tb.putcol(cols[2], spw)
tb.putcol(cols[3], ant1)
tb.putcol(cols[4], ant2)
tb.putcol(cols[5], intv)
tb.putcol(cols[6], scan)
tb.putcol(cols[7], obid)
tb.putcol(cols[8], cpar)
tb.putcol(cols[9], para)
tb.putcol(cols[10], flag)
tb.putcol(cols[11], snr)
tb.close()
return tb_out
def concateovsa(vis, concatvis, datacolumn='corrected', keep_orig_ms=True, cols2rm="model,corrected", freqtol="", dirtol="", respectname=False,
timesort=True, copypointing=True, visweightscale=[], forcesingleephemfield=""):
if concatvis[-1] == os.path.sep:
concatvis = concatvis[:-1]
if os.path.sep not in concatvis:
visprefix = './'
else:
visprefix = os.path.dirname(concatvis) + os.path.sep
msfiles = vis
msfiles_ = []
for idx, ll in enumerate(msfiles):
if str(ll).endswith('/'):
msfiles[idx] = str(ll)[:-1]
datacolumn = datacolumn.lower()
if datacolumn == 'data':
print 'DATA columns will be concatenated.'
for ll in msfiles:
clearcal(vis=str(ll), addmodel=True)
elif datacolumn == 'corrected':
# try:
print 'CORRECTED columns will be concatenated.'
tmpdir = os.path.join(visprefix, 'tmp_ms') + os.path.sep
if not os.path.exists(tmpdir):
os.makedirs(tmpdir)
for ll in msfiles:
msfile_ = os.path.join(tmpdir, os.path.basename(str(ll)))
msfiles_.append(msfile_)
split(vis=str(ll), outputvis=msfile_, datacolumn='corrected')
clearcal(vis=msfile_, addmodel=True)
else:
raise ValueError('Please set datacolumn to be "data" or "corrected"!')
if msfiles_:
concat(vis=msfiles_, concatvis=concatvis, freqtol=freqtol, dirtol=dirtol, respectname=respectname, timesort=timesort,
copypointing=copypointing, visweightscale=visweightscale, forcesingleephemfield=forcesingleephemfield)
os.system('rm -rf {}'.format(tmpdir))
else:
concat(vis=msfiles, concatvis=concatvis, freqtol=freqtol, dirtol=dirtol, respectname=respectname, timesort=timesort,
copypointing=copypointing, visweightscale=visweightscale, forcesingleephemfield=forcesingleephemfield)
# Change all observation ids to be the same (zero)
tb.open(concatvis + '/OBSERVATION', nomodify=False)
nobs = tb.nrows()
tim0 = tb.getcell('TIME_RANGE', 0)[0]
tim1 = tb.getcell('TIME_RANGE', nobs - 1)[1]
tb.removerows([i + 1 for i in range(nobs - 1)])
tb.putcell('TIME_RANGE', 0, [tim0, tim1])
tb.close()
tb.open(concatvis + '/DATA_DESCRIPTION', nomodify=False)
nrows = tb.nrows()
pol_id = tb.getcol('POLARIZATION_ID')
tb.removerows(np.where(pol_id != 0)[0])
tb.close()
tb.open(concatvis, nomodify=False)
dd_id = tb.getcol('DATA_DESC_ID')
idx_dd_id, = np.where(dd_id >= nrows / 2)
dd_id[idx_dd_id] = dd_id[idx_dd_id] - nrows / 2
tb.putcol('DATA_DESC_ID', dd_id)
tb.close()
tb.open(concatvis + '/FIELD', nomodify=False)
nobs = tb.nrows()
tb.removerows([i + 1 for i in range(nobs - 1)])
tb.close()
tb.open(concatvis + '/SOURCE', nomodify=False)
nobs = tb.nrows()
tb.removerows([i + 1 for i in range(nobs - 1)])
tb.close()
tb.open(concatvis, nomodify=False)
obsid = tb.getcol('OBSERVATION_ID')
newobsid = np.zeros(len(obsid), dtype='int')
tb.putcol('OBSERVATION_ID', newobsid)
fldid = tb.getcol('FIELD_ID')
newfldid = np.zeros(len(fldid), dtype='int')
tb.putcol('FIELD_ID', newfldid)
colnames = tb.colnames()
cols2rm = cols2rm.upper()
cols2rm = cols2rm.split(',')
for l in range(len(cols2rm)):
col = cols2rm[l] + '_DATA'
if col in colnames:
try:
tb.removecols(col)
print 'Column {} removed.'.format(col)
except:
pass
tb.close()
if msfiles_ != [] and msfiles_ != msfiles:
for ll in msfiles_:
os.system('rm -rf {}'.format(ll))
if not keep_orig_ms:
for ll in msfiles:
os.system('rm -rf {}'.format(ll))
def compTables(referencetab, testtab, excludecols, tolerance=0.001, mode="percentage", startrow = 0, nrow = -1, rowincr = 1):
"""
compTables - compare two CASA tables
referencetab - the table which is assumed to be correct
testtab - the table which is to be compared to referencetab
excludecols - list of column names which are to be ignored
tolerance - permitted fractional difference (default 0.001 = 0.1 percent)
mode - comparison is made as "percentage", "absolute", "phaseabsdeg" (for complex numbers = difference of the phases in degrees)
"""
rval = True
tb2 = casac.table()
tb.open(referencetab)
cnames = tb.colnames()
tb2.open(testtab)
try:
for c in cnames:
if c in excludecols:
continue
print "\nTesting column " + c
a = 0
try:
a = tb.getcol(c,startrow=startrow,nrow=nrow,rowincr=rowincr)
except:
rval = False
print 'Error accessing column ', c, ' in table ', referencetab
print sys.exc_info()[0]
break
b = 0
try:
b = tb2.getcol(c,startrow=startrow,nrow=nrow,rowincr=rowincr)
except:
rval = False
print 'Error accessing column ', c, ' in table ', testtab
print sys.exc_info()[0]
break
if not (len(a)==len(b)):
print 'Column ',c,' has different length in tables ', referencetab, ' and ', testtab
print a
print b
rval = False
break
else:
differs = False
if not (a==b).all():
for i in range(0,len(a)):
if (isinstance(a[i],float)):
if ((mode=="percentage") and (abs(a[i]-b[i]) > tolerance*abs(a[i]))) or ((mode=="absolute") and (abs(a[i]-b[i]) > tolerance)):
print "Column " + c + " differs"
print "Row=" + str(i)
print "Reference file value: " + str(a[i])
print "Input file value: " + str(b[i])
if (mode=="percentage"):
print "Tolerance is {0}%; observed difference was {1} %".format (tolerance * 100, 100*abs(a[i]-b[i])/abs(a[i]))
else:
print "Absolute tolerance is {0}; observed difference: {1}".format (tolerance, (abs(a[i]-b[i])))
differs = True
rval = False
break
elif (isinstance(a[i],int) or isinstance(a[i],np.int32)):
if (abs(a[i]-b[i]) > 0):
print "Column " + c + " differs"
print "Row=" + str(i)
print "Reference file value: " + str(a[i])
print "Input file value: " + str(b[i])
if (mode=="percentage"):
print "tolerance in % should be " + str(100*abs(a[i]-b[i])/abs(a[i]))
else:
print "absolute tolerance should be " + str(abs(a[i]-b[i]))
differs = True
rval = False
break
elif (isinstance(a[i],str) or isinstance(a[i],np.bool_)):
if not (a[i]==b[i]):
print "Column " + c + " differs"
print "Row=" + str(i)
print "Reference file value: " + str(a[i])
print "Input file value: " + str(b[i])
if (mode=="percentage"):
print "tolerance in % should be " + str(100*abs(a[i]-b[i])/abs(a[i]))
else:
print "absolute tolerance should be " + str(abs(a[i]-b[i]))
differs = True
rval = False
break
elif (isinstance(a[i],list)) or (isinstance(a[i],np.ndarray)):
for j in range(0,len(a[i])):
if differs: break
if ((isinstance(a[i][j],float)) or (isinstance(a[i][j],int))):
if ((mode=="percentage") and (abs(a[i][j]-b[i][j]) > tolerance*abs(a[i][j]))) or ((mode=="absolute") and (abs(a[i][j]-b[i][j]) > tolerance)):
print "Column " + c + " differs"
print "(Row,Element)=(" + str(j) + "," + str(i) + ")"
print "Reference file value: " + str(a[i][j])
print "Input file value: " + str(b[i][j])
if (mode=="percentage"):
print "Tolerance in % should be " + str(100*abs(a[i][j]-b[i][j])/abs(a[i][j]))
else:
print "Absolute tolerance should be " + str(abs(a[i][j]-b[i][j]))
differs = True
rval = False
break
elif (isinstance(a[i][j],list)) or (isinstance(a[i][j],np.ndarray)):
for k in range(0,len(a[i][j])):
if differs: break
if ( ((mode=="percentage") and (abs(a[i][j][k]-b[i][j][k]) > tolerance*abs(a[i][j][k]))) \
or ((mode=="absolute") and (abs(a[i][j][k]-b[i][j][k]) > tolerance)) \
or ((mode=="phaseabsdeg") and (phasediffabsdeg(a[i][j][k],b[i][j][k])>tolerance)) \
):
print "Column " + c + " differs"
print "(Row,Channel,Corr)=(" + str(k) + "," + str(j) + "," + str(i) + ")"
print "Reference file value: " + str(a[i][j][k])
print "Input file value: " + str(b[i][j][k])
if (mode=="percentage"):
print "Tolerance in % should be " + str(100*abs(a[i][j][k]-b[i][j][k])/abs(a[i][j][k]))
elif (mode=="absolute"):
print "Absolute tolerance should be " + str(abs(a[i][j][k]-b[i][j][k]))
elif (mode=="phaseabsdeg"):
print "Phase tolerance in degrees should be " + str(phasediffabsdeg(a[i][j][k],b[i][j][k]))
else:
print "Unknown comparison mode: ",mode
differs = True
rval = False
break
else:
print "Unknown data type: ",type(a[i])
differs = True
rval = False
break
if not differs: print "Column " + c + " PASSED"
finally:
tb.close()
tb2.close()
return rval
def compVarColTables(referencetab, testtab, varcol, tolerance=0.):
'''Compare a variable column of two tables.
referencetab --> a reference table
testtab --> a table to verify
varcol --> the name of a variable column (str)
Returns True or False.
'''
retval = True
tb2 = casac.table()
tb.open(referencetab)
cnames = tb.colnames()
tb2.open(testtab)
col = varcol
if tb.isvarcol(col) and tb2.isvarcol(col):
try:
# First check
if tb.nrows() != tb2.nrows():
print 'Length of %s differ from %s, %s!=%s'%(referencetab,testtab,len(rk),len(tk))
retval = False
else:
for therow in xrange(tb.nrows()):
rdata = tb.getcell(col,therow)
tdata = tb2.getcell(col,therow)
# if not (rdata==tdata).all():
if not rdata.all()==tdata.all():
if (tolerance>0.):
differs=False
for j in range(0,len(rdata)):
### if (type(rdata[j])==float or type(rdata[j])==int):
if ((isinstance(rdata[j],float)) or (isinstance(rdata[j],int))):
if (abs(rdata[j]-tdata[j]) > tolerance*abs(rdata[j]+tdata[j])):
# print 'Column ', col,' differs in tables ', referencetab, ' and ', testtab
# print therow, j
# print rdata[j]
# print tdata[j]
differs = True
### elif (type(rdata[j])==list or type(rdata[j])==np.ndarray):
elif (isinstance(rdata[j],list)) or (isinstance(rdata[j],np.ndarray)):
for k in range(0,len(rdata[j])):
if (abs(rdata[j][k]-tdata[j][k]) > tolerance*abs(rdata[j][k]+tdata[j][k])):
# print 'Column ', col,' differs in tables ', referencetab, ' and ', testtab
# print therow, j, k
# print rdata[j][k]
# print tdata[j][k]
differs = True
if differs:
print 'ERROR: Column %s of %s and %s do not agree within tolerance %s'%(col,referencetab, testtab, tolerance)
retval = False
break
else:
print 'ERROR: Column %s of %s and %s do not agree.'%(col,referencetab, testtab)
print 'ERROR: First row to differ is row=%s'%therow
retval = False
break
finally:
tb.close()
tb2.close()
else:
print 'Columns are not varcolumns.'
retval = False
if retval:
print 'Column %s of %s and %s agree'%(col,referencetab, testtab)
return retval
def importeovsa(idbfiles,
timebin=None,
width=None,
visprefix=None,
nocreatms=False,
doconcat=False,
modelms=''):
casalog.origin('importeovsa')
# # Initialize the helper class
# pdh = ParallelDataHelper("importeovsa", locals())
#
# # Validate input and output parameters
# try:
# pdh.setupIO()
# except Exception, instance:
# casalog.post('%s' % instance, 'ERROR')
# return False
if type(idbfiles) == Time:
filelist = ri.get_trange_files(idbfiles)
else:
# If input type is not Time, assume that it is the list of files to read
filelist = idbfiles
if type(filelist) == str:
filelist = [filelist]
for f in filelist:
if not os.path.exists(f):
casalog.post("Some files in filelist are invalid. Aborting...")
return False
if not visprefix:
visprefix = './'
if not timebin:
timebin = '0s'
if not width:
width = 1
if not modelms:
if nocreatms:
filename = filelist[0]
modelms = ipe.creatms(filename, visprefix)
else:
if not os.path.exists(modelms):
if nocreatms:
filename = filelist[0]
modelms = ipe.creatms(filename, visprefix)
msfile = []
time_concat = []
for filename in filelist:
uv = aipy.miriad.UV(filename)
# if filename.split('/')[-1][0:3] == 'UDB':
# uv_str = uv_hex_rm(uv)
# else:
# uv_str = uv
uv.select('antennae', 0, 1, include=True)
uv.select('polarization', -5, -5, include=True)
times = []
uv.rewind()
for preamble, data in uv.all():
uvw, t, (i, j) = preamble
times.append(t)
uv.select('clear', -1, -1, include=True)
times = ipe.jd2mjds(np.asarray(times))
inttime = np.median((times - np.roll(times, 1))[1:]) / 60
time_steps = len(times)
time_concat.append(int((times[-1] - times[0]) / 60 + inttime))
time0 = time.time()
if 'antlist' in uv.vartable:
ants = uv['antlist'].replace('\x00', '')
antlist = map(int, ants.split())
else:
antlist = [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16]
good_idx = np.where(uv['sfreq'] > 0)[0]
nf = len(good_idx)
npol = uv['npol']
nants = uv['nants']
source_id = uv['source'].replace('\x00', '')
sfreq = uv['sfreq'][good_idx]
sdf = uv['sdf'][good_idx]
ra, dec = uv['ra'], uv['dec']
nbl = nants * (nants - 1) / 2
bl2ord = ipe.bl_list2(nants)
npairs = nbl + nants
flag = np.ones((npol, nf, time_steps, npairs), dtype=bool)
out = np.zeros((npol, nf, time_steps, npairs),
dtype=np.complex64) # Cross-correlations
uvwarray = np.zeros((3, time_steps, npairs), dtype=np.float)
chan_band = ipe.get_band(sfreq=sfreq, sdf=sdf)
nband = len(chan_band)
uv.rewind()
l = -1
for preamble, data in uv.all():
uvw, t, (i0, j0) = preamble
i = antlist.index(i0 + 1)
j = antlist.index(j0 + 1)
if i > j:
# Reverse order of indices
j = antlist.index(i0 + 1)
i = antlist.index(j0 + 1)
# Assumes uv['pol'] is one of -5, -6, -7, -8
k = -5 - uv['pol']
l += 1
out[k, :, l / (npairs * npol), bl2ord[i0, j0]] = data.data
flag[k, :, l / (npairs * npol), bl2ord[i0, j0]] = data.mask
# if i != j:
if k == 3:
uvwarray[:, l / (npairs * npol),
bl2ord[i0,
j0]] = -uvw * constants.speed_of_light / 1e9
nrows = time_steps * npairs
out = out.reshape(npol, nf, nrows)
flag = flag.reshape(npol, nf, nrows)
uvwarray = uvwarray.reshape(3, nrows)
uvwarray = np.tile(uvwarray, (1, nband))
sigma = np.ones((4, nrows), dtype=np.float) + 1
sigma = np.tile(sigma, (1, nband))
casalog.post(
'IDB File {0} is readed in --- {1:10.2f} seconds ---'.format(
filename, (time.time() - time0)))
msname = list(filename.split('/')[-1])
msname = visprefix + ''.join(msname) + '.ms'
if not nocreatms:
modelms = ipe.creatms(filename, visprefix)
os.system('mv {} {}'.format(modelms, msname))
else:
casalog.post('----------------------------------------')
casalog.post('copying standard MS to {0}'.format(
msname, (time.time() - time0)))
casalog.post('----------------------------------------')
os.system("rm -fr %s" % msname)
os.system("cp -r " + " %s" % modelms + " %s" % msname)
casalog.post(
'Standard MS is copied to {0} in --- {1:10.2f} seconds ---'.
format(msname, (time.time() - time0)))
tb.open(msname, nomodify=False)
casalog.post('----------------------------------------')
casalog.post("Updating the main table of" '%s' % msname)
casalog.post('----------------------------------------')
for l, cband in enumerate(chan_band):
time1 = time.time()
nchannels = len(cband['cidx'])
for row in range(nrows):
tb.putcell('DATA', (row + l * nrows),
out[:, cband['cidx'][0]:cband['cidx'][-1] + 1, row])
tb.putcell(
'FLAG', (row + l * nrows),
flag[:, cband['cidx'][0]:cband['cidx'][-1] + 1, row])
casalog.post(
'---spw {0:02d} is updated in --- {1:10.2f} seconds ---'.
format((l + 1),
time.time() - time1))
tb.putcol('UVW', uvwarray)
tb.putcol('SIGMA', sigma)
tb.putcol('WEIGHT', 1.0 / sigma**2)
timearr = times
timearr = timearr.reshape(1, time_steps, 1)
timearr = np.tile(timearr, (nband, 1, npairs))
timearr = timearr.reshape(nband * npairs * time_steps)
tb.putcol('TIME', timearr)
tb.putcol('TIME_CENTROID', timearr)
scan_id = tb.getcol('SCAN_NUMBER')
scan_id *= 0
tb.putcol('SCAN_NUMBER', scan_id)
colnames = tb.colnames()
cols2rm = ["MODEL_DATA", "CORRECTED_DATA"]
for l in range(len(cols2rm)):
if cols2rm[l] in colnames:
tb.removecols(cols2rm[l])
tb.close()
casalog.post('----------------------------------------')
casalog.post("Updating the OBSERVATION table of" '%s' % msname)
casalog.post('----------------------------------------')
tb.open(msname + '/OBSERVATION', nomodify=False)
tb.putcol(
'TIME_RANGE',
np.asarray([times[0] - 0.5 * inttime,
times[-1] + 0.5 * inttime]).reshape(2, 1))
tb.putcol('OBSERVER', ['EOVSA team'])
tb.close()
casalog.post('----------------------------------------')
casalog.post("Updating the POINTING table of" '%s' % msname)
casalog.post('----------------------------------------')
tb.open(msname + '/POINTING', nomodify=False)
timearr = times.reshape(1, time_steps, 1)
timearr = np.tile(timearr, (nband, 1, nants))
timearr = timearr.reshape(nband * time_steps * nants)
tb.putcol('TIME', timearr)
tb.putcol('TIME_ORIGIN', timearr) # - 0.5 * delta_time)
direction = tb.getcol('DIRECTION')
direction[0, 0, :] = ra
direction[1, 0, :] = dec
tb.putcol('DIRECTION', direction)
target = tb.getcol('TARGET')
target[0, 0, :] = ra
target[1, 0, :] = dec
tb.putcol('TARGET', target)
tb.close()
casalog.post('----------------------------------------')
casalog.post("Updating the SOURCE table of" '%s' % msname)
casalog.post('----------------------------------------')
tb.open(msname + '/SOURCE', nomodify=False)
radec = tb.getcol('DIRECTION')
radec[0], radec[1] = ra, dec
tb.putcol('DIRECTION', radec)
name = np.array([source_id], dtype='|S{0}'.format(len(source_id) + 1))
tb.putcol('NAME', name)
tb.close()
casalog.post('----------------------------------------')
casalog.post("Updating the DATA_DESCRIPTION table of" '%s' % msname)
casalog.post('----------------------------------------')
tb.open(msname + '/DATA_DESCRIPTION/', nomodify=False)
pol_id = tb.getcol('POLARIZATION_ID')
pol_id *= 0
tb.putcol('POLARIZATION_ID', pol_id)
# spw_id = tb.getcol('SPECTRAL_WINDOW_ID')
# spw_id *= 0
# tb.putcol('SPECTRAL_WINDOW_ID', spw_id)
tb.close()
# pdb.set_trace()
casalog.post('----------------------------------------')
casalog.post("Updating the POLARIZATION table of" '%s' % msname)
casalog.post('----------------------------------------')
tb.open(msname + '/POLARIZATION/', nomodify=False)
print tb.nrows()
tb.removerows(rownrs=np.arange(1, nband, dtype=int))
tb.close()
casalog.post('----------------------------------------')
casalog.post("Updating the FIELD table of" '%s' % msname)
casalog.post('----------------------------------------')
tb.open(msname + '/FIELD/', nomodify=False)
delay_dir = tb.getcol('DELAY_DIR')
delay_dir[0], delay_dir[1] = ra, dec
tb.putcol('DELAY_DIR', delay_dir)
phase_dir = tb.getcol('PHASE_DIR')
phase_dir[0], phase_dir[1] = ra, dec
tb.putcol('PHASE_DIR', phase_dir)
reference_dir = tb.getcol('REFERENCE_DIR')
reference_dir[0], reference_dir[1] = ra, dec
tb.putcol('REFERENCE_DIR', reference_dir)
name = np.array([source_id], dtype='|S{0}'.format(len(source_id) + 1))
tb.putcol('NAME', name)
tb.close()
# FIELD: DELAY_DIR, PHASE_DIR, REFERENCE_DIR, NAME
del out, flag, uvwarray, uv, timearr, sigma
gc.collect() #
if not (timebin == '0s' and width == 1):
split(vis=msname,
outputvis=msname + '.split',
datacolumn='data',
timebin=timebin,
width=width,
keepflags=False)
os.system('rm -rf {}'.format(msname))
msfile.append(msname + '.split')
else:
msfile.append(msname)
casalog.post("finished in --- %s seconds ---" % (time.time() - time0))
if doconcat:
msname = list(filelist[0].split('/')[-1])
concatvis = visprefix + ''.join(msname) + '-{:d}m.ms'.format(
int(sum(time_concat)))
concat(vis=msfile, concatvis=concatvis, timesort=True)
# Change all observation ids to be the same (zero)
tb.open(concatvis + '/OBSERVATION', nomodify=False)
nobs = tb.nrows()
tb.removerows([i + 1 for i in range(nobs - 1)])
tb.close()
tb.open(concatvis, nomodify=False)
obsid = tb.getcol('OBSERVATION_ID')
newobsid = np.zeros(len(obsid), dtype='int')
tb.putcol('OBSERVATION_ID', newobsid)
tb.close()
for ll in msfile:
os.system('rm -rf {}'.format(ll))
return True
