def importeovsa_iter(filelist, timebin, width, visprefix, nocreatms, modelms,
doscaling, keep_nsclms, fileidx):
from taskinit import tb, casalog
filename = filelist[fileidx]
uv = aipy.miriad.UV(filename)
# try:
msname0 = list(filename.split('/')[-1])
msname = visprefix + ''.join(msname0) + '.ms'
uv.select('antennae', 0, 1, include=True)
uv.select('polarization', -5, -5, include=True)
times = []
uv.rewind()
for preamble, data in uv.all():
uvw, t, (i, j) = preamble
times.append(t)
uv.select('clear', -1, -1, include=True)
times = ipe.jd2mjds(np.asarray(times))
inttime = np.median((times - np.roll(times, 1))[1:]) / 60
time_steps = len(times)
durtim = int((times[-1] - times[0]) / 60 + inttime)
time0 = time.time()
if 'antlist' in uv.vartable:
ants = uv['antlist'].replace('\x00', '')
antlist = map(int, ants.split())
else:
antlist = [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16]
good_idx = np.where(uv['sfreq'] > 0)[0]
nf = len(good_idx)
npol = uv['npol']
nants = uv['nants']
source_id = uv['source'].replace('\x00', '')
sfreq = uv['sfreq'][good_idx]
sdf = uv['sdf'][good_idx]
ra, dec = uv['ra'], uv['dec']
nbl = nants * (nants - 1) / 2
bl2ord = ipe.bl_list2(nants)
npairs = nbl + nants
flag = np.ones((npol, nf, time_steps, npairs), dtype=bool)
out = np.zeros((npol, nf, time_steps, npairs),
dtype=np.complex64) # Cross-correlations
uvwarray = np.zeros((3, time_steps, npairs), dtype=np.float)
chan_band = ipe.get_band(sfreq=sfreq, sdf=sdf)
nband = len(chan_band)
uv.rewind()
l = -1
for preamble, data in uv.all():
uvw, t, (i0, j0) = preamble
i = antlist.index(i0 + 1)
j = antlist.index(j0 + 1)
if i > j:
# Reverse order of indices
j = antlist.index(i0 + 1)
i = antlist.index(j0 + 1)
# Assumes uv['pol'] is one of -5, -6, -7, -8
k = -5 - uv['pol']
l += 1
data = ma.masked_array(ma.masked_invalid(data), fill_value=0.0)
out[k, :, l / (npairs * npol), bl2ord[i0, j0]] = data.data
flag[k, :, l / (npairs * npol), bl2ord[i0, j0]] = data.mask
# if i != j:
if k == 3:
uvwarray[:, l / (npairs * npol),
bl2ord[i0, j0]] = -uvw * constants.speed_of_light / 1e9
nrows = time_steps * npairs
if doscaling:
out2 = out.copy()
for i0 in antlist:
for j0 in antlist:
if i0 < j0:
i, j = i0 - 1, j0 - 1
out2[:, :, :,
bl2ord[i, j]] = out[:, :, :, bl2ord[i, j]] / np.sqrt(
np.abs(out[:, :, :, bl2ord[i, i]]) *
np.abs(out[:, :, :, bl2ord[j, j]]))
out2 = out2.reshape(npol, nf, nrows)
out2[np.isnan(out2)] = 0
out2[np.isinf(out2)] = 0
# out2 = ma.masked_array(ma.masked_invalid(out2), fill_value=0.0)
out = out.reshape(npol, nf, nrows) * 1e4
flag = flag.reshape(npol, nf, nrows)
uvwarray = uvwarray.reshape(3, nrows)
uvwarray = np.tile(uvwarray, (1, nband))
sigma = np.ones((4, nrows), dtype=np.float) + 1
sigma = np.tile(sigma, (1, nband))
casalog.post('IDB File {0} is readed in --- {1:10.2f} seconds ---'.format(
filename, (time.time() - time0)))
if not nocreatms:
modelms = ipe.creatms(filename, visprefix)
os.system('mv {} {}'.format(modelms, msname))
else:
casalog.post('----------------------------------------')
casalog.post('copying standard MS to {0}'.format(
msname, (time.time() - time0)))
casalog.post('----------------------------------------')
os.system("rm -fr %s" % msname)
os.system("cp -r " + " %s" % modelms + " %s" % msname)
casalog.post(
'Standard MS is copied to {0} in --- {1:10.2f} seconds ---'.format(
msname, (time.time() - time0)))
tb.open(msname, nomodify=False)
casalog.post('----------------------------------------')
casalog.post("Updating the main table of" '%s' % msname)
casalog.post('----------------------------------------')
for l, cband in enumerate(chan_band):
time1 = time.time()
nchannels = len(cband['cidx'])
for row in range(nrows):
if not doscaling or keep_nsclms:
tb.putcell('DATA', (row + l * nrows),
out[:, cband['cidx'][0]:cband['cidx'][-1] + 1, row])
tb.putcell('FLAG', (row + l * nrows),
flag[:, cband['cidx'][0]:cband['cidx'][-1] + 1, row])
casalog.post(
'---spw {0:02d} is updated in --- {1:10.2f} seconds ---'.format(
(l + 1),
time.time() - time1))
tb.putcol('UVW', uvwarray)
tb.putcol('SIGMA', sigma)
tb.putcol('WEIGHT', 1.0 / sigma**2)
timearr = times
timearr = timearr.reshape(1, time_steps, 1)
timearr = np.tile(timearr, (nband, 1, npairs))
timearr = timearr.reshape(nband * npairs * time_steps)
tb.putcol('TIME', timearr)
tb.putcol('TIME_CENTROID', timearr)
scan_id = tb.getcol('SCAN_NUMBER')
scan_id *= 0
tb.putcol('SCAN_NUMBER', scan_id)
colnames = tb.colnames()
cols2rm = ["MODEL_DATA", "CORRECTED_DATA"]
for l in range(len(cols2rm)):
if cols2rm[l] in colnames:
tb.removecols(cols2rm[l])
tb.close()
casalog.post('----------------------------------------')
casalog.post("Updating the OBSERVATION table of" '%s' % msname)
casalog.post('----------------------------------------')
tb.open(msname + '/OBSERVATION', nomodify=False)
tb.putcol(
'TIME_RANGE',
np.asarray([times[0] - 0.5 * inttime,
times[-1] + 0.5 * inttime]).reshape(2, 1))
tb.putcol('OBSERVER', ['EOVSA team'])
tb.close()
casalog.post('----------------------------------------')
casalog.post("Updating the POINTING table of" '%s' % msname)
casalog.post('----------------------------------------')
tb.open(msname + '/POINTING', nomodify=False)
timearr = times.reshape(1, time_steps, 1)
timearr = np.tile(timearr, (nband, 1, nants))
timearr = timearr.reshape(nband * time_steps * nants)
tb.putcol('TIME', timearr)
tb.putcol('TIME_ORIGIN', timearr) # - 0.5 * delta_time)
direction = tb.getcol('DIRECTION')
direction[0, 0, :] = ra
direction[1, 0, :] = dec
tb.putcol('DIRECTION', direction)
target = tb.getcol('TARGET')
target[0, 0, :] = ra
target[1, 0, :] = dec
tb.putcol('TARGET', target)
tb.close()
casalog.post('----------------------------------------')
casalog.post("Updating the SOURCE table of" '%s' % msname)
casalog.post('----------------------------------------')
tb.open(msname + '/SOURCE', nomodify=False)
radec = tb.getcol('DIRECTION')
radec[0], radec[1] = ra, dec
tb.putcol('DIRECTION', radec)
name = np.array([source_id], dtype='|S{0}'.format(len(source_id) + 1))
tb.putcol('NAME', name)
tb.close()
casalog.post('----------------------------------------')
casalog.post("Updating the DATA_DESCRIPTION table of" '%s' % msname)
casalog.post('----------------------------------------')
tb.open(msname + '/DATA_DESCRIPTION/', nomodify=False)
pol_id = tb.getcol('POLARIZATION_ID')
pol_id *= 0
tb.putcol('POLARIZATION_ID', pol_id)
# spw_id = tb.getcol('SPECTRAL_WINDOW_ID')
# spw_id *= 0
# tb.putcol('SPECTRAL_WINDOW_ID', spw_id)
tb.close()
# casalog.post('----------------------------------------')
# casalog.post("Updating the POLARIZATION table of" '%s' % msname)
# casalog.post('----------------------------------------')
# tb.open(msname + '/POLARIZATION/', nomodify=False)
# tb.removerows(rownrs=np.arange(1, nband, dtype=int))
# tb.close()
casalog.post('----------------------------------------')
casalog.post("Updating the FIELD table of" '%s' % msname)
casalog.post('----------------------------------------')
tb.open(msname + '/FIELD/', nomodify=False)
delay_dir = tb.getcol('DELAY_DIR')
delay_dir[0], delay_dir[1] = ra, dec
tb.putcol('DELAY_DIR', delay_dir)
phase_dir = tb.getcol('PHASE_DIR')
phase_dir[0], phase_dir[1] = ra, dec
tb.putcol('PHASE_DIR', phase_dir)
reference_dir = tb.getcol('REFERENCE_DIR')
reference_dir[0], reference_dir[1] = ra, dec
tb.putcol('REFERENCE_DIR', reference_dir)
name = np.array([source_id], dtype='|S{0}'.format(len(source_id) + 1))
tb.putcol('NAME', name)
tb.close()
# FIELD: DELAY_DIR, PHASE_DIR, REFERENCE_DIR, NAME
# del out, flag, uvwarray, uv, timearr, sigma
# gc.collect() #
if doscaling:
if keep_nsclms:
msname_scl = visprefix + ''.join(msname0) + '_scl.ms'
os.system('cp -r {} {}'.format(msname, msname_scl))
else:
msname_scl = msname
tb.open(msname_scl, nomodify=False)
casalog.post('----------------------------------------')
casalog.post("Updating the main table of" '%s' % msname_scl)
casalog.post('----------------------------------------')
for l, cband in enumerate(chan_band):
time1 = time.time()
for row in range(nrows):
tb.putcell(
'DATA', (row + l * nrows),
out2[:, cband['cidx'][0]:cband['cidx'][-1] + 1, row])
casalog.post(
'---spw {0:02d} is updated in --- {1:10.2f} seconds ---'.
format((l + 1),
time.time() - time1))
tb.close()
if not (timebin == '0s' and width == 1):
msfile = msname + '.split'
if doscaling:
split(vis=msname_scl,
outputvis=msname_scl + '.split',
datacolumn='data',
timebin=timebin,
width=width,
keepflags=False)
os.system('rm -rf {}'.format(msname_scl))
msfile_scl = msname_scl + '.split'
if not (doscaling and not keep_nsclms):
split(vis=msname,
outputvis=msname + '.split',
datacolumn='data',
timebin=timebin,
width=width,
keepflags=False)
os.system('rm -rf {}'.format(msname))
else:
msfile = msname
if doscaling:
msfile_scl = msname_scl
casalog.post("finished in --- %s seconds ---" % (time.time() - time0))
if doscaling:
return [True, msfile, msfile_scl, durtim]
else:
return [True, msfile, durtim]
def importeovsa(idbfiles=None,
ncpu=None,
timebin=None,
width=None,
visprefix=None,
udb_corr=True,
nocreatms=None,
doconcat=None,
modelms=None,
doscaling=False,
keep_nsclms=False):
casalog.origin('importeovsa')
# if type(idbfiles) == Time:
# filelist = ri.get_trange_files(idbfiles)
# else:
# # If input type is not Time, assume that it is the list of files to read
filelist = idbfiles
if type(filelist) == str:
filelist = [filelist]
filelist_tmp = []
for ll in filelist:
if not os.path.exists(ll):
casalog.post("Warning: {} not exist.".format(ll))
else:
filelist_tmp.append(ll)
filelist = filelist_tmp
if not filelist:
casalog.post("No file in idbfiles list exists. Abort.")
return False
for idx, ll in enumerate(filelist):
if ll[-1] == '/':
filelist[idx] = ll[:-1]
if not visprefix:
visprefix = './'
if not timebin:
timebin = '0s'
if not width:
width = 1
import sys
print 10
sys.stdout.flush()
if udb_corr:
udbcorr_path = visprefix + '/tmp_UDBcorr/'
print 0
sys.stdout.flush()
if not os.path.exists(udbcorr_path):
os.makedirs(udbcorr_path)
print 1
sys.stdout.flush()
from eovsapy import pipeline_cal as pc
print 2
sys.stdout.flush()
filelist_tmp = []
for ll in filelist:
filelist_tmp.append(
pc.udb_corr(ll, outpath=udbcorr_path, calibrate=True))
filelist = filelist_tmp
print 3
sys.stdout.flush()
print 4
sys.stdout.flush()
if not modelms:
if nocreatms:
filename = filelist[0]
modelms = ipe.creatms(filename, visprefix)
else:
if not os.path.exists(modelms):
if nocreatms:
filename = filelist[0]
modelms = ipe.creatms(filename, visprefix)
iterable = range(len(filelist))
t0 = time.time()
casalog.post(
'Perform importeovsa in parallel with {} CPUs...'.format(ncpu))
if ncpu == 1:
res = []
for fidx, ll in enumerate(filelist):
res.append(
importeovsa_iter(filelist, timebin, width, visprefix,
nocreatms, modelms, doscaling, keep_nsclms,
fidx))
if ncpu > 1:
import multiprocessing as mprocs
from functools import partial
imppart = partial(importeovsa_iter, filelist, timebin, width,
visprefix, nocreatms, modelms, doscaling,
keep_nsclms)
pool = mprocs.Pool(ncpu)
res = pool.map(imppart, iterable)
pool.close()
pool.join()
# print res
t1 = time.time()
timelapse = t1 - t0
print 'It took %f secs to complete' % timelapse
# results = pd.DataFrame({'succeeded': [], 'msfile': [], 'durtim': []})
# for r in res:
# results = results.append(pd.DataFrame({'succeeded': [r[0]], 'msfile': [r[1]], 'durtim': [r[2]]}))
# try:
succeeded = []
msfile = []
durtim = []
if doscaling:
msfile_scl = []
for r in res:
succeeded.append(r[0])
msfile.append(r[1])
msfile_scl.append(r[2])
durtim.append(r[3])
results = {
'succeeded': succeeded,
'msfile': msfile,
'msfile_scl': msfile_scl,
'durtim': durtim
}
else:
for r in res:
succeeded.append(r[0])
msfile.append(r[1])
durtim.append(r[2])
results = {'succeeded': succeeded, 'msfile': msfile, 'durtim': durtim}
# except:
# print 'errors occurred when creating the output summary.'
if doconcat:
from suncasa.tasks import concateovsa_cli as ce
msname = os.path.basename(filelist[0])
durtim = int(np.array(results['durtim']).sum())
if doscaling:
msfiles = list(
np.array(results['msfile_scl'])[np.where(
np.array(results['succeeded']) == True)])
if keep_nsclms:
concatvis = visprefix + msname + '-{:d}m{}.ms'.format(
durtim, '_scl')
else:
concatvis = visprefix + msname + '-{:d}m{}.ms'.format(
durtim, '')
else:
msfiles = list(
np.array(results['msfile'])[np.where(
np.array(results['succeeded']) == True)])
concatvis = visprefix + msname + '-{:d}m{}.ms'.format(durtim, '')
ce.concateovsa(msfiles,
concatvis,
datacolumn='data',
keep_orig_ms=True,
cols2rm="model,corrected")
return True
if udb_corr:
os.system('rm -rf {}'.format(udbcorr_path))
def importeovsa(idbfiles=None,
ncpu=None,
timebin=None,
width=None,
visprefix=None,
nocreatms=None,
doconcat=None,
modelms=None,
doscaling=True,
keep_nsclms=False):
casalog.origin('importeovsa')
# if type(idbfiles) == Time:
# filelist = ri.get_trange_files(idbfiles)
# else:
# # If input type is not Time, assume that it is the list of files to read
filelist = idbfiles
if type(filelist) == str:
filelist = [filelist]
filelist_tmp = []
for f in filelist:
if not os.path.exists(f):
casalog.post("Warning: {} not exist.".format(f))
else:
filelist_tmp.append(f)
filelist = filelist_tmp
if not filelist:
casalog.post("No file in idbfiles list exists. Abort.")
return False
for idx, f in enumerate(filelist):
if f[-1] == '/':
filelist[idx] = f[:-1]
if not visprefix:
visprefix = './'
if not timebin:
timebin = '0s'
if not width:
width = 1
if not modelms:
if nocreatms:
filename = filelist[0]
modelms = ipe.creatms(filename, visprefix)
else:
if not os.path.exists(modelms):
if nocreatms:
filename = filelist[0]
modelms = ipe.creatms(filename, visprefix)
iterable = range(len(filelist))
imppart = partial(importeovsa_iter, filelist, timebin, width, visprefix,
nocreatms, modelms, doscaling, keep_nsclms)
t0 = time.time()
casalog.post(
'Perform importeovsa in parallel with {} CPUs...'.format(ncpu))
pool = mprocs.Pool(ncpu)
res = pool.map(imppart, iterable)
pool.close()
pool.join()
# print res
t1 = time.time()
timelapse = t1 - t0
print 'It took %f secs to complete' % timelapse
# results = pd.DataFrame({'succeeded': [], 'msfile': [], 'durtim': []})
# for r in res:
# results = results.append(pd.DataFrame({'succeeded': [r[0]], 'msfile': [r[1]], 'durtim': [r[2]]}))
# try:
succeeded = []
msfile = []
durtim = []
if doscaling:
msfile_scl = []
for r in res:
succeeded.append(r[0])
msfile.append(r[1])
msfile_scl.append(r[2])
durtim.append(r[3])
results = {
'succeeded': succeeded,
'msfile': msfile,
'msfile_scl': msfile_scl,
'durtim': durtim
}
else:
for r in res:
succeeded.append(r[0])
msfile.append(r[1])
durtim.append(r[2])
results = {'succeeded': succeeded, 'msfile': msfile, 'durtim': durtim}
# except:
# print 'errors occurred when creating the output summary.'
if doconcat:
msname = os.path.basename(filelist[0])
if doscaling:
msfiles = list(
np.array(results['msfile_scl'])[np.where(
np.array(results['succeeded']) == True)])
durtim = int(np.array(results['durtim']).sum())
if keep_nsclms:
ce.concateovsa(msname + '-{:d}m{}.ms'.format(durtim, '_scl'),
msfiles, visprefix)
else:
ce.concateovsa(msname + '-{:d}m{}.ms'.format(durtim, ''),
msfiles, visprefix)
else:
msfiles = list(
np.array(results['msfile'])[np.where(
np.array(results['succeeded']) == True)])
durtim = int(results['durtim'].sum())
ce.concateovsa(msname + '-{:d}m{}.ms'.format(durtim, ''), msfiles,
visprefix)
return True
