def concateovsa(msname,
msfiles,
visprefix='./',
doclearcal=True,
keep_orig_ms=False,
cols2rm=["MODEL_DATA", "CORRECTED_DATA"]):
concatvis = visprefix + msname
msfiles_ = []
for idx, ll in enumerate(msfiles):
if str(ll).endswith('/'):
msfiles[idx] = str(ll)[:-1]
if doclearcal:
print 'Warning: Corrected column in the input ms file will be cleared!!!'
for ll in msfiles:
clearcal(vis=str(ll), addmodel=True)
else:
try:
tmpdir = visprefix + '/tmp_ms/'
if not os.path.exists(tmpdir):
os.makedirs(tmpdir)
for ll in msfiles:
msfile_ = tmpdir + os.path.basename(str(ll))
msfiles_.append(msfile_)
split(vis=str(ll), outputvis=msfile_, datacolumn='corrected')
clearcal(vis=msfile_, addmodel=True)
except:
print 'Warning: Corrected column not found in the input ms file.'
msfiles_ = msfiles
if msfiles_:
concat(vis=msfiles_, concatvis=concatvis, timesort=True)
else:
concat(vis=msfiles, concatvis=concatvis, timesort=True)
# Change all observation ids to be the same (zero)
tb.open(concatvis + '/OBSERVATION', nomodify=False)
nobs = tb.nrows()
tim0 = tb.getcell('TIME_RANGE', 0)[0]
tim1 = tb.getcell('TIME_RANGE', nobs - 1)[1]
tb.removerows([i + 1 for i in range(nobs - 1)])
tb.putcell('TIME_RANGE', 0, [tim0, tim1])
tb.close()
tb.open(concatvis, nomodify=False)
obsid = tb.getcol('OBSERVATION_ID')
newobsid = np.zeros(len(obsid), dtype='int')
tb.putcol('OBSERVATION_ID', newobsid)
colnames = tb.colnames()
for l in range(len(cols2rm)):
if cols2rm[l] in colnames:
try:
tb.removecols(cols2rm[l])
except:
pass
tb.close()
if msfiles_ != [] and msfiles_ != msfiles:
for ll in msfiles_:
os.system('rm -rf {}'.format(ll))
if not keep_orig_ms:
for ll in msfiles:
os.system('rm -rf {}'.format(ll))
msmd = msmdtool()
tb = tbtool()
from astropy.io import fits
from astropy import wcs
mses = list(Qmses.keys())
fullpath_mses = ['../' + ms[:-3] + "_continuum.ms"
for ms in mses if ms in Qmses]
raw_and_corr_vis = 'continuum_concatenated_raw_and_corr.ms'
if not os.path.exists(raw_and_corr_vis):
assert concat(vis=fullpath_mses, concatvis=raw_and_corr_vis,
# should be used but isn't freqtol='5MHz',
)
cont_vis = 'continuum_concatenated_selfcal_wterms.ms'
if not os.path.exists(cont_vis):
assert split(vis=raw_and_corr_vis, outputvis=cont_vis,
datacolumn='corrected')
# this is OK because there should be no corrected column
clearcal(vis=cont_vis, addmodel=True)
imagename = '18A-229_Q_mosaic_selfcal_iter0_dirty'
if not os.path.exists(imagename+".image.tt0.pbcor"):
# do a full-mosaic clean to enable mask creation
tclean(
vis=cont_vis,
def importeovsa(idbfiles,
timebin=None,
width=None,
visprefix=None,
nocreatms=False,
doconcat=False,
modelms=''):
casalog.origin('importeovsa')
# # Initialize the helper class
# pdh = ParallelDataHelper("importeovsa", locals())
#
# # Validate input and output parameters
# try:
# pdh.setupIO()
# except Exception, instance:
# casalog.post('%s' % instance, 'ERROR')
# return False
if type(idbfiles) == Time:
filelist = ri.get_trange_files(idbfiles)
else:
# If input type is not Time, assume that it is the list of files to read
filelist = idbfiles
if type(filelist) == str:
filelist = [filelist]
for f in filelist:
if not os.path.exists(f):
casalog.post("Some files in filelist are invalid. Aborting...")
return False
if not visprefix:
visprefix = './'
if not timebin:
timebin = '0s'
if not width:
width = 1
if not modelms:
if nocreatms:
filename = filelist[0]
modelms = creatms(filename, visprefix)
else:
if not os.path.exists(modelms):
if nocreatms:
filename = filelist[0]
modelms = creatms(filename, visprefix)
msfile = []
time_concat = []
for filename in filelist:
uv = aipy.miriad.UV(filename)
# if filename.split('/')[-1][0:3] == 'UDB':
# uv_str = uv_hex_rm(uv)
# else:
# uv_str = uv
uv.select('antennae', 0, 1, include=True)
uv.select('polarization', -5, -5, include=True)
times = []
uv.rewind()
for preamble, data in uv.all():
uvw, t, (i, j) = preamble
times.append(t)
uv.select('clear', -1, -1, include=True)
times = jd2mjds(np.asarray(times))
inttime = np.median((times - np.roll(times, 1))[1:]) / 60
time_steps = len(times)
time_concat.append(int((times[-1] - times[0]) / 60 + inttime))
time0 = time.time()
if 'antlist' in uv.vartable:
ants = uv['antlist'].replace('\x00', '')
antlist = map(int, ants.split())
else:
antlist = [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16]
good_idx = np.where(uv['sfreq'] > 0)[0]
nf = len(good_idx)
npol = uv['npol']
nants = uv['nants']
source_id = uv['source'].replace('\x00', '')
sfreq = uv['sfreq'][good_idx]
sdf = uv['sdf'][good_idx]
ra, dec = uv['ra'], uv['dec']
nbl = nants * (nants - 1) / 2
bl2ord = bl_list2(nants)
npairs = nbl + nants
flag = np.ones((npol, nf, time_steps, npairs), dtype=bool)
out = np.zeros((npol, nf, time_steps, npairs),
dtype=np.complex64) # Cross-correlations
uvwarray = np.zeros((3, time_steps, npairs), dtype=np.float)
chan_band = get_band(sfreq=sfreq, sdf=sdf)
nband = len(chan_band)
uv.rewind()
l = -1
for preamble, data in uv.all():
uvw, t, (i0, j0) = preamble
i = antlist.index(i0 + 1)
j = antlist.index(j0 + 1)
if i > j:
# Reverse order of indices
j = antlist.index(i0 + 1)
i = antlist.index(j0 + 1)
# Assumes uv['pol'] is one of -5, -6, -7, -8
k = -5 - uv['pol']
l += 1
out[k, :, l / (npairs * npol), bl2ord[i0, j0]] = data.data
flag[k, :, l / (npairs * npol), bl2ord[i0, j0]] = data.mask
if i != j:
if k == 3:
uvwarray[:, l / (npairs * npol), bl2ord[
i0, j0]] = -uvw * constants.speed_of_light / 1e9
nrows = time_steps * npairs
out = out.reshape(npol, nf, nrows)
flag = flag.reshape(npol, nf, nrows)
uvwarray = uvwarray.reshape(3, nrows)
uvwarray = np.tile(uvwarray, (1, nband))
sigma = np.ones((4, nrows), dtype=np.float) + 1
sigma = np.tile(sigma, (1, nband))
casalog.post(
'IDB File {0} is readed in --- {1:10.2f} seconds ---'.format(
filename, (time.time() - time0)))
msname = list(filename.split('/')[-1])
msname = visprefix + ''.join(msname) + '-{:d}m.ms'.format(
int((times[-1] - times[0]) / 60 + inttime))
if not nocreatms:
modelms = creatms(filename, visprefix)
os.system('mv {} {}'.format(modelms, msname))
else:
casalog.post('----------------------------------------')
casalog.post('copying standard MS to {0}'.format(
msname, (time.time() - time0)))
casalog.post('----------------------------------------')
os.system("rm -fr %s" % msname)
os.system("cp -r " + " %s" % modelms + " %s" % msname)
casalog.post(
'Standard MS is copied to {0} in --- {1:10.2f} seconds ---'.
format(msname, (time.time() - time0)))
tb.open(msname, nomodify=False)
casalog.post('----------------------------------------')
casalog.post("Updating the main table of" '%s' % msname)
casalog.post('----------------------------------------')
for l, cband in enumerate(chan_band):
time1 = time.time()
nchannels = len(cband['cidx'])
for row in range(nrows):
tb.putcell('DATA', (row + l * nrows),
out[:, cband['cidx'][0]:cband['cidx'][-1] + 1, row])
tb.putcell(
'FLAG', (row + l * nrows),
flag[:, cband['cidx'][0]:cband['cidx'][-1] + 1, row])
casalog.post(
'---spw {0:02d} is updated in --- {1:10.2f} seconds ---'.
format((l + 1),
time.time() - time1))
tb.putcol('UVW', uvwarray)
tb.putcol('SIGMA', sigma)
tb.putcol('WEIGHT', 1.0 / sigma**2)
timearr = times
timearr = timearr.reshape(1, time_steps, 1)
timearr = np.tile(timearr, (nband, 1, npairs))
timearr = timearr.reshape(nband * npairs * time_steps)
tb.putcol('TIME', timearr)
tb.putcol('TIME_CENTROID', timearr)
scan_id = tb.getcol('SCAN_NUMBER')
scan_id *= 0
tb.putcol('SCAN_NUMBER', scan_id)
colnames = tb.colnames()
cols2rm = ["MODEL_DATA", "CORRECTED_DATA"]
for l in range(len(cols2rm)):
if cols2rm[l] in colnames:
tb.removecols(cols2rm[l])
tb.close()
casalog.post('----------------------------------------')
casalog.post("Updating the OBSERVATION table of" '%s' % msname)
casalog.post('----------------------------------------')
tb.open(msname + '/OBSERVATION', nomodify=False)
tb.putcol(
'TIME_RANGE',
np.asarray([times[0] - 0.5 * inttime,
times[-1] + 0.5 * inttime]).reshape(2, 1))
tb.putcol('OBSERVER', ['EOVSA team'])
tb.close()
casalog.post('----------------------------------------')
casalog.post("Updating the POINTING table of" '%s' % msname)
casalog.post('----------------------------------------')
tb.open(msname + '/POINTING', nomodify=False)
timearr = times.reshape(1, time_steps, 1)
timearr = np.tile(timearr, (nband, 1, nants))
timearr = timearr.reshape(nband * time_steps * nants)
tb.putcol('TIME', timearr)
tb.putcol('TIME_ORIGIN', timearr) # - 0.5 * delta_time)
direction = tb.getcol('DIRECTION')
direction[0, 0, :] = ra
direction[1, 0, :] = dec
tb.putcol('DIRECTION', direction)
target = tb.getcol('TARGET')
target[0, 0, :] = ra
target[1, 0, :] = dec
tb.putcol('TARGET', target)
tb.close()
casalog.post('----------------------------------------')
casalog.post("Updating the SOURCE table of" '%s' % msname)
casalog.post('----------------------------------------')
tb.open(msname + '/SOURCE', nomodify=False)
radec = tb.getcol('DIRECTION')
radec[0], radec[1] = ra, dec
tb.putcol('DIRECTION', radec)
name = np.array([source_id], dtype='|S{0}'.format(len(source_id) + 1))
tb.putcol('NAME', name)
tb.close()
casalog.post('----------------------------------------')
casalog.post("Updating the DATA_DESCRIPTION table of" '%s' % msname)
casalog.post('----------------------------------------')
tb.open(msname + '/DATA_DESCRIPTION/', nomodify=False)
pol_id = tb.getcol('POLARIZATION_ID')
pol_id *= 0
tb.putcol('POLARIZATION_ID', pol_id)
# spw_id = tb.getcol('SPECTRAL_WINDOW_ID')
# spw_id *= 0
# tb.putcol('SPECTRAL_WINDOW_ID', spw_id)
tb.close()
casalog.post('----------------------------------------')
casalog.post("Updating the POLARIZATION table of" '%s' % msname)
casalog.post('----------------------------------------')
tb.open(msname + '/POLARIZATION/', nomodify=False)
tb.removerows(rownrs=np.arange(1, nband, dtype=int))
tb.close()
casalog.post('----------------------------------------')
casalog.post("Updating the FIELD table of" '%s' % msname)
casalog.post('----------------------------------------')
tb.open(msname + '/FIELD/', nomodify=False)
delay_dir = tb.getcol('DELAY_DIR')
delay_dir[0], delay_dir[1] = ra, dec
tb.putcol('DELAY_DIR', delay_dir)
phase_dir = tb.getcol('PHASE_DIR')
phase_dir[0], phase_dir[1] = ra, dec
tb.putcol('PHASE_DIR', phase_dir)
reference_dir = tb.getcol('REFERENCE_DIR')
reference_dir[0], reference_dir[1] = ra, dec
tb.putcol('REFERENCE_DIR', reference_dir)
name = np.array([source_id], dtype='|S{0}'.format(len(source_id) + 1))
tb.putcol('NAME', name)
tb.close()
# FIELD: DELAY_DIR, PHASE_DIR, REFERENCE_DIR, NAME
del out, flag, uvwarray, uv, timearr, sigma
gc.collect() #
if not (timebin == '0s' and width == 1):
split(vis=msname,
outputvis=msname + '.split',
datacolumn='data',
timebin=timebin,
width=width,
keepflags=False)
os.system('rm -rf {}'.format(msname))
msfile.append(msname + '.split')
else:
msfile.append(msname)
casalog.post("finished in --- %s seconds ---" % (time.time() - time0))
if doconcat:
msname = list(filelist[0].split('/')[-1])
concatvis = visprefix + ''.join(msname) + '-{:d}m.ms'.format(
int(sum(time_concat)))
concat(vis=msfile, concatvis=concatvis, timesort=True)
# Change all observation ids to be the same (zero)
tb.open(concatvis + '/OBSERVATION', nomodify=False)
nobs = tb.nrows()
tb.removerows([i + 1 for i in range(nobs - 1)])
tb.close()
tb.open(concatvis, nomodify=False)
obsid = tb.getcol('OBSERVATION_ID')
newobsid = np.zeros(len(obsid), dtype='int')
tb.putcol('OBSERVATION_ID', newobsid)
tb.close()
for ll in msfile:
os.system('rm -rf {}'.format(ll))
return True
def concatSMAdataset(targetlist='allsources',
finalvis='allsources',
offset=100,
ignoreContinuumSpws=True,
timesort=True,
overwrite=True,
skipFinalConcat=False,
relabelAsTopo=True,
importdata=True):
"""
This function is meant to be run after you have run auto_fits in MIR/IDL for each source in
an SMA dataset that has been despiked and had tsys applied, and then imported each of the
resulting FITS files into measurement sets. It will produce a fully concatenated ms, by
first concatenating each spw of a target into a target ms, then concatenating all targets.
The offset parameter is used to avoid collisions in the scan numbers between targets.
targetlist: list of sourcenames (i.e. the initial characters of each per-source/per-spw input ms); either a python list or a comma-delimited list
finalvis: name of final concatenated ms
offset: first offset will be zero, subsequent will be 1*offset, 2*offset, etc.
timesort: passed to concat task for final measurement set
relabelAsTopo: default is True
Todd Hunter Nov-2019
"""
if importdata:
uvfits = glob.glob('*.UVFITS')
for f in uvfits:
if ignoreContinuumSpws and f.find('_C1_') > 0:
continue
dataset = f.lower().replace('.uvfits', '.ms.part')
importuvfits(f, dataset)
setMeasFreqRef(dataset)
if type(targetlist) == str:
targetlist = targetlist.split(',')
outputvislist = []
for i, target in enumerate(targetlist):
concatvis = target + '_concat.ms.part'
outputvis = target + '_concat_offset.ms.part'
if not os.path.exists(concatvis) or overwrite:
if os.path.exists(concatvis):
shutil.rmtree(concatvis)
if ignoreContinuumSpws:
vislist = sorted(glob.glob(target + '*_s*.ms.part'))
else:
vislist = sorted(glob.glob(target + '*.ms.part'))
print("Concatenating %d ms: %s" % (len(vislist), vislist))
concat(vislist, concatvis=concatvis, timesort=False)
setMeasFreqRef(concatvis) # relabels all spws as TOPO
if not os.path.exists(outputvis) or overwrite:
if os.path.exists(outputvis):
shutil.rmtree(outputvis)
myoffset = i * offset
print("Finished %s now offsetting by %d" % (concatvis, myoffset))
offsetScanNumbers(concatvis, offset=myoffset, outputvis=outputvis)
print("Finished ", outputvis)
outputvislist.append(outputvis)
if not skipFinalConcat:
if finalvis == '':
finalvis = 'sma.ms'
print("Doing final concatenation")
concat(outputvislist, concatvis=finalvis, timesort=timesort)
listobslist(finalvis, overwrite=True)
return finalvis
else:
return concatvis
def concateovsa(vis, concatvis, datacolumn='corrected', keep_orig_ms=True, cols2rm="model,corrected", freqtol="", dirtol="", respectname=False,
timesort=True, copypointing=True, visweightscale=[], forcesingleephemfield=""):
if concatvis[-1] == os.path.sep:
concatvis = concatvis[:-1]
if os.path.sep not in concatvis:
visprefix = './'
else:
visprefix = os.path.dirname(concatvis) + os.path.sep
msfiles = vis
msfiles_ = []
for idx, ll in enumerate(msfiles):
if str(ll).endswith('/'):
msfiles[idx] = str(ll)[:-1]
datacolumn = datacolumn.lower()
if datacolumn == 'data':
print 'DATA columns will be concatenated.'
for ll in msfiles:
clearcal(vis=str(ll), addmodel=True)
elif datacolumn == 'corrected':
# try:
print 'CORRECTED columns will be concatenated.'
tmpdir = os.path.join(visprefix, 'tmp_ms') + os.path.sep
if not os.path.exists(tmpdir):
os.makedirs(tmpdir)
for ll in msfiles:
msfile_ = os.path.join(tmpdir, os.path.basename(str(ll)))
msfiles_.append(msfile_)
split(vis=str(ll), outputvis=msfile_, datacolumn='corrected')
clearcal(vis=msfile_, addmodel=True)
else:
raise ValueError('Please set datacolumn to be "data" or "corrected"!')
if msfiles_:
concat(vis=msfiles_, concatvis=concatvis, freqtol=freqtol, dirtol=dirtol, respectname=respectname, timesort=timesort,
copypointing=copypointing, visweightscale=visweightscale, forcesingleephemfield=forcesingleephemfield)
os.system('rm -rf {}'.format(tmpdir))
else:
concat(vis=msfiles, concatvis=concatvis, freqtol=freqtol, dirtol=dirtol, respectname=respectname, timesort=timesort,
copypointing=copypointing, visweightscale=visweightscale, forcesingleephemfield=forcesingleephemfield)
# Change all observation ids to be the same (zero)
tb.open(concatvis + '/OBSERVATION', nomodify=False)
nobs = tb.nrows()
tim0 = tb.getcell('TIME_RANGE', 0)[0]
tim1 = tb.getcell('TIME_RANGE', nobs - 1)[1]
tb.removerows([i + 1 for i in range(nobs - 1)])
tb.putcell('TIME_RANGE', 0, [tim0, tim1])
tb.close()
tb.open(concatvis + '/DATA_DESCRIPTION', nomodify=False)
nrows = tb.nrows()
pol_id = tb.getcol('POLARIZATION_ID')
tb.removerows(np.where(pol_id != 0)[0])
tb.close()
tb.open(concatvis, nomodify=False)
dd_id = tb.getcol('DATA_DESC_ID')
idx_dd_id, = np.where(dd_id >= nrows / 2)
dd_id[idx_dd_id] = dd_id[idx_dd_id] - nrows / 2
tb.putcol('DATA_DESC_ID', dd_id)
tb.close()
tb.open(concatvis + '/FIELD', nomodify=False)
nobs = tb.nrows()
tb.removerows([i + 1 for i in range(nobs - 1)])
tb.close()
tb.open(concatvis + '/SOURCE', nomodify=False)
nobs = tb.nrows()
tb.removerows([i + 1 for i in range(nobs - 1)])
tb.close()
tb.open(concatvis, nomodify=False)
obsid = tb.getcol('OBSERVATION_ID')
newobsid = np.zeros(len(obsid), dtype='int')
tb.putcol('OBSERVATION_ID', newobsid)
fldid = tb.getcol('FIELD_ID')
newfldid = np.zeros(len(fldid), dtype='int')
tb.putcol('FIELD_ID', newfldid)
colnames = tb.colnames()
cols2rm = cols2rm.upper()
cols2rm = cols2rm.split(',')
for l in range(len(cols2rm)):
col = cols2rm[l] + '_DATA'
if col in colnames:
try:
tb.removecols(col)
print 'Column {} removed.'.format(col)
except:
pass
tb.close()
if msfiles_ != [] and msfiles_ != msfiles:
for ll in msfiles_:
os.system('rm -rf {}'.format(ll))
if not keep_orig_ms:
for ll in msfiles:
os.system('rm -rf {}'.format(ll))
def pimporteovsa(idbfiles,
ncpu=8,
timebin=None,
width=None,
visprefix=None,
nocreatms=False,
doconcat=False,
modelms=''):
casalog.origin('pimporteovsa')
# # Initialize the helper class
# pdh = ParallelDataHelper("importeovsa", locals())
#
# # Validate input and output parameters
# try:
# pdh.setupIO()
# except Exception, instance:
# casalog.post('%s' % instance, 'ERROR')
# return False
# import glob
# idbfiles = glob.glob('IDB*_unrot')[:4]
# ncpu = 4
# timebin = None
# width = None
# visprefix = None
# nocreatms = False
# doconcat = False
# modelms = ''
# if not (type(ncpu) is int):
# casalog.post('ncpu should be an integer')
# ncpu = 8
if type(idbfiles) == Time:
filelist = ri.get_trange_files(idbfiles)
else:
# If input type is not Time, assume that it is the list of files to read
filelist = idbfiles
if type(filelist) == str:
filelist = [filelist]
for f in filelist:
if not os.path.exists(f):
casalog.post("Some files in filelist are invalid. Aborting...")
# return False
if not visprefix:
visprefix = './'
if not timebin:
timebin = '0s'
if not width:
width = 1
if not modelms:
if nocreatms:
filename = filelist[0]
modelms = ipe.creatms(filename, visprefix)
else:
if not os.path.exists(modelms):
if nocreatms:
filename = filelist[0]
modelms = ipe.creatms(filename, visprefix)
iterable = range(len(filelist))
imppart = partial(importeovsa_iter, filelist, timebin, width, visprefix,
nocreatms, modelms)
t0 = time.time()
casalog.post('Perform importeovsa in parallel ...')
pool = mp.Pool(ncpu)
res = pool.map(imppart, iterable)
pool.close()
pool.join()
t1 = time.time()
timelapse = t1 - t0
print 'It took %f secs to complete' % timelapse
results = pd.DataFrame({'succeeded': [], 'msfile': [], 'durtim': []})
for r in res:
results = results.append(
pd.DataFrame({
'succeeded': [r[0]],
'msfile': [r[1]],
'durtim': [r[2]]
}))
if doconcat:
msname = list(os.path.basename(filelist[0]))
concatvis = visprefix + ''.join(msname) + '-{:d}m.ms'.format(
int(results['durtim'].sum()))
msfile = results.loc[results['succeeded'] == True, :].msfile.tolist()
concat(vis=msfile, concatvis=concatvis, timesort=True)
# Change all observation ids to be the same (zero)
tb.open(concatvis + '/OBSERVATION', nomodify=False)
nobs = tb.nrows()
tb.removerows([i + 1 for i in range(nobs - 1)])
tb.close()
tb.open(concatvis, nomodify=False)
obsid = tb.getcol('OBSERVATION_ID')
newobsid = np.zeros(len(obsid), dtype='int')
tb.putcol('OBSERVATION_ID', newobsid)
tb.close()
for ll in msfile:
os.system('rm -rf {}'.format(ll))
return True
def importeovsa(idbfiles, timebin=None, width=None, visprefix=None, nocreatms=True, doconcat=False):
casalog.origin('importeovsa')
# # Initialize the helper class
# pdh = ParallelDataHelper("importeovsa", locals())
#
# # Validate input and output parameters
# try:
# pdh.setupIO()
# except Exception, instance:
# casalog.post('%s' % instance, 'ERROR')
# return False
if type(idbfiles) == Time:
filelist = ri.get_trange_files(idbfiles)
else:
# If input type is not Time, assume that it is the list of files to read
filelist = idbfiles
if type(filelist) == str:
filelist = [filelist]
for f in filelist:
if not os.path.exists(f):
casalog.post("Some files in filelist are invalid. Aborting...")
return False
if not visprefix:
visprefix = './'
if not timebin:
timebin = '0s'
if not width:
width = 1
if nocreatms:
filename = filelist[0]
modelms = creatms(filename, visprefix)
msfile = []
for filename in filelist:
uv = aipy.miriad.UV(filename)
# if uv['source'].lower() == 'sun':
# visprefix = visprefix + 'sun/'
# if not os.path.exists(visprefix):
# os.mkdir(visprefix)
# else:
# visprefix = visprefix + 'calibrator/'
# if not os.path.exists(visprefix):
# os.mkdir(visprefix)
# uv.rewind()
start_time = 0 # The start and stop times are referenced to ref_time_jd in second
end_time = 600
delta_time = 1
time_steps = (end_time - start_time) / delta_time
time0 = time.time()
if 'antlist' in uv.vartable:
ants = uv['antlist']
antlist = map(int, ants.split())
else:
antlist = [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16]
good_idx = np.arange(len(uv['sfreq']))
ref_time_jd = uv['time']
ref_time_mjd = (ref_time_jd - 2400000.5) * 24. * 3600. + 0.5 * delta_time
nf = len(good_idx)
npol = uv['npol']
nants = uv['nants']
source_id = uv['source']
ra, dec = uv['ra'], uv['dec']
nbl = nants * (nants - 1) / 2
bl2ord = bl_list2(nants)
npairs = nbl + nants
flag = np.ones((npol, nf, time_steps, npairs), dtype=bool)
out = np.zeros((npol, nf, time_steps, npairs), dtype=np.complex64) # Cross-correlations
uvwarray = np.zeros((3, time_steps, npairs), dtype=np.float)
bandedge = get_band_edge(nband=34)
nband = len(bandedge) - 1
uv.rewind()
l = -1
for preamble, data in uv.all():
uvw, t, (i0, j0) = preamble
i = antlist.index(i0 + 1)
j = antlist.index(j0 + 1)
if i > j:
# Reverse order of indices
j = antlist.index(i0 + 1)
i = antlist.index(j0 + 1)
# Assumes uv['pol'] is one of -5, -6, -7, -8
k = -5 - uv['pol']
l += 1
out[k, :, l / (npairs * npol), bl2ord[i0, j0]] = data.data
flag[k, :, l / (npairs * npol), bl2ord[i0, j0]] = data.mask
if i != j:
if k == 3:
uvwarray[:, l / (npairs * npol), bl2ord[i0, j0]] = -uvw * constants.speed_of_light / 1e9
nrows = time_steps * npairs
out = out.reshape(npol, nf, nrows)
flag = flag.reshape(npol, nf, nrows)
uvwarray = uvwarray.reshape(3, nrows)
uvwarray = np.tile(uvwarray, (1, nband))
sigma = np.ones((4, nrows), dtype=np.float) + 1
sigma = np.tile(sigma, (1, nband))
casalog.post('IDB File {0} is readed in --- {1:10.2f} seconds ---'.format(filename, (time.time() - time0)))
msname = list(filename.split('/')[-1])
msname.insert(11, 'T')
msname = visprefix + source_id.upper() + '_' + ''.join(msname[3:]) + '-10m.ms'
if not nocreatms:
modelms = creatms(filename, visprefix)
os.system('mv {} {}'.format(modelms, msname))
else:
casalog.post('----------------------------------------')
casalog.post('copying standard MS to {0}'.format(msname, (time.time() - time0)))
casalog.post('----------------------------------------')
os.system("rm -fr %s" % msname)
os.system("cp -r " + " %s" % modelms + " %s" % msname)
casalog.post(
'Standard MS is copied to {0} in --- {1:10.2f} seconds ---'.format(msname, (time.time() - time0)))
tb.open(msname, nomodify=False)
casalog.post('----------------------------------------')
casalog.post("Updating the main table of" '%s' % msname)
casalog.post('----------------------------------------')
for l, bdedge in enumerate(bandedge[:-1]):
time1 = time.time()
nchannels = (bandedge[l + 1] - bandedge[l])
for row in range(nrows):
tb.putcell('DATA', (row + l * nrows), out[:, bandedge[l]:bandedge[l + 1], row])
tb.putcell('FLAG', (row + l * nrows), flag[:, bandedge[l]:bandedge[l + 1], row])
casalog.post('---spw {0:02d} is updated in --- {1:10.2f} seconds ---'.format((l + 1), time.time() - time1))
tb.putcol('UVW', uvwarray)
tb.putcol('SIGMA', sigma)
tb.putcol('WEIGHT', 1.0 / sigma ** 2)
timearr = np.arange((time_steps), dtype=np.float)
timearr = timearr.reshape(1, time_steps, 1)
timearr = np.tile(timearr, (nband, 1, npairs))
timearr = timearr.reshape(nband * npairs * time_steps) + ref_time_mjd
tb.putcol('TIME', timearr)
tb.putcol('TIME_CENTROID', timearr)
scan_id = tb.getcol('SCAN_NUMBER')
scan_id *= 0
tb.putcol('SCAN_NUMBER', scan_id)
colnames = tb.colnames()
cols2rm = ["MODEL_DATA", "CORRECTED_DATA"]
for l in range(len(cols2rm)):
if cols2rm[l] in colnames:
tb.removecols(cols2rm[l])
tb.close()
casalog.post('----------------------------------------')
casalog.post("Updating the OBSERVATION table of" '%s' % msname)
casalog.post('----------------------------------------')
tb.open(msname + '/OBSERVATION', nomodify=False)
tb.putcol('TIME_RANGE',
np.asarray([ref_time_mjd - 0.5 * delta_time, ref_time_mjd + end_time - 0.5 * delta_time]).reshape(
2, 1))
tb.putcol('OBSERVER', ['EOVSA team'])
tb.close()
casalog.post('----------------------------------------')
casalog.post("Updating the POINTING table of" '%s' % msname)
casalog.post('----------------------------------------')
tb.open(msname + '/POINTING', nomodify=False)
timearr = np.arange((time_steps), dtype=np.float).reshape(1, time_steps, 1)
timearr = np.tile(timearr, (nband, 1, nants))
timearr = timearr.reshape(nband * time_steps * nants) + ref_time_mjd
tb.putcol('TIME', timearr)
tb.putcol('TIME_ORIGIN', timearr - 0.5 * delta_time)
direction = tb.getcol('DIRECTION')
direction[0, 0, :] = ra
direction[1, 0, :] = dec
tb.putcol('DIRECTION', direction)
target = tb.getcol('TARGET')
target[0, 0, :] = ra
target[1, 0, :] = dec
tb.putcol('TARGET', target)
tb.close()
casalog.post('----------------------------------------')
casalog.post("Updating the SOURCE table of" '%s' % msname)
casalog.post('----------------------------------------')
tb.open(msname + '/SOURCE', nomodify=False)
radec = tb.getcol('DIRECTION')
radec[0], radec[1] = ra, dec
tb.putcol('DIRECTION', radec)
name = np.array([source_id], dtype='|S{0}'.format(len(source_id) + 1))
tb.putcol('NAME', name)
tb.close()
casalog.post('----------------------------------------')
casalog.post("Updating the DATA_DESCRIPTION table of" '%s' % msname)
casalog.post('----------------------------------------')
tb.open(msname + '/DATA_DESCRIPTION/', nomodify=False)
pol_id = tb.getcol('POLARIZATION_ID')
pol_id *= 0
tb.putcol('POLARIZATION_ID', pol_id)
# spw_id = tb.getcol('SPECTRAL_WINDOW_ID')
# spw_id *= 0
# tb.putcol('SPECTRAL_WINDOW_ID', spw_id)
tb.close()
casalog.post('----------------------------------------')
casalog.post("Updating the POLARIZATION table of" '%s' % msname)
casalog.post('----------------------------------------')
tb.open(msname + '/POLARIZATION/', nomodify=False)
tb.removerows(rownrs=np.arange(1, nband, dtype=int))
tb.close()
casalog.post('----------------------------------------')
casalog.post("Updating the FIELD table of" '%s' % msname)
casalog.post('----------------------------------------')
tb.open(msname + '/FIELD/', nomodify=False)
delay_dir = tb.getcol('DELAY_DIR')
delay_dir[0], delay_dir[1] = ra, dec
tb.putcol('DELAY_DIR', delay_dir)
phase_dir = tb.getcol('PHASE_DIR')
phase_dir[0], phase_dir[1] = ra, dec
tb.putcol('PHASE_DIR', phase_dir)
reference_dir = tb.getcol('REFERENCE_DIR')
reference_dir[0], reference_dir[1] = ra, dec
tb.putcol('REFERENCE_DIR', reference_dir)
name = np.array([source_id], dtype='|S{0}'.format(len(source_id) + 1))
tb.putcol('NAME', name)
tb.close()
# FIELD: DELAY_DIR, PHASE_DIR, REFERENCE_DIR, NAME
del out, flag, uvwarray, uv, timearr, sigma
gc.collect() #
if not (timebin == '0s' and width == 1):
split(vis=msname, outputvis=msname + '.split', datacolumn='data', timebin=timebin, width=width)
os.system('rm -rf {}'.format(msname))
msfile.append(msname + '.split')
else:
msfile.append(msname)
casalog.post("finished in --- %s seconds ---" % (time.time() - time0))
# msname = visprefix + source_id.upper() + '_' + ''.join(msname[3:]) + '-10m.ms'
if doconcat:
msname = list(filelist[0].split('/')[-1])
msname.insert(11, 'T')
concatvis = visprefix + ''.join(msname[3:]) + '-{}m.ms'.format(10.0 * len(msfile))
concat(vis=msfile, concatvis=concatvis)
for ll in msfile:
os.system('rm -rf {}'.format(ll))
return True
msmd.open(outputvis)
ref0 = msmd.reffreq(0)
msmd.close()
if ref0['refer'] == 'TOPO':
mst.open(outputvis, nomodify=False)
mst.regridspw(outframe='LSRK',
mode='vrad',
restfreq=23.722633335e9)
mst.close()
splitvises.append(outputvis)
concat(
vis=splitvises,
concatvis=merged_ms,
)
imagename = 'NH322_zoom_on_SgrB2M_40to80kms'
if not os.path.exists(imagename + ".image.pbcor.fits"):
tclean(
vis=merged_ms,
imagename=imagename,
imsize=500,
cell='0.02arcsec',
field=field,
savemodel='modelcolumn',
restfreq=restfreq,
nchan=50, # 0.8 km/s channels
start='40km/s',
phasecenter='J2000 17h47m20.178 -28d23m04.109',