def checkMS(msFile, cont=True):
""" get the number of spw and channels in the measurement set
for continuum: combine into 1 channel and spw before running uvmcmcfit
Parameters
----------
msFile: string
CASA measurement set
Returns
-------
nspw:
number of spw
nchan:
number of channels
"""
from taskinit import ms
ms.open(msFile)
metadata = ms.metadata()
ms.done()
nspw = metadata.nspw()
for i in range(nspw):
# number of channels in each spw
nchan = metadata.nchan(spw)
metadata.done()
return nspw, nchan
def MSpcd2(msFile):
"""using CASA taskinit.ms
Parameters
----------
msFile: string
measurement set filename
Returns
-------
pcd:
phase center
Note
----
may need further modification if nField > 1
unless it is the same for all fields
(need further investigation)
"""
from taskinit import ms
ms.open(msFile)
pc = ms.getfielddirmeas()
if not isinstance(pc, dict) is True:
pc()
epoch = pc['refer']
pcd_dec = pc['m1']['value'] * 180 / numpy.pi
pcd_ra = pc['m0']['value'] * 180 / numpy.pi
if pcd_ra < 0:
pcd_ra += 360
ms.done()
pcd = [pcd_ra, pcd_dec]
return pcd
def get_phs_center(msfile):
""" get the phase center of the MS file using CASA taskinit.ms
Parameters
----------
msfile: str
measurement set filename
Returns
-------
pcd:
phase center
Note
----
may need further modification if nField > 1 unless it is the same for all fields (need further investigation)
"""
from taskinit import ms
ms.open(msfile)
pc = ms.getfielddirmeas()
if not isinstance(pc, dict) is True:
pc()
epoch = pc['refer']
pcd_dec = pc['m1']['value'] * 180 / np.pi
pcd_ra = pc['m0']['value'] * 180 / np.pi
if pcd_ra < 0:
pcd_ra += 360
ms.done()
pcd = [pcd_ra, pcd_dec]
return pcd
def ant_trange(vis):
''' Figure out nominal times for tracking of old EOVSA antennas, and return time
range in CASA format
'''
import eovsa_array as ea
from astropy.time import Time
from taskinit import ms
# Get timerange from the visibility file
# msinfo = dict.fromkeys(['vis', 'scans', 'fieldids', 'btimes', 'btimestr', 'inttimes', 'ras', 'decs', 'observatory'])
ms.open(vis)
# metadata = ms.metadata()
scans = ms.getscansummary()
sk = np.sort(scans.keys())
vistrange = np.array([scans[sk[0]]['0']['BeginTime'],scans[sk[-1]]['0']['EndTime']])
# Get the Sun transit time, based on the date in the vis file name (must have UDByyyymmdd in the name)
aa = ea.eovsa_array()
date = vis.split('UDB')[-1][:8]
slashdate = date[:4] + '/' + date[4:6] + '/' + date[6:8]
aa.date = slashdate
sun = aa.cat['Sun']
mjd_transit = Time(aa.next_transit(sun).datetime(), format='datetime').mjd
# Construct timerange limits based on +/- 3h55m from transit time (when all dishes are nominally tracking)
# and clip the visibility range not to exceed those limits
mjdrange = np.clip(vistrange, mjd_transit - 0.1632, mjd_transit + 0.1632)
trange = Time(mjdrange[0], format='mjd').iso[:19] + '~' + Time(mjdrange[1], format='mjd').iso[:19]
trange = trange.replace('-', '/').replace(' ', '/')
return trange
def _checkfile(filename, datacolumn):
import re
# Currently this supports only ms files
# As such there is no reason to check filetype.
# If it cannot be opened as ms it will not be supported.
# if re.match('^.*[mM][sS]/*$', filename) is not None:
try:
from taskinit import ms
ms.open(filename)
ms.done()
except ImportError:
# This probably means that it was run from a pure python session.
# We will relegate any checks that it is a valid ms file to the
# stacker.
if not os.access(filename, os.F_OK):
raise RuntimeError(
'Could not find data file "{}".'.format(filename))
filename = filename
filetype = FILE_TYPE_MS
fileoptions = 0
if datacolumn == 'data':
fileoptions = MS_DATACOLUMN_DATA
elif datacolumn == 'model' or datacolumn == 'model_data':
fileoptions = MS_MODELCOLUMN_DATA
# elif re.match('^.*[fF][iI][tT][sS]$', filename) is not None:
# raise NotImplementedError('FITS format is currently not supported.')
return filetype, filename, fileoptions
def time2filename(msfile, timerange='', spw=''):
from astropy.time import Time
tb.open(msfile)
starttim = Time(tb.getcell('TIME', 0) / 24. / 3600., format='mjd')
endtim = Time(tb.getcell('TIME', tb.nrows() - 1) / 24. / 3600., format='mjd')
tb.close()
datstr = starttim.iso[:10]
ms.open(msfile)
metadata = ms.metadata()
observatory = metadata.observatorynames()[0]
ms.close()
if timerange is None or timerange == '':
starttim1 = starttim
endtim1 = endtim
else:
(tstart, tend) = timerange.split('~')
if tstart[2] == ':':
starttim1 = Time(datstr + 'T' + tstart)
endtim1 = Time(datstr + 'T' + tend)
else:
starttim1 = Time(qa.quantity(tstart, 'd')['value'], format='mjd')
endtim1 = Time(qa.quantity(tend, 'd')['value'], format='mjd')
midtime = Time((starttim1.mjd + endtim1.mjd) / 2., format='mjd')
tstr = midtime.to_datetime().strftime('{}_%Y%m%dT%H%M%S'.format(observatory))
if spw:
spstr = 'spw{}'.format(spw.replace('~', '-'))
filename = '.'.join([tstr, spstr])
else:
filename = tstr
return filename
def getspwfromfreq(vis, freqrange):
from taskinit import ms
ms.open(vis)
axisInfo = ms.getdata(["axis_info"], ifraxis=True)
spwInfo = ms.getspectralwindowinfo()
freqInfo = axisInfo["axis_info"]["freq_axis"]["chan_freq"].swapaxes(0, 1) / 1e9
freqInfo_ravel = freqInfo.ravel()
timeInfo = axisInfo["axis_info"]["time_axis"]['MJDseconds']
mstimran = ms.range(["time"])
ms.close()
freq0, freq1 = freqrange.split(' ')[0].split('~')
freq0, freq1 = float(freq0), float(freq1)
for ll in [freq0, freq1]:
if not freqInfo_ravel[0] <= ll <= freqInfo_ravel[-1]:
raise ValueError('Selected frequency out of range!!!')
freqIdx0 = np.where(freqInfo == freq0)
freqIdx1 = np.where(freqInfo == freq1)
sz_freqInfo = freqInfo.shape
ms_spw = ['{}'.format(ll) for ll in xrange(freqIdx0[0], freqIdx1[0] + 1)]
if len(ms_spw) == 1:
ms_chan = ['{}~{}'.format(freqIdx0[1][0], freqIdx1[1][0])]
else:
ms_chan = ['{}~{}'.format(freqIdx0[1][0], sz_freqInfo[1] - 1)] + ['0~{}'.format(sz_freqInfo[1] - 1) for ll in
xrange(freqIdx0[0] + 1, freqIdx1[0])]
ms_chan.append('0~{}'.format(freqIdx1[1][0]))
spw = ','.join('{}:{}'.format(t[0], t[1]) for t in zip(ms_spw, ms_chan))
return spw
def read_ms(vis):
''' Read a CASA ms file and return a dictionary of amplitude, phase, uvdistance,
uvangle, frequency (GHz) and time (MJD). Currently only returns the XX IF channel.
vis Name of the visibility (ms) folder
'''
ms.open(vis)
spwinfo = ms.getspectralwindowinfo()
nspw = len(spwinfo.keys())
for i in range(nspw):
print('Working on spw', i)
ms.selectinit(datadescid=0, reset=True)
ms.selectinit(datadescid=i)
if i == 0:
spw = ms.getdata(['amplitude', 'phase', 'u', 'v', 'axis_info'], ifraxis=True)
xxamp = spw['amplitude']
xxpha = spw['phase']
fghz = spw['axis_info']['freq_axis']['chan_freq'][:, 0] / 1e9
band = np.ones_like(fghz) * i
mjd = spw['axis_info']['time_axis']['MJDseconds'] / 86400.
uvdist = np.sqrt(spw['u'] ** 2 + spw['v'] ** 2)
uvang = np.angle(spw['u'] + 1j * spw['v'])
else:
spw = ms.getdata(['amplitude', 'phase', 'axis_info'], ifraxis=True)
xxamp = np.concatenate((xxamp, spw['amplitude']), 1)
xxpha = np.concatenate((xxpha, spw['phase']), 1)
fg = spw['axis_info']['freq_axis']['chan_freq'][:, 0] / 1e9
fghz = np.concatenate((fghz, fg))
band = np.concatenate((band, np.ones_like(fg) * i))
ms.close()
return {'amp': xxamp, 'phase': xxpha, 'fghz': fghz, 'band': band, 'mjd': mjd, 'uvdist': uvdist, 'uvangle': uvang}
def make_pbfile(vis, pbfile):
from taskinit import im, ms, ia, qa, tb
import numpy as np
from scipy.constants import c
ms.open(vis)
fields = ms.range('field_id')['field_id']
ms.done()
im.open(vis)
im.selectvis(field=fields[0])
ms.open(vis)
freq = np.mean(ms.range('chan_freq')['chan_freq'])
phase_dir = ms.range('phase_dir')['phase_dir']['direction']
ms.done()
phase_dir = phase_dir[0][0], phase_dir[1][0]
phase_dir = [
qa.formxxx(str(phase_dir[0]) + 'rad', format='hms'),
qa.formxxx(str(phase_dir[1]) + 'rad', format='dms')
]
phase_dir = 'J2000 ' + ' '.join(phase_dir)
tb.open(vis + '/ANTENNA/')
dishdia = np.min(tb.getcol('DISH_DIAMETER'))
tb.done()
# pb of 512 pix cover pb down to 0.001
# ensure largest pixel to pixel var to .01
minpb = 0.001
nx = 512
cellconv = (nx * np.sqrt(np.log(2) / np.log(1 / minpb)))**-1
beam = c / freq / dishdia
cell = {}
cell['value'] = beam * cellconv
cell['unit'] = 'rad'
# nx = int(3*3e8/freq/dishdia*1.22*180/
# math.pi*3600/qa.convert(advise['cell'],
# 'arcsec')['value'])
# Chosen as to be 3 times fwhm of primary beam,
# should include up to approximately .01 of peak flux
im.defineimage(nx=nx, ny=nx, cellx=cell, celly=cell, phasecenter=phase_dir)
im.setvp(dovp=True)
im.makeimage(type='pb', image=pbfile)
im.done()
ia.open(pbfile)
cs = ia.coordsys()
cs.setreferencevalue(type='direction', value=[0., 0.])
ia.setcoordsys(cs.torecord())
ia.maskhandler('delete', 'mask0')
ia.done()
def insertdiskmodel(vis, sizescale=1.0, fdens=None, dsize=None, xmlfile='SOLDISK.xml', writediskinfoonly=False,
active=False, overwrite=True):
# Apply size scale adjustment (default is no adjustment)
for i in range(len(dsize)):
num, unit = dsize[i].split('arc')
dsize[i] = str(float(num) * sizescale)[:6] + 'arc' + unit
msfile = vis
ms.open(msfile)
spwinfo = ms.getspectralwindowinfo()
nspw = len(spwinfo.keys())
ms.close()
diskimdir = 'diskim/'
if not os.path.exists(diskimdir):
os.makedirs(diskimdir)
frq = []
spws = range(nspw)
for sp in spws:
spw = spwinfo[str(sp)]
frq.append('{:.4f}GHz'.format((spw['RefFreq'] + spw['TotalWidth'] / 2.0) / 1e9))
frq = np.array(frq)
writediskxml(dsize, fdens, frq, xmlfile=xmlfile)
if not writediskinfoonly:
tb.open(msfile + '/FIELD')
phadir = tb.getcol('PHASE_DIR').flatten()
tb.close()
ra = phadir[0]
dec = phadir[1]
direction = 'J2000 ' + str(ra) + 'rad ' + str(dec) + 'rad'
diskim = []
for sp in tqdm(spws, desc='Generating {} disk models'.format(nspw), ascii=True):
diskim.append(
mk_diskmodel(outname=diskimdir + 'disk{:02d}_'.format(sp), bdwidth=spwinfo[str(sp)],
direction=direction,
reffreq=frq[sp],
flux=fdens[sp], eqradius=dsize[sp], polradius=dsize[sp], overwrite=overwrite))
if not active:
delmod(msfile, otf=True, scr=True)
for sp in tqdm(spws, desc='Inserting disk model', ascii=True):
ft(vis=msfile, spw=str(sp), field='', model=str(diskim[sp]), nterms=1,
reffreq="", complist="", incremental=True, usescratch=True)
return msfile, diskim
def guesspb(vis):
"""
Produces a PrimaryBeamModel from a measurementset
"""
print(vis)
from taskinit import ms, tb, qa
ms.open(vis)
freq = (np.mean(ms.range('chan_freq')['chan_freq']) / 1e9)
ms.done()
tb.open(vis + '/OBSERVATION')
telescope = tb.getcol('TELESCOPE_NAME')[0]
tb.done()
pbfile = '{0}-{1:.1f}GHz.pb'.format(telescope, freq)
print(pbfile)
if not os.access(pbfile, os.F_OK):
stacker.make_pbfile(vis, pbfile)
return MSPrimaryBeamModel(pbfile)
def read_msinfo(vis=None, msinfofile=None):
# read MS information #
msinfo = dict.fromkeys(['vis', 'scans', 'fieldids', 'btimes', 'btimestr', \
'inttimes', 'ras', 'decs'])
ms.open(vis)
scans = ms.getscansummary()
scanids = sorted(scans.keys(), key=lambda x: int(x))
nscanid = len(scanids)
btimes = []
btimestr = []
etimes = []
fieldids = []
inttimes = []
dirs = []
ras = []
decs = []
for i in range(nscanid):
btimes.append(scans[scanids[i]]['0']['BeginTime'])
etimes.append(scans[scanids[i]]['0']['EndTime'])
fieldid = scans[scanids[i]]['0']['FieldId']
fieldids.append(fieldid)
dir = ms.getfielddirmeas('PHASE_DIR', fieldid)
dirs.append(dir)
ras.append(dir['m0'])
decs.append(dir['m1'])
inttimes.append(scans[scanids[i]]['0']['IntegrationTime'])
ms.close()
btimestr = [qa.time(qa.quantity(btimes[i], 'd'), form='fits', prec=10)[0] \
for i in range(nscanid)]
msinfo['vis'] = vis
msinfo['scans'] = scans
msinfo['fieldids'] = fieldids
msinfo['btimes'] = btimes
msinfo['btimestr'] = btimestr
msinfo['inttimes'] = inttimes
msinfo['ras'] = ras
msinfo['decs'] = decs
if msinfofile:
np.savez(msinfofile, vis=vis, scans=scans, fieldids=fieldids, \
btimes=btimes, btimestr=btimestr, inttimes=inttimes, \
ras=ras, decs=decs)
return msinfo
def read_horizons(vis):
import urllib2
import ssl
if not os.path.exists(vis):
print 'Input ms data ' + vis + ' does not exist! '
return -1
try:
# ms.open(vis)
# summary = ms.summary()
# ms.close()
# btime = Time(summary['BeginTime'], format='mjd')
# etime = Time(summary['EndTime'], format='mjd')
## alternative way to avoid conflicts with importeovsa, if needed -- more time consuming
ms.open(vis)
metadata = ms.metadata()
if metadata.observatorynames()[0] == 'EVLA':
observatory_code = '-5'
elif metadata.observatorynames()[0] == 'EOVSA':
observatory_code = '-81'
elif metadata.observatorynames()[0] == 'ALMA':
observatory_code = '-7'
ms.close()
tb.open(vis)
btime = Time(tb.getcell('TIME', 0) / 24. / 3600., format='mjd')
etime = Time(tb.getcell('TIME',
tb.nrows() - 1) / 24. / 3600.,
format='mjd')
tb.close()
print "Beginning time of this scan " + btime.iso
print "End time of this scan " + etime.iso
cmdstr = "http://ssd.jpl.nasa.gov/horizons_batch.cgi?batch=l&TABLE_TYPE='OBSERVER'&QUANTITIES='1,17,20'&CSV_FORMAT='YES'&ANG_FORMAT='DEG'&CAL_FORMAT='BOTH'&SOLAR_ELONG='0,180'&CENTER='{}@399'&COMMAND='10'&START_TIME='".format(
observatory_code
) + btime.iso.replace(
' ', ','
) + "'&STOP_TIME='" + etime.iso[:-4].replace(
' ', ','
) + "'&STEP_SIZE='1 m'&SKIP_DAYLT='NO'&EXTRA_PREC='YES'&APPARENT='REFRACTED'"
try:
context = ssl._create_unverified_context()
f = urllib2.urlopen(cmdstr, context=context)
except:
f = urllib2.urlopen(cmdstr)
except:
print 'error in reading ms file: ' + vis + ' to obtain the ephemeris!'
return -1
# inputs:
# ephemfile:
# OBSERVER output from JPL Horizons for topocentric coordinates with for example
# target=Sun, observer=VLA=-5
# extra precision, quantities 1,17,20, REFRACTION
# routine goes through file to find $$SOE which is start of ephemeris and ends with $$EOE
# outputs: a Python dictionary containing the following:
# timestr: date and time as a string
# time: modified Julian date
# ra: right ascention, in rad
# dec: declination, in rad
# rastr: ra in string
# decstr: dec in string
# p0: solar p angle, CCW with respect to the celestial north pole
# delta: distance from the disk center to the observer, in AU
# delta_dot: time derivative of delta, in the light of sight direction. Negative means it is moving toward the observer
#
# initialize the return dictionary
ephem0 = dict.fromkeys(['time', 'ra', 'dec', 'delta', 'p0'])
lines = f.readlines()
f.close()
nline = len(lines)
istart = 0
for i in range(nline):
line = lines[i]
if line[0:5] == '$$SOE': # start recording
istart = i + 1
if line[0:5] == '$$EOE': # end recording
iend = i
newlines = lines[istart:iend]
nrec = len(newlines)
ephem_ = []
t = []
ra = []
dec = []
p0 = []
delta = []
for line in newlines:
items = line.split(',')
# t.append({'unit':'mjd','value':Time(float(items[1]),format='jd').mjd})
# ra.append({'unit': 'rad', 'value': np.radians(float(items[4]))})
# dec.append({'unit': 'rad', 'value': np.radians(float(items[5]))})
# p0.append({'unit': 'deg', 'value': float(items[6])})
# delta.append({'unit': 'au', 'value': float(items[8])})
t.append(Time(float(items[1]), format='jd').mjd)
ra.append(np.radians(float(items[4])))
dec.append(np.radians(float(items[5])))
p0.append(float(items[6]))
delta.append(float(items[8]))
# convert list of dictionary to a dictionary of arrays
ephem = {'time': t, 'ra': ra, 'dec': dec, 'p0': p0, 'delta': delta}
return ephem
def get_dspec(vis=None, savespec=True, specfile=None, bl='', uvrange='', field='', scan='', datacolumn='data',
domedian=False, timeran=None, spw=None, timebin='0s', regridfreq=False, fillnan=None, verbose=False):
# from split_cli import split_cli as split
if vis.endswith('/'):
vis = vis[:-1]
msfile = vis
if not spw:
spw = ''
if not timeran:
timeran = ''
if not bl:
bl = ''
if domedian:
if not uvrange:
uvrange = '0.2~0.8km'
bl = ''
else:
uvrange = ''
# Open the ms and plot dynamic spectrum
if verbose:
print('Splitting selected data...')
vis_spl = './tmpms.splitted'
if os.path.exists(vis_spl):
os.system('rm -rf ' + vis_spl)
# split(vis=msfile, outputvis=vis_spl, timerange=timeran, antenna=bl, field=field, scan=scan, spw=spw,
# uvrange=uvrange, timebin=timebin, datacolumn=datacolumn)
ms.open(msfile, nomodify=False)
ms.split(outputms=vis_spl, whichcol=datacolumn, time=timeran, spw=spw, baseline=bl, field=field, scan=scan,
uvrange=uvrange, timebin=timebin)
ms.close()
if verbose:
print('Regridding into a single spectral window...')
# print('Reading data spw by spw')
try:
tb.open(vis_spl + '/POLARIZATION')
corrtype = tb.getcell('CORR_TYPE', 0)
pols = [stokesenum[p] for p in corrtype]
tb.close()
except:
pols = []
if regridfreq:
ms.open(vis_spl, nomodify=False)
ms.cvel(outframe='LSRK', mode='frequency', interp='nearest')
ms.selectinit(datadescid=0, reset=True)
data = ms.getdata(['amplitude', 'time', 'axis_info'], ifraxis=True)
specamp = data['amplitude']
freq = data['axis_info']['freq_axis']['chan_freq']
else:
ms.open(vis_spl)
ms.selectinit(datadescid=0, reset=True)
spwinfo = ms.getspectralwindowinfo()
specamp = []
freq = []
time = []
for descid in range(len(spwinfo.keys())):
ms.selectinit(datadescid=0, reset=True)
ms.selectinit(datadescid=descid)
data = ms.getdata(['amplitude', 'time', 'axis_info'], ifraxis=True)
specamp_ = data['amplitude']
freq_ = data['axis_info']['freq_axis']['chan_freq']
time_ = data['time']
if fillnan is not None:
flag_ = ms.getdata(['flag', 'time', 'axis_info'], ifraxis=True)['flag']
if type(fillnan) in [int, float, long]:
specamp_[flag_] = float(fillnan)
else:
specamp_[flag_] = 0.0
specamp.append(specamp_)
freq.append(freq_)
time.append(time_)
specamp = np.concatenate(specamp, axis=1)
freq = np.concatenate(freq, axis=0)
ms.selectinit(datadescid=0, reset=True)
ms.close()
os.system('rm -rf ' + vis_spl)
(npol, nfreq, nbl, ntim) = specamp.shape
freq = freq.reshape(nfreq)
if verbose:
print('npol, nfreq, nbl, ntime:', (npol, nfreq, nbl, ntim))
spec = np.swapaxes(specamp, 2, 1)
tim = data['time']
if domedian:
if verbose:
print('doing median of all the baselines')
# mask zero values before median
spec_masked = np.ma.masked_where(spec < 1e-9, spec)
spec_med = np.ma.filled(np.ma.median(spec_masked, axis=1), fill_value=0.)
nbl = 1
ospec = spec_med.reshape((npol, nbl, nfreq, ntim))
else:
ospec = spec
# Save the dynamic spectral data
if savespec:
if not specfile:
specfile = msfile + '.dspec.npz'
if os.path.exists(specfile):
os.system('rm -rf ' + specfile)
np.savez(specfile, spec=ospec, tim=tim, freq=freq,
timeran=timeran, spw=spw, bl=bl, uvrange=uvrange, pol=pols)
if verbose:
print('Median dynamic spectrum saved as: ' + specfile)
return {'spec': ospec, 'tim': tim, 'freq': freq, 'timeran': timeran, 'spw': spw, 'bl': bl, 'uvrange': uvrange,
'pol': pols}
def get_dspec(vis=None, savespec=True, specfile=None, bl='', uvrange='', field='', scan='', datacolumn='data',
domedian=False, timeran=None, spw=None, timebin='0s', regridfreq=False, fillnan=None, verbose=False,
usetbtool=False):
# from split_cli import split_cli as split
if vis.endswith('/'):
vis = vis[:-1]
msfile = vis
if not spw:
spw = ''
if not timeran:
timeran = ''
if domedian:
if not uvrange:
uvrange = '0.2~0.8km'
# bl = ''
else:
uvrange = ''
if not bl:
bl = ''
else:
uvrange = ''
# Open the ms and plot dynamic spectrum
if verbose:
print('Splitting selected data...')
if usetbtool:
try:
tb.open(vis + '/POLARIZATION')
corrtype = tb.getcell('CORR_TYPE', 0)
pols = [stokesenum[p] for p in corrtype]
tb.close()
except:
pols = []
antmask = []
if uvrange is not '' or bl is not '':
ms.open(vis)
ms.selectinit(datadescid=0)
mdata = ms.metadata()
antlist = mdata.antennaids()
mdata.done()
staql = {'uvdist': uvrange, 'baseline': bl, 'spw': spw, 'field': field, 'scan': scan, 'timerange': timeran}
### todo the selection only works for uvrange and bl. To make the selection of other items works,
## I need to make mask for other items.
a = ms.msselect(staql)
mdata = ms.metadata()
baselines = mdata.baselines()
for lidx, l in enumerate(antlist):
antmask.append(baselines[l][antlist[lidx:]])
antmask = np.hstack(antmask)
mdata.done()
ms.close()
tb.open(vis)
spwtb = tbtool()
spwtb.open(vis + '/SPECTRAL_WINDOW')
ptb = tbtool()
ptb.open(vis + '/POLARIZATION')
ms.open(vis)
spwlist = []
mdata = ms.metadata()
nspw = mdata.nspw()
nbl = mdata.nbaselines() + mdata.nantennas()
nscans = mdata.nscans()
spw_nfrq = [] # List of number of frequencies in each spw
for i in range(nspw):
spw_nfrq.append(mdata.nchan(i))
spw_nfrq = np.array(spw_nfrq)
nf = np.sum(spw_nfrq)
smry = mdata.summary()
scan_ntimes = [] # List of number of times in each scan
for iscan in range(nscans):
scan_ntimes.append(
smry['observationID=0']['arrayID=0']['scan=' + str(iscan)]['fieldID=0']['nrows'] / nspw / nbl)
scan_ntimes = np.array(scan_ntimes)
scan_ntimes_integer = scan_ntimes.astype(np.int)
if len(np.where(scan_ntimes % scan_ntimes_integer != 0)[0]) != 0:
# if True:
scan_ntimes = [] # List of number of times in each scan
for iscan in range(nscans):
scan_ntimes.append(
len(smry['observationID=0']['arrayID=0']['scan=' + str(iscan)]['fieldID=0'].keys()) - 6)
scan_ntimes = np.array(scan_ntimes)
else:
scan_ntimes = scan_ntimes_integer
nt = np.sum(scan_ntimes)
times = tb.getcol('TIME')
if times[nbl] - times[0] != 0:
# This is frequency/scan sort order
order = 'f'
elif times[nbl * nspw - 1] - times[0] != 0:
# This is time sort order
order = 't'
npol = ptb.getcol('NUM_CORR', 0, 1)[0]
ptb.close()
freq = np.zeros(nf, float)
times = np.zeros(nt, float)
if order == 't':
specamp = np.zeros((npol, nf, nbl, nt), np.complex)
for j in range(nt):
fptr = 0
# Loop over spw
for i in range(nspw):
# Get channel frequencies for this spw (annoyingly comes out as shape (nf, 1)
cfrq = spwtb.getcol('CHAN_FREQ', i, 1)[:, 0]
if j == 0:
# Only need this the first time through
spwlist += [i] * len(cfrq)
if i == 0:
times[j] = tb.getcol('TIME', nbl * (i + nspw * j), 1) # Get the time
spec_ = tb.getcol('DATA', nbl * (i + nspw * j), nbl) # Get complex data for this spw
flag = tb.getcol('FLAG', nbl * (i + nspw * j), nbl) # Get flags for this spw
nfrq = len(cfrq)
# Apply flags
if type(fillnan) in [int, float]:
spec_[flag] = float(fillnan)
else:
spec_[flag] = 0.0
# Insert data for this spw into larger array
specamp[:, fptr:fptr + nfrq, :, j] = spec_
freq[fptr:fptr + nfrq] = cfrq
fptr += nfrq
else:
specf = np.zeros((npol, nf, nt, nbl), np.complex) # Array indexes are swapped
iptr = 0
for j in range(nscans):
# Loop over scans
for i in range(nspw):
# Loop over spectral windows
s = scan_ntimes[j]
f = spw_nfrq[i]
s1 = np.sum(scan_ntimes[:j]) # Start time index
s2 = np.sum(scan_ntimes[:j + 1]) # End time index
f1 = np.sum(spw_nfrq[:i]) # Start freq index
f2 = np.sum(spw_nfrq[:i + 1]) # End freq index
spec_ = tb.getcol('DATA', iptr, nbl * s)
flag = tb.getcol('FLAG', iptr, nbl * s)
if j == 0:
cfrq = spwtb.getcol('CHAN_FREQ', i, 1)[:, 0]
freq[f1:f2] = cfrq
spwlist += [i] * len(cfrq)
times[s1:s2] = tb.getcol('TIME', iptr, nbl * s).reshape(s, nbl)[:, 0] # Get the times
iptr += nbl * s
# Apply flags
if type(fillnan) in [int, float]:
spec_[flag] = float(fillnan)
else:
spec_[flag] = 0.0
# Insert data for this spw into larger array
specf[:, f1:f2, s1:s2] = spec_.reshape(npol, f, s, nbl)
# Swap the array indexes back to the desired order
specamp = np.swapaxes(specf, 2, 3)
tb.close()
spwtb.close()
ms.close()
if len(antmask) > 0:
specamp = specamp[:, :, np.where(antmask)[0], :]
(npol, nfreq, nbl, ntim) = specamp.shape
tim = times
else:
# Open the ms and plot dynamic spectrum
if verbose:
print('Splitting selected data...')
vis_spl = './tmpms.splitted'
if os.path.exists(vis_spl):
os.system('rm -rf ' + vis_spl)
# split(vis=msfile, outputvis=vis_spl, timerange=timeran, antenna=bl, field=field, scan=scan, spw=spw,
# uvrange=uvrange, timebin=timebin, datacolumn=datacolumn)
try:
from split_cli import split_cli as split
split(vis=msfile, outputvis=vis_spl, datacolumn=datacolumn, timerange=timeran, spw=spw, antenna=bl,
field=field,
scan=scan, uvrange=uvrange, timebin=timebin)
except:
ms.open(msfile, nomodify=True)
ms.split(outputms=vis_spl, whichcol=datacolumn, time=timeran, spw=spw, baseline=bl, field=field, scan=scan,
uvrange=uvrange, timebin=timebin)
ms.close()
if verbose:
print('Regridding into a single spectral window...')
# print('Reading data spw by spw')
try:
tb.open(vis_spl + '/POLARIZATION')
corrtype = tb.getcell('CORR_TYPE', 0)
pols = [stokesenum[p] for p in corrtype]
tb.close()
except:
pols = []
if regridfreq:
ms.open(vis_spl, nomodify=False)
ms.cvel(outframe='LSRK', mode='frequency', interp='nearest')
ms.selectinit(datadescid=0, reset=True)
data = ms.getdata(['amplitude', 'time', 'axis_info'], ifraxis=True)
specamp = data['amplitude']
freq = data['axis_info']['freq_axis']['chan_freq']
else:
ms.open(vis_spl)
ms.selectinit(datadescid=0, reset=True)
spwinfo = ms.getspectralwindowinfo()
specamp = []
freq = []
time = []
for descid in range(len(spwinfo.keys())):
ms.selectinit(datadescid=0, reset=True)
ms.selectinit(datadescid=descid)
data = ms.getdata(['amplitude', 'time', 'axis_info'], ifraxis=True)
specamp_ = data['amplitude']
freq_ = data['axis_info']['freq_axis']['chan_freq']
time_ = data['time']
if fillnan is not None:
flag_ = ms.getdata(['flag', 'time', 'axis_info'], ifraxis=True)['flag']
if type(fillnan) in [int, float, long]:
specamp_[flag_] = float(fillnan)
else:
specamp_[flag_] = 0.0
specamp.append(specamp_)
freq.append(freq_)
time.append(time_)
specamp = np.concatenate(specamp, axis=1)
freq = np.concatenate(freq, axis=0)
ms.selectinit(datadescid=0, reset=True)
ms.close()
os.system('rm -rf ' + vis_spl)
(npol, nfreq, nbl, ntim) = specamp.shape
freq = freq.reshape(nfreq)
tim = data['time']
if verbose:
print('npol, nfreq, nbl, ntime:', (npol, nfreq, nbl, ntim))
spec = np.swapaxes(specamp, 2, 1)
if domedian:
if verbose:
print('doing median of all the baselines')
# mask zero values before median
# spec_masked = np.ma.masked_where(spec < 1e-9, spec)
# spec_masked2 = np.ma.masked_invalid(spec)
# spec_masked = np.ma.masked_array(spec, mask=np.logical_or(spec_masked.mask, spec_masked2.mask))
# spec_med = np.ma.filled(np.ma.median(spec_masked, axis=1), fill_value=0.)
spec = np.abs(spec)
spec_med = np.nanmedian(spec, axis=1)
nbl = 1
ospec = spec_med.reshape((npol, nbl, nfreq, ntim))
else:
ospec = spec
# Save the dynamic spectral data
if savespec:
if not specfile:
specfile = msfile + '.dspec.npz'
if os.path.exists(specfile):
os.system('rm -rf ' + specfile)
np.savez(specfile, spec=ospec, tim=tim, freq=freq,
timeran=timeran, spw=spw, bl=bl, uvrange=uvrange, pol=pols)
if verbose:
print('Median dynamic spectrum saved as: ' + specfile)
return {'spec': ospec, 'tim': tim, 'freq': freq, 'timeran': timeran, 'spw': spw, 'bl': bl, 'uvrange': uvrange,
'pol': pols}
def subvs(vis=None,
outputvis=None,
timerange=None,
spw=None,
timoffset=4,
windowlen=10,
windowtype='flat',
splitsel=True,
reverse=False,
overwrite=False):
"""Vector-subtraction in UV using selected time ranges and spectral channels as background
subvs is a function to do UV vector-subtraction. By selecting gliding averaging window,
only the low-frequency signals corresponding to the background continuum emission remain.
As a result, a uv subtraction of original dynamic spectrum from the background can improve
fine stucture such as fibers. Subvs can be used to subtract the background
continuum emission to separate the time-dependent emission, e.g. solar
coherent radio bursts.
Keyword arguments:
vis -- Name of input visibility file
default: none; example: vis='sun_type3.ms'
outputvis -- Name of output visibility file
default: none; example: outputvis='sun_type3.sub.ms'
timerange -- Select the time range in the data to be subtracted from.
timerange = 'YYYY/MM/DD/hh:mm:ss~YYYY/MM/DD/hh:mm:ss'
Note: if YYYY/MM/DD is missing date, timerange defaults to the
first day in the dataset
timerange='09:14:0~09:54:0' picks 40 min on first day
timerange='25:00:00~27:30:00' picks 1 hr to 3 hr 30min
on next day
spw -- Select spectral window/channel.
default = '' all the spectral channels. Example: spw='0:1~20'
timoffset -- After the convolution, each single channel of smoothed uv vectors
in time series are misaligned from original data. Setting the
timoffset allow you to shift by the specifed amount of data points.
windowlen -- Choose the width the gliding window for smoothing.
windowtype -- Choose the type o gliding window. Options available are 'flat',
'hanning', 'hamming', 'bartlett', 'blackman'
splitsel -- True of False. default = False. If splitsel = False, then the entire input
measurement set is copied as the output measurement set (outputvis), with
background subtracted at selected timerange and spectral channels.
If splitsel = True,then only the selected timerange and spectral channels
are copied into the output measurement set (outputvis).
reverse -- True or False. default = False. If reverse = False, then the times indicated
by subtime1 and/or subtime2 are treated as background and subtracted; If reverse
= True, then reverse the sign of the background-subtracted data. The option can
be used for mapping absorptive structure.
overwrite -- True or False. default = False. If overwrite = True and outputvis
already exists, the selected subtime and spw in the already existing
output measurement set will be replaced with subtracted visibilities
"""
#check the visbility ms
if not outputvis or outputvis.isspace():
raise ValueError, 'Please specify outputvis.'
if os.path.exists(outputvis):
if overwrite:
print "The already existing output measurement set will be updated."
else:
raise ValueError, "Output MS %s already exists - will not overwrite." % outputvis
else:
if not splitsel:
shutil.copytree(vis, outputvis)
else:
ms.open(vis, nomodify=True)
ms.split(outputvis, spw=spw, time=timerange, whichcol='DATA')
ms.close()
# check and specify time range and channel
if timerange and (type(timerange) == str):
[btimeo, etimeo] = timerange.split('~')
btimeosec = qa.getvalue(qa.convert(qa.totime(btimeo), 's'))
etimeosec = qa.getvalue(qa.convert(qa.totime(etimeo), 's'))
timebinosec = etimeosec - btimeosec
if timebinosec < 0:
raise Exception, 'Negative timebin! Please check the "timerange" parameter.'
casalog.post('Selected timerange: ' + timerange +
' as the time for UV subtraction.')
else:
casalog.post(
'Output timerange not specified, using the entire timerange')
if spw and (type(spw) == str):
[spwid, chanran] = spw.split(':')
[bchan, echan] = chanran.split('~')
nchan = int(echan) - int(bchan) + 1
else:
casalog.post('spw not specified, use all frequency channels')
#select data range to be smoothed
ms.open(vis, nomodify=True)
#Select the spw id
ms.msselect({'time': timerange})
ms.selectinit(datadescid=int(spwid))
ms.selectchannel(nchan, int(bchan), 1, 1)
rec = ms.getdata(['data', 'time', 'axis_info'], ifraxis=True)
#print 'shape of the frequency matrix ',rec1['axis_info']['freq_axis']['chan_freq'].shape
sz = rec['data'].shape
print 'dimension of selected background for smoothing', rec['data'].shape
#the data shape is (n_pol,n_channel,n_baseline,n_time), no need to reshape
#rec1['data']=rec1['data'].reshape(sz1[0],sz1[1],sz1[2],nspw,sz1[3]/nspw,order='F')
#print 'reshaped rec1 ', rec1['data'].shape
if not (timoffset and (type(timoffset) == int)):
timoffset = int(4)
for i in range(rec['data'].shape[0]):
for j in range(rec['data'].shape[1]):
for k in range(rec['data'].shape[2]):
rec['data'][i, j,
k, :] = task_smooth.smooth(rec['data'][i, j, k, :],
timoffset, windowlen,
windowtype)
casalog.post('Smoothing the visibilities in timerange: ' + timerange)
ms.close()
#do UV subtraction, according to timerange and spw
ms.open(outputvis, nomodify=False)
if not splitsel:
#outputvis is identical to input visibility, do the selection
if timerange and (type(timerange == str)):
ms.msselect({'time': timerange})
if spw and (type(spw) == str):
ms.selectinit(datadescid=int(spwid))
nchan = int(echan) - int(bchan) + 1
ms.selectchannel(nchan, int(bchan), 1, 1)
else:
#outputvis is splitted, selections have already applied, select all the data
ms.selectinit(datadescid=0)
orec = ms.getdata(['data', 'time', 'axis_info'], ifraxis=True)
b_rows = orec['data'].shape[2]
nchan = orec['data'].shape[1]
#szo=orec['data'].shape
print 'dimension of output data', orec['data'].shape
#orec['data']=orec['data'].reshape(szo[0],szo[1],szo[2],nspw,szo[3]/nspw,order='F')
#print 'reshaped rec1 ', orec['data'].shape
t_rows = orec['data'].shape[3]
casalog.post('Number of baselines: ' + str(b_rows))
casalog.post('Number of spectral channels: ' + str(nchan))
casalog.post('Number of time pixels: ' + str(t_rows))
casalog.post('Subtracting background defined in timerange: ' + timerange)
for i in range(t_rows):
orec['data'][:, :, :, i] -= rec['data'][:, :, :, i]
if reverse:
orec['data'][:, :, :, i] = -orec['data'][:, :, :, i]
del orec['time']
del orec['axis_info']
ms.putdata(orec)
ms.close()
def svplot(vis, timerange=None, spw='', workdir='./', specfile=None, bl=None, uvrange=None,
stokes='RR,LL', dmin=None, dmax=None,
goestime=None, reftime=None,
xycen=None, fov=[500.,500.], xyrange=None, restoringbeam=[''], robust=0.0,
niter=500, imsize=[512], cell=['5.0arcsec'],interactive=False,
usemsphacenter=True, imagefile=None, fitsfile=None, plotaia=True,
aiawave=171, aiafits=None, savefig=False, mkmovie=False, overwrite=True, ncpu=10, twidth=1, verbose=True):
'''
Required inputs:
vis: calibrated CASA measurement set
Important optional inputs:
timerange: timerange for clean. Standard CASA time selection format.
If not provided, use the entire range (*BE CAREFUL, COULD BE VERY SLOW*)
spw: spectral window selection following the CASA syntax.
Examples: spw='1:2~60' (spw id 1, channel range 2-60); spw='*:1.2~1.3GHz' (selects all channels within 1.2-1.3 GHz; note the *)
specfile: supply dynamic spectrum save file (from suncasa.utils.dspec2.get_dspec()). Otherwise
generate a median dynamic spectrum on the fly
Optional inputs:
bl: baseline to generate dynamic spectrum
uvrange: uvrange to select baselines for generating dynamic spectrum
stokes: polarization of the clean image, can be 'RR,LL' or 'I,V'
dmin,dmax: color bar parameter
goestime: goes plot time, example ['2016/02/18 18:00:00','2016/02/18 23:00:00']
rhessisav: rhessi savefile
reftime: reftime for the image
xycen: center of the image in helioprojective coordinates (HPLN/HPLT), in arcseconds. Example: [900, -150.]
fov: field of view in arcsecs. Example: [500., 500.]
xyrange: field of view in solar XY coordinates. Format: [[x1,x2],[y1,y2]]. Example: [[900., 1200.],[0,300]]
***NOTE: THIS PARAMETER OVERWRITES XYCEN AND FOV***
aiawave: wave length of aia file in a
imagefile: if imagefile provided, use it. Otherwise do clean and generate a new one.
fitsfile: if fitsfile provided, use it. Otherwise generate a new one
savefig: whether to save the figure
Example:
'''
if xycen:
xc, yc = xycen
xlen, ylen = fov
if parse_version(sunpy.__version__)>parse_version('0.8.0'):
xyrange = [[xc - xlen / 2.0, yc - ylen / 2.0], [xc + xlen / 2.0, yc + ylen / 2.0]]
else:
xyrange = [[xc - xlen / 2.0, xc + xlen / 2.0], [yc - ylen / 2.0, yc + ylen / 2.0]]
stokes_allowed = ['RR,LL', 'I,V', 'RRLL', 'IV']
if not stokes in stokes_allowed:
print 'wrong stokes parameter ' + str(stokes) + '. Allowed values are ' + ', '.join(stokes_allowed)
return -1
if stokes == 'RRLL':
stokes = 'RR,LL'
if stokes == 'IV':
stokes = 'I,V'
if vis[-1] == '/':
vis = vis[:-1]
if not os.path.exists(vis):
print 'input measurement not exist'
return -1
if aiafits is None:
aiafits = ''
# split the data
# generating dynamic spectrum
if not os.path.exists(workdir):
os.makedirs(workdir)
if specfile:
try:
specdata = np.load(specfile)
except:
print('Provided dynamic spectrum file not numpy npz. Generating one from the visibility data')
specfile = os.path.join(workdir, os.path.basename(vis) + '.dspec.npz')
dspec_external(vis, workdir=workdir, specfile=specfile)
specdata = np.load(specfile) # specdata = ds.get_dspec(vis, domedian=True, verbose=True)
else:
print('Dynamic spectrum file not provided; Generating one from the visibility data')
# specdata = ds.get_dspec(vis, domedian=True, verbose=True)
specfile = os.path.join(workdir, os.path.basename(vis) + '.dspec.npz')
dspec_external(vis, workdir=workdir, specfile=specfile)
specdata = np.load(specfile)
tb.open(vis)
starttim = Time(tb.getcell('TIME', 0) / 24. / 3600., format='mjd')
endtim = Time(tb.getcell('TIME', tb.nrows() - 1) / 24. / 3600., format='mjd')
tb.close()
datstr = starttim.iso[:10]
if timerange is None or timerange == '':
starttim1 = starttim
endtim1 = endtim
timerange = '{0}~{1}'.format(starttim.iso.replace('-', '/').replace(' ', '/'), endtim.iso.replace('-', '/').replace(' ', '/'))
else:
try:
(tstart, tend) = timerange.split('~')
if tstart[2] == ':':
starttim1 = Time(datstr + 'T' + tstart)
endtim1 = Time(datstr + 'T' + tend)
timerange = '{0}/{1}~{0}/{2}'.format(datstr.replace('-', '/'), tstart, tend)
else:
starttim1 = Time(qa.quantity(tstart, 'd')['value'], format='mjd')
endtim1 = Time(qa.quantity(tend, 'd')['value'], format='mjd')
except ValueError:
print "keyword 'timerange' in wrong format"
midtime_mjd = (starttim1.mjd + endtim1.mjd) / 2.
if vis.endswith('/'):
vis = vis[:-1]
visname = os.path.basename(vis)
bt = starttim1.plot_date
et = endtim1.plot_date
# find out min and max frequency for plotting in dynamic spectrum
ms.open(vis)
metadata = ms.metadata()
observatory = metadata.observatorynames()[0]
spwInfo = ms.getspectralwindowinfo()
nspw = len(spwInfo)
if not spw:
spw = '0~' + str(nspw - 1)
staql = {'timerange': timerange, 'spw': spw}
if ms.msselect(staql, onlyparse=True):
ndx = ms.msselectedindices()
chan_sel = ndx['channel']
nspw = chan_sel.shape[0]
bspw = chan_sel[0, 0]
bchan = chan_sel[0, 1]
espw = chan_sel[-1, 0]
echan = chan_sel[-1, 2]
bfreq = spwInfo[str(bspw)]['Chan1Freq'] + spwInfo[str(bspw)]['ChanWidth'] * bchan
efreq = spwInfo[str(espw)]['Chan1Freq'] + spwInfo[str(espw)]['ChanWidth'] * echan
bfreqghz = bfreq / 1e9
efreqghz = efreq / 1e9
if verbose:
print 'selected timerange {}'.format(timerange)
print 'selected frequency range {0:6.3f} to {1:6.3f} GHz'.format(bfreqghz, efreqghz)
else:
print "spw or timerange selection failed. Aborting..."
ms.close()
return -1
ms.close()
if observatory == 'EOVSA':
print 'Provide stokes: ' + str(stokes) + '. However EOVSA has linear feeds. Force stokes to be IV'
stokes = 'I,V'
if mkmovie:
plt.ioff()
# fig = plt.figure(figsize=(12, 7.5), dpi=100)
if fitsfile:
pass
else:
if not imagefile:
# from ptclean_cli import ptclean_cli as ptclean
eph = hf.read_horizons(t0=Time(midtime_mjd, format='mjd'))
if observatory == 'EOVSA' or (not usemsphacenter):
phasecenter = ''
else:
phasecenter = 'J2000 ' + str(eph['ra'][0])[:15] + 'rad ' + str(eph['dec'][0])[:15] + 'rad'
print 'use phasecenter: ' + phasecenter
qlookfitsdir = os.path.join(workdir, 'qlookfits/')
qlookfigdir = os.path.join(workdir, 'qlookimgs/')
imresfile = os.path.join(qlookfitsdir, '{}.imres.npz'.format(os.path.basename(vis)))
if overwrite:
imres = mk_qlook_image(vis, twidth=twidth, ncpu=ncpu, imagedir=qlookfitsdir, phasecenter=phasecenter, stokes=stokes,
c_external=True)
else:
if os.path.exists(imresfile):
imres = np.load(imresfile)
imres = imres['imres'].item()
else:
print('Image results file not found; Creating new images.')
imres = mk_qlook_image(vis, twidth=twidth, ncpu=ncpu, imagedir=qlookfitsdir, phasecenter=phasecenter, stokes=stokes,
c_external=True)
if not os.path.exists(qlookfigdir):
os.makedirs(qlookfigdir)
plt_qlook_image(imres, figdir=qlookfigdir, specdata=specdata, verbose=True, stokes=stokes, fov=xyrange)
else:
spec = specdata['spec']
(npol, nbl, nfreq, ntim) = spec.shape
tidx = range(ntim)
fidx = range(nfreq)
tim = specdata['tim']
freq = specdata['freq']
freqghz = freq / 1e9
spec_tim = Time(specdata['tim'] / 3600. / 24., format='mjd')
timstrr = spec_tim.plot_date
plt.ion()
fig = plt.figure(figsize=(12, 7), dpi=100)
gs1 = gridspec.GridSpec(3, 1)
gs1.update(left=0.08, right=0.32, wspace=0.05)
gs2 = gridspec.GridSpec(2, 2)
gs2.update(left=0.38, right=0.98, hspace=0.02, wspace=0.02)
spec_1 = np.absolute(spec[0, 0, :, :])
spec_2 = np.absolute(spec[1, 0, :, :])
if observatory == 'EVLA':
# circular feeds
polstr = ['RR', 'LL']
if observatory == 'EOVSA' or observatory == 'ALMA':
# linear feeds
polstr = ['XX', 'YY']
print 'plot the dynamic spectrum in pol ' + ' & '.join(polstr)
ax1 = plt.subplot(gs1[0])
ax1.pcolormesh(timstrr, freqghz, spec_1, cmap='jet', vmin=dmin, vmax=dmax)
ax1.set_xlim(timstrr[tidx[0]], timstrr[tidx[-1]])
ax1.xaxis_date()
ax1.xaxis.set_major_formatter(DateFormatter("%H:%M:%S"))
# ax1.set_xticklabels(['']*10)
ax1.set_ylim(freqghz[fidx[0]], freqghz[fidx[-1]])
ax1.set_ylabel('Frequency (GHz)', fontsize=10)
ax1.set_title(observatory + ' ' + datstr + ' ' + polstr[0] + ' & ' + polstr[1], fontsize=12)
ax1.set_autoscale_on(False)
ax1.add_patch(patches.Rectangle((bt, bfreqghz), et - bt, efreqghz - bfreqghz, ec='w', fill=False))
ax1.plot([(bt + et) / 2.], [(bfreqghz + efreqghz) / 2.], '*w', ms=12)
for tick in ax1.get_xticklabels():
tick.set_fontsize(8)
for tick in ax1.get_yticklabels():
tick.set_fontsize(8)
ax2 = plt.subplot(gs1[1])
ax2.pcolormesh(timstrr, freqghz, spec_2, cmap='jet', vmin=dmin, vmax=dmax)
ax2.set_xlim(timstrr[tidx[0]], timstrr[tidx[-1]])
ax2.xaxis_date()
ax2.xaxis.set_major_formatter(DateFormatter("%H:%M:%S"))
ax2.set_ylim(freqghz[fidx[0]], freqghz[fidx[-1]])
ax2.set_ylabel('Frequency (GHz)', fontsize=10)
for tick in ax2.get_xticklabels():
tick.set_fontsize(8)
for tick in ax2.get_yticklabels():
tick.set_fontsize(8)
ax2.set_autoscale_on(False)
ax2.add_patch(patches.Rectangle((bt, bfreqghz), et - bt, efreqghz - bfreqghz, ec='w', fill=False))
ax2.plot([(bt + et) / 2.], [(bfreqghz + efreqghz) / 2.], '*w', ms=12)
# Second part: GOES plot
if goestime:
btgoes = goestime[0]
etgoes = goestime[1]
else:
datstrg = datstr.replace('-', '/')
btgoes = datstrg + ' ' + qa.time(qa.quantity(tim[0] - 1800, 's'), form='clean', prec=9)[0]
etgoes = datstrg + ' ' + qa.time(qa.quantity(tim[tidx[-1] - 1] + 1800, 's'), form='clean', prec=9)[0]
if verbose:
print 'Acquire GOES soft X-ray data in from ' + btgoes + ' to ' + etgoes
ax3 = plt.subplot(gs1[2])
try:
from sunpy import lightcurve as lc
from sunpy.time import TimeRange
goest = lc.GOESLightCurve.create(TimeRange(btgoes, etgoes))
except:
goesscript = os.path.join(workdir, 'goes.py')
goesdatafile = os.path.join(workdir, 'goes.dat')
os.system('rm -rf {}'.format(goesscript))
fi = open(goesscript, 'wb')
fi.write('import os \n')
fi.write('from sunpy.time import TimeRange \n')
fi.write('from sunpy import lightcurve as lc \n')
fi.write('import pickle \n')
fi.write('goesplottim = TimeRange("{0}", "{1}") \n'.format(btgoes, etgoes))
fi.write('goes = lc.GOESLightCurve.create(goesplottim) \n')
fi.write('fi2 = open("{}", "wb") \n'.format(goesdatafile))
fi.write('pickle.dump(goes, fi2) \n')
fi.write('fi2.close()')
fi.close()
try:
os.system('python {}'.format(goesscript))
os.system('rm -rf {}'.format(goesscript))
except NameError:
print "Bad input names"
except ValueError:
print "Bad input values"
except:
print "Unexpected error:", sys.exc_info()[0]
print "Error in generating GOES light curves. Proceed without GOES..."
if os.path.exists(goesdatafile):
fi1 = file(goesdatafile, 'rb')
goest = pickle.load(fi1)
fi1.close()
try:
dates = mpl.dates.date2num(parse_time(goest.data.index))
goesdif = np.diff(goest.data['xrsb'])
gmax = np.nanmax(goesdif)
gmin = np.nanmin(goesdif)
ran = gmax - gmin
db = 2.8 / ran
goesdifp = goesdif * db + gmin + (-6)
ax3.plot_date(dates, np.log10(goest.data['xrsb']), '-', label='1.0--8.0 $\AA$', color='red', lw=2)
ax3.plot_date(dates[0:-1], goesdifp, '-', label='derivate', color='blue', lw=0.4)
ax3.set_ylim([-7, -3])
ax3.set_yticks([-7, -6, -5, -4, -3])
ax3.set_yticklabels([r'$10^{-7}$', r'$10^{-6}$', r'$10^{-5}$', r'$10^{-4}$', r'$10^{-3}$'])
ax3.set_title('Goes Soft X-ray', fontsize=12)
ax3.set_ylabel('Watts m$^{-2}$')
ax3.set_xlabel(datetime.datetime.isoformat(goest.data.index[0])[0:10])
ax3.axvspan(dates[899], dates[dates.size - 899], alpha=0.2)
ax2 = ax3.twinx()
# ax2.set_yscale("log")
ax2.set_ylim([-7, -3])
ax2.set_yticks([-7, -6, -5, -4, -3])
ax2.set_yticklabels(['B', 'C', 'M', 'X', ''])
ax3.yaxis.grid(True, 'major')
ax3.xaxis.grid(False, 'major')
ax3.legend(prop={'size': 6})
formatter = mpl.dates.DateFormatter('%H:%M')
ax3.xaxis.set_major_formatter(formatter)
ax3.fmt_xdata = mpl.dates.DateFormatter('%H:%M')
except:
print 'Error in downloading GOES soft X-ray data. Proceeding with out soft X-ray plot.'
# third part
# start to download the fits files
if plotaia:
if not aiafits:
newlist = []
items = glob.glob('*.fits')
for names in items:
str1 = starttim1.iso[:4] + '_' + starttim1.iso[5:7] + '_' + starttim1.iso[8:10] + 't' + starttim1.iso[
11:13] + '_' + starttim1.iso[14:16]
str2 = str(aiawave)
if names.endswith(".fits"):
if names.find(str1) != -1 and names.find(str2) != -1:
newlist.append(names)
newlist.append('0')
if newlist and os.path.exists(newlist[0]):
aiafits = newlist[0]
else:
print 'downloading the aiafits file'
wave1 = aiawave - 3
wave2 = aiawave + 3
t1 = Time(starttim1.mjd - 0.02 / 24., format='mjd')
t2 = Time(endtim1.mjd + 0.02 / 24., format='mjd')
try:
from sunpy.net import vso
client = vso.VSOClient()
qr = client.query(vso.attrs.Time(t1.iso, t2.iso), vso.attrs.Instrument('aia'), vso.attrs.Wave(wave1 * u.AA, wave2 * u.AA))
res = client.get(qr, path='{file}')
except:
SdoDownloadscript = os.path.join(workdir, 'SdoDownload.py')
os.system('rm -rf {}'.format(SdoDownloadscript))
fi = open(SdoDownloadscript, 'wb')
fi.write('from sunpy.net import vso \n')
fi.write('from astropy import units as u \n')
fi.write('client = vso.VSOClient() \n')
fi.write(
"qr = client.query(vso.attrs.Time('{0}', '{1}'), vso.attrs.Instrument('aia'), vso.attrs.Wave({2} * u.AA, {3} * u.AA)) \n".format(
t1.iso, t2.iso, wave1, wave2))
fi.write("res = client.get(qr, path='{file}') \n")
fi.close()
try:
os.system('python {}'.format(SdoDownloadscript))
except NameError:
print "Bad input names"
except ValueError:
print "Bad input values"
except:
print "Unexpected error:", sys.exc_info()[0]
print "Error in Downloading AIA fits files. Proceed without AIA..."
# Here something is needed to check whether it has finished downloading the fits files or not
if not aiafits:
newlist = []
items = glob.glob('*.fits')
for nm in items:
str1 = starttim1.iso[:4] + '_' + starttim1.iso[5:7] + '_' + starttim1.iso[8:10] + 't' + starttim1.iso[
11:13] + '_' + starttim1.iso[14:16]
str2 = str(aiawave)
if nm.find(str1) != -1 and nm.find(str2) != -1:
newlist.append(nm)
if newlist:
aiafits = newlist[0]
print 'AIA fits ' + aiafits + ' selected'
else:
print 'no AIA fits files found. Proceed without AIA'
try:
aiamap = smap.Map(aiafits)
except:
print 'error in reading aiafits. Proceed without AIA'
# RCP or I
ax4 = plt.subplot(gs2[0, 0])
ax5 = plt.subplot(gs2[1, 0])
# LCP or V
ax6 = plt.subplot(gs2[0, 1])
ax7 = plt.subplot(gs2[1, 1])
if fitsfile:
pass
else:
if not imagefile:
eph = hf.read_horizons(t0=Time(midtime_mjd, format='mjd'))
if observatory == 'EOVSA' or (not usemsphacenter):
print 'This is EOVSA data'
# use RA and DEC from FIELD ID 0
tb.open(vis+'/FIELD')
phadir = tb.getcol('PHASE_DIR').flatten()
tb.close()
ra0 = phadir[0]
dec0 = phadir[1]
if stokes == 'RRLL' or stokes == 'RR,LL':
print 'Provide stokes: ' + str(stokes) + '. However EOVSA has linear feeds. Force stokes to be IV'
stokes = 'I,V'
else:
ra0 = eph['ra'][0]
dec0 = eph['dec'][0]
if not xycen:
# use solar disk center as default
phasecenter = 'J2000 ' + str(ra0) + 'rad ' + str(dec0) + 'rad'
else:
x0 = np.radians(xycen[0]/3600.)
y0 = np.radians(xycen[1]/3600.)
p0 = np.radians(eph['p0'][0]) # p angle in radians
raoff = -((x0) * np.cos(p0) - y0 * np.sin(p0))/np.cos(eph['dec'][0])
decoff = (x0) * np.sin(p0) + y0 * np.cos(p0)
newra = ra0 + raoff
newdec = dec0 + decoff
phasecenter = 'J2000 ' + str(newra) + 'rad ' + str(newdec) + 'rad'
imagename = os.path.join(workdir, visname + '.outim')
if os.path.exists(imagename + '.image') or os.path.exists(imagename + '.flux'):
os.system('rm -rf ' + imagename + '.*')
sto = stokes.replace(',', '')
print 'do clean for ' + timerange + ' in spw ' + spw + ' stokes ' + sto
print 'Original phasecenter: '+ str(ra0) + str(dec0)
print 'use phasecenter: ' + phasecenter
clean(vis=vis, imagename=imagename, selectdata=True, spw=spw, timerange=timerange, stokes=sto,
niter=niter, interactive=interactive, npercycle=50, imsize=imsize, cell=cell, restoringbeam=restoringbeam,
weighting='briggs', robust=robust, phasecenter=phasecenter)
os.system('rm -rf ' + imagename + '.psf')
os.system('rm -rf ' + imagename + '.flux')
os.system('rm -rf ' + imagename + '.model')
os.system('rm -rf ' + imagename + '.mask')
os.system('rm -rf ' + imagename + '.residual')
imagefile = imagename + '.image'
fitsfile = imagefile + '.fits'
hf.imreg(vis=vis, ephem=eph, imagefile=imagefile, timerange=timerange, reftime=reftime, fitsfile=fitsfile, verbose=True, overwrite=True)
print 'fits file ' + fitsfile + ' selected'
ax4.cla()
ax5.cla()
ax6.cla()
ax7.cla()
rfits = fitsfile
try:
hdulist = fits.open(rfits)
hdu = hdulist[0]
(npol, nf, nx, ny) = hdu.data.shape
rmap = smap.Map(hdu.data[0, 0, :, :], hdu.header)
except:
print 'radio fits file not recognized by sunpy.map. Aborting...'
return -1
if npol > 1:
rmap1 = smap.Map(hdu.data[0, 0, :, :], hdu.header)
rmap2 = smap.Map(hdu.data[1, 0, :, :], hdu.header)
XX, YY = np.meshgrid(np.arange(rmap.data.shape[1]), np.arange(rmap.data.shape[0]))
try:
rmapx, rmapy = rmap.pixel_to_data(XX * u.pix, YY * u.pix)
except:
rmapxy = rmap.pixel_to_data(XX * u.pix, YY * u.pix)
rmapx = rmapxy.Tx
rmapy = rmapxy.Ty
if not xyrange:
if xycen:
x0 = xycen[0] * u.arcsec
y0 = xycen[1] * u.arcsec
if not xycen:
row, col = rmap1.data.shape
positon = np.nanargmax(rmap1.data)
m, n = divmod(positon, col)
x0 = rmap1.xrange[0] + rmap1.scale[1] * (n + 0.5) * u.pix
y0 = rmap1.yrange[0] + rmap1.scale[0] * (m + 0.5) * u.pix
if len(fov) == 1:
fov=[fov]*2
sz_x = fov[0] * u.arcsec
sz_y = fov[1] * u.arcsec
x1 = x0 - sz_x/2.
x2 = x0 + sz_x/2.
y1 = y0 - sz_y/2.
y2 = y0 + sz_y/2.
xyrange = [[x1.value, x2.value], [y1.value, y2.value]]
else:
sz_x = xyrange[0][1] - xyrange[0][0]
sz_y = xyrange[1][1] - xyrange[1][0]
clevels1 = np.linspace(0.2, 0.9, 5)
if stokes.split(',')[1] == 'V':
clevels2 = np.array([0.8, -0.6, -0.4, -0.2, 0.2, 0.4, 0.6, 0.8])
else:
clevels2 = np.linspace(0.2, 0.9, 5)
if 'aiamap' in vars():
aiamap.plot_settings['cmap'] = plt.get_cmap('binary')
if rmap:
title = 'AIA {0:.0f} + {1} {2:6.3f} GHz'.format(aiamap.wavelength.value, observatory, (bfreqghz + efreqghz) / 2.0)
else:
title = 'AIA {0:.0f}'.format(aiamap.wavelength.value)
aiamap.plot(axes=ax4)
ax4.set_title(title + ' ' + stokes.split(',')[0], fontsize=12)
aiamap.draw_limb()
aiamap.draw_grid()
aiamap.draw_rectangle((xyrange[0][0], xyrange[1][0]) * u.arcsec, sz_x, sz_y)
aiamap.plot(axes=ax6)
ax6.set_title(title + ' ' + stokes.split(',')[1], fontsize=12)
aiamap.draw_limb()
aiamap.draw_grid()
aiamap.draw_rectangle((xyrange[0][0], xyrange[1][0]) * u.arcsec, sz_x, sz_y)
if rmap:
ax4.contour(rmapx.value, rmapy.value, rmap1.data, levels=clevels1 * np.nanmax(rmap1.data), cmap=cm.jet)
ax6.contour(rmapx.value, rmapy.value, rmap2.data, levels=clevels2 * np.nanmax(rmap2.data), cmap=cm.RdBu)
ax4.text(0.02, 0.02, 'AIA {0:.0f} '.format(aiamap.wavelength.value) + aiamap.date.strftime('%H:%M:%S'),
verticalalignment='bottom', horizontalalignment='left', transform=ax4.transAxes, color='k',
fontsize=10)
ax6.text(0.02, 0.02, 'AIA {0:.0f} '.format(aiamap.wavelength.value) + aiamap.date.strftime('%H:%M:%S'),
verticalalignment='bottom', horizontalalignment='left', transform=ax6.transAxes, color='k',
fontsize=10)
else:
title = '{0} {1:6.3f} GHz'.format(observatory, (bfreqghz + efreqghz) / 2.0)
rmap1.plot(axes=ax4, cmap=cm.jet)
ax4.set_title(title + ' ' + stokes.split(',')[0], fontsize=12)
rmap1.draw_limb()
rmap1.draw_grid()
rmap1.draw_rectangle((xyrange[0][0], xyrange[1][0]) * u.arcsec, sz_x, sz_y)
rmap2.plot(axes=ax6, cmap=cm.RdBu)
ax6.set_title(title + ' ' + stokes.split(',')[1], fontsize=12)
rmap2.draw_limb()
rmap2.draw_grid()
# ax4.contour(rmapx.value, rmapy.value, rmap1.data, levels=np.linspace(0.2, 0.9, 5) * np.nanmax(rmap1.data),
# cmap=cm.gray)
# ax6.contour(rmapx.value, rmapy.value, rmap2.data, levels=np.linspace(0.2, 0.9, 5) * np.nanmax(rmap2.data),
# cmap=cm.gray)
rmap2.draw_rectangle((xyrange[0][0], xyrange[1][0]) * u.arcsec, sz_x, sz_y)
ax4.set_xlim(-1200, 1200)
ax4.set_ylim(-1200, 1200)
ax6.set_xlim(-1200, 1200)
ax6.set_ylim(-1200, 1200)
try:
subrmap1 = rmap1.submap(xyrange[0] * u.arcsec, xyrange[1] * u.arcsec)
subrmap2 = rmap2.submap(xyrange[0] * u.arcsec, xyrange[1] * u.arcsec)
except:
bl = SkyCoord(xyrange[0][0] * u.arcsec, xyrange[1][0] * u.arcsec, frame=rmap1.coordinate_frame)
tr = SkyCoord(xyrange[0][1] * u.arcsec, xyrange[1][1] * u.arcsec, frame=rmap1.coordinate_frame)
subrmap1 = rmap1.submap(bl, tr)
subrmap2 = rmap2.submap(bl, tr)
XX, YY = np.meshgrid(np.arange(subrmap1.data.shape[1]), np.arange(subrmap1.data.shape[0]))
try:
subrmapx, subrmapy = subrmap1.pixel_to_data(XX * u.pix, YY * u.pix)
except:
subrmapxy = subrmap1.pixel_to_data(XX * u.pix, YY * u.pix)
subrmapx = subrmapxy.Tx
subrmapy = subrmapxy.Ty
if 'aiamap' in vars():
try:
subaiamap = aiamap.submap(xyrange[0] * u.arcsec, xyrange[1] * u.arcsec)
except:
bl = SkyCoord(xyrange[0][0] * u.arcsec, xyrange[1][0] * u.arcsec, frame=aiamap.coordinate_frame)
tr = SkyCoord(xyrange[0][1] * u.arcsec, xyrange[1][1] * u.arcsec, frame=aiamap.coordinate_frame)
subaiamap = aiamap.submap(bl, tr)
subaiamap.plot(axes=ax5, title='')
subaiamap.draw_limb()
subaiamap.draw_grid()
subaiamap.plot(axes=ax7, title='')
subaiamap.draw_limb()
subaiamap.draw_grid()
ax5.contour(subrmapx.value, subrmapy.value, subrmap1.data, levels=clevels1 * np.nanmax(subrmap1.data), cmap=cm.jet)
ax7.contour(subrmapx.value, subrmapy.value, subrmap2.data, levels=clevels2 * np.nanmax(subrmap2.data),
cmap=cm.RdBu) # subaiamap.draw_rectangle((fov[0][0], fov[1][0]) * u.arcsec, 400 * u.arcsec, 400 * u.arcsec)
else:
subrmap1.plot(axes=ax5, cmap=cm.jet, title='')
subrmap1.draw_limb()
subrmap1.draw_grid()
subrmap2.plot(axes=ax7, cmap=cm.RdBu, title='')
subrmap2.draw_limb()
subrmap2.draw_grid() # ax5.contour(subrmapx.value, subrmapy.value, subrmap1.data, # levels=clevels1 * np.nanmax(subrmap1.data), cmap=cm.gray) # ax7.contour(subrmapx.value, subrmapy.value, subrmap2.data, # levels=clevels2 * np.nanmax(subrmap2.data), cmap=cm.gray) # subrmap1.draw_rectangle((fov[0][0], fov[1][0]) * u.arcsec, 400 * u.arcsec, 400 * u.arcsec) # subrmap2.draw_rectangle((fov[0][0], fov[1][0]) * u.arcsec, 400 * u.arcsec, 400 * u.arcsec)
ax5.set_xlim(xyrange[0])
ax5.set_ylim(xyrange[1])
ax5.text(0.02, 0.02, observatory + ' ' + rmap.date.strftime('%H:%M:%S.%f')[:-3], verticalalignment='bottom',
horizontalalignment='left', transform=ax5.transAxes, color='k', fontsize=10)
ax7.set_xlim(xyrange[0])
ax7.set_ylim(xyrange[1])
ax7.text(0.02, 0.02, observatory + ' ' + rmap.date.strftime('%H:%M:%S.%f')[:-3], verticalalignment='bottom',
horizontalalignment='left', transform=ax7.transAxes, color='k', fontsize=10)
fig.show()
def subvs(vis=None,
outputvis=None,
timerange=None,
spw=None,
subtime1=None,
subtime2=None,
splitsel=True,
reverse=False,
overwrite=False):
"""Perform vector subtraction for visibilities
Keyword arguments:
vis -- Name of input visibility file (MS)
default: none; example: vis='ngc5921.ms'
outputvis -- Name of output uv-subtracted visibility file (MS)
default: none; example: outputvis='ngc5921_src.ms'
timerange -- Time range of performing the UV subtraction:
default='' means all times. examples:
timerange = 'YYYY/MM/DD/hh:mm:ss~YYYY/MM/DD/hh:mm:ss'
timerange = 'hh:mm:ss~hh:mm:ss'
spw -- Select spectral window/channel.
default = '' all the spectral channels. Example: spw='0:1~20'
subtime1 -- Time range 1 of the background to be subtracted from the data
default='' means all times. format:
timerange = 'YYYY/MM/DD/hh:mm:ss~YYYY/MM/DD/hh:mm:ss'
timerange = 'hh:mm:ss~hh:mm:ss'
subtime2 -- Time range 2 of the backgroud to be subtracted from the data
default='' means all times. examples:
timerange = 'YYYY/MM/DD/hh:mm:ss~YYYY/MM/DD/hh:mm:ss'
timerange = 'hh:mm:ss~hh:mm:ss'
splitsel -- True or False. default = False. If splitsel = False, then the entire input
measurement set is copied as the output measurement set (outputvis), with
background subtracted at selected timerange and spectral channels.
If splitsel = True,then only the selected timerange and spectral channels
are copied into the output measurement set (outputvis).
reverse -- True or False. default = False. If reverse = False, then the times indicated
by subtime1 and/or subtime2 are treated as background and subtracted; If reverse
= True, then reverse the sign of the background-subtracted data. The option can
be used for mapping absorptive structure.
overwrite -- True or False. default = False. If overwrite = True and
outputvis already exists, the selected subtime and spw in the
output measurment set will be replaced with background subtracted
visibilities
"""
#check the visbility ms
if not outputvis or outputvis.isspace():
raise ValueError, 'Please specify outputvis'
if os.path.exists(outputvis):
if overwrite:
print "The already existing output measurement set will be updated."
else:
raise ValueError, "Output MS %s already exists - will not overwrite." % outputvis
else:
if not splitsel:
shutil.copytree(vis, outputvis)
else:
ms.open(vis, nomodify=True)
ms.split(outputvis, spw=spw, time=timerange, whichcol='DATA')
ms.close()
#define and check the time ranges
if subtime1 and (type(subtime1) == str):
[bsubtime1, esubtime1] = subtime1.split('~')
bsubtime1sec = qa.getvalue(qa.convert(qa.totime(bsubtime1), 's'))
esubtime1sec = qa.getvalue(qa.convert(qa.totime(esubtime1), 's'))
timebin1sec = esubtime1sec - bsubtime1sec
if timebin1sec < 0:
raise Exception, 'Negative timebin! Please check the "subtime1" parameter.'
casalog.post('Selected timerange 1: ' + subtime1 +
' as background for uv subtraction.')
else:
raise Exception, 'Please enter at least one timerange as the background'
if subtime2 and (type(subtime2) == str):
[bsubtime2, esubtime2] = subtime2.split('~')
bsubtime2sec = qa.getvalue(qa.convert(qa.totime(bsubtime2), 's'))
esubtime2sec = qa.getvalue(qa.convert(qa.totime(esubtime2), 's'))
timebin2sec = esubtime2sec - bsubtime2sec
if timebin2sec < 0:
raise Exception, 'Negative timebin! Please check the "subtime2" parameter.'
timebin2 = str(timebin2sec) + 's'
casalog.post('Selected timerange 2: ' + subtime2 +
' as background for uv subtraction.')
#plus 1s is to ensure averaging over the entire timerange
else:
casalog.post(
'Timerange 2 not selected, using only timerange 1 as background')
if timerange and (type(timerange) == str):
[btimeo, etimeo] = timerange.split('~')
btimeosec = qa.getvalue(qa.convert(qa.totime(btimeo), 's'))
etimeosec = qa.getvalue(qa.convert(qa.totime(etimeo), 's'))
timebinosec = etimeosec - btimeosec
if timebinosec < 0:
raise Exception, 'Negative timebin! Please check the "timerange" parameter.'
casalog.post('Selected timerange: ' + timerange +
' as the time for UV subtraction.')
else:
casalog.post(
'Output timerange not specified, using the entire timerange')
if spw and (type(spw) == str):
[spwid, chanran] = spw.split(':')
[bchan, echan] = chanran.split('~')
else:
casalog.post('spw not specified, use all frequency channels')
#Select the background indicated by subtime1
ms.open(vis, nomodify=True)
#Select the spw id
ms.msselect({'time': subtime1})
if spw and (type(spw) == str):
ms.selectinit(datadescid=int(spwid))
nchan = int(echan) - int(bchan) + 1
ms.selectchannel(nchan, int(bchan), 1, 1)
rec1 = ms.getdata(['data', 'time', 'axis_info'], ifraxis=True)
#print 'shape of the frequency matrix ',rec1['axis_info']['freq_axis']['chan_freq'].shape
sz1 = rec1['data'].shape
print 'dimension of selected background 1', rec1['data'].shape
#the data shape is (n_pol,n_channel,n_baseline,n_time), no need to reshape
#rec1['data']=rec1['data'].reshape(sz1[0],sz1[1],sz1[2],nspw,sz1[3]/nspw,order='F')
#print 'reshaped rec1 ', rec1['data'].shape
rec1avg = np.average(rec1['data'], axis=3)
casalog.post('Averaging the visibilities in subtime1: ' + subtime1)
ms.close()
if subtime2 and (type(subtime2) == str):
ms.open(vis, nomodify=True)
#Select the spw id
ms.msselect({'time': subtime2})
if spw and (type(spw) == str):
ms.selectinit(datadescid=0)
nchan = int(echan) - int(bchan) + 1
ms.selectchannel(nchan, int(bchan), 1, 1)
rec2 = ms.getdata(['data', 'time', 'axis_info'], ifraxis=True)
sz2 = rec2['data'].shape
print 'dimension of selected background 2', rec2['data'].shape
#rec2['data']=rec2['data'].reshape(sz2[0],sz2[1],sz2[2],nspw,sz2[3]/nspw,order='F')
#print 'reshaped rec1 ', rec2['data'].shape
rec2avg = np.average(rec2['data'], axis=3)
ms.close()
casalog.post('Averaged the visibilities in subtime2: ' + subtime2)
#do UV subtraction, according to timerange and spw
ms.open(outputvis, nomodify=False)
if not splitsel:
#outputvis is identical to input visibility, do the selection
if timerange and (type(timerange == str)):
ms.msselect({'time': timerange})
if spw and (type(spw) == str):
ms.selectinit(datadescid=int(spwid))
nchan = int(echan) - int(bchan) + 1
ms.selectchannel(nchan, int(bchan), 1, 1)
else:
#outputvis is splitted, selections have already applied, select all the data
ms.selectinit(datadescid=0)
orec = ms.getdata(['data', 'time', 'axis_info'], ifraxis=True)
b_rows = orec['data'].shape[2]
nchan = orec['data'].shape[1]
#szo=orec['data'].shape
print 'dimension of output data', orec['data'].shape
#orec['data']=orec['data'].reshape(szo[0],szo[1],szo[2],nspw,szo[3]/nspw,order='F')
#print 'reshaped rec1 ', orec['data'].shape
t_rows = orec['data'].shape[3]
casalog.post('Number of baselines: ' + str(b_rows))
casalog.post('Number of spectral channels: ' + str(nchan))
casalog.post('Number of time pixels: ' + str(t_rows))
if subtime1 and (not subtime2):
casalog.post(
'Only "subtime1" is defined, subtracting background defined in subtime1: '
+ subtime1)
t1 = (np.amax(rec1['time']) + np.amin(rec1['time'])) / 2.
print 't1: ', qa.time(qa.quantity(t1, 's'), form='ymd', prec=10)
for i in range(t_rows):
orec['data'][:, :, :, i] -= rec1avg
if reverse:
orec['data'][:, :, :, i] = -orec['data'][:, :, :, i]
if subtime1 and subtime2 and (type(subtime2) == str):
casalog.post(
'Both subtime1 and subtime2 are specified, doing linear interpolation between "subtime1" and "subtime2"'
)
t1 = (np.amax(rec1['time']) + np.amin(rec1['time'])) / 2.
t2 = (np.amax(rec2['time']) + np.amin(rec2['time'])) / 2.
touts = orec['time']
print 't1: ', qa.time(qa.quantity(t1, 's'), form='ymd', prec=10)
print 't2: ', qa.time(qa.quantity(t2, 's'), form='ymd', prec=10)
for i in range(t_rows):
tout = touts[i]
if tout > np.amax([t1, t2]):
tout = np.amax([t1, t2])
elif tout < np.amin([t1, t2]):
tout = np.amin([t1, t2])
orec['data'][:, :, :,
i] -= (rec2avg - rec1avg) * (tout -
t1) / (t2 - t1) + rec1avg
if reverse:
orec['data'][:, :, :, i] = -orec['data'][:, :, :, i]
#orec['data']=orec['data'].reshape(szo[0],szo[1],szo[2],szo[3],order='F')
#put the modified data back into the output visibility set
del orec['time']
del orec['axis_info']
ms.putdata(orec)
ms.close()
def WEI_plot(dofile=True,
vis=None,
timerange=None,
spw='',
aiafits='',
imagehead='',
workdir='',
spwCol=3,
phasecenter='J2000 11h00m48 06d14m60'):
if vis[-1] == '/':
vis = vis[:-1]
ms.open(vis)
spwInfo = ms.getspectralwindowinfo()
ms.close()
tb.open(vis)
starttim = Time(tb.getcell('TIME', 0) / 24. / 3600., format='mjd')
endtim = Time(tb.getcell('TIME',
tb.nrows() - 1) / 24. / 3600.,
format='mjd')
tb.close()
tb.open(vis + '/SPECTRAL_WINDOW')
reffreqs = tb.getcol('REF_FREQUENCY')
bdwds = tb.getcol('TOTAL_BANDWIDTH')
cfreqs = reffreqs + bdwds / 2.
tb.close()
sbeam = 35.
#get timerange from vis file
if not timerange:
timerange = '{0}~{1}'.format(
starttim.iso.replace('-', '/').replace(' ', '/'),
endtim.iso.replace('-', '/').replace(' ', '/'))
nspw = len(spwInfo)
#draw plot aia
fig = plt.figure(figsize=(12, 7), dpi=100)
gs1 = gridspec.GridSpec(4, 3)
gs1.update(left=0.08, right=0.32, wspace=0.05)
ax1 = plt.subplot(gs1[11])
aiamap = smap.Map(aiafits)
aiamap.plot(axes=ax1)
#do clean spwCol by spwCol
for cur_spwCol in range(0, np.floor_divide(nspw, spwCol)):
if ((cur_spwCol + 1) * spwCol) < nspw:
cur_spwRange = str(cur_spwCol * spwCol + 1) + '~' + str(
(cur_spwCol + 1) * spwCol)
else:
cur_spwRange = str(cur_spwCol * spwCol + 1) + '~' + '31'
imagename = imagehead + cur_spwRange + 'SPWs'
cur_eovsaFits = imagename + '.fits'
if cur_spwCol < 6:
cur_mask = '/srg/ywei/data/eovsa/mask/sep_6mask_' + str(
cur_spwCol + 1) + '.rgn'
else:
cur_mask = '/srg/ywei/data/eovsa/mask/sep_6/mask_6.rgn'
if dofile:
#clean(vis=vis, spw=cur_spwRange, timerange=timerange, imagename=imagename, imsize=[256,256], niter=100, cell=['2arcsec'] )
#clean(vis=vis, spw=cur_spwRange, timerange=timerange, imagename=imagename, imsize=[512,512], niter=1000, cell=['1arcsec'],stokes='XX', gain=0.05,weighting='briggs', mode='mfs',imagermode='csclean',psfmode='clark',robust=0.0,restoringbeam = ['10.0arcsec'], mask=cur_mask,pbcor=True)
clean(vis=vis,
spw=cur_spwRange,
timerange=timerange,
imagename=imagename,
imsize=[512, 512],
niter=1000,
cell=['1arcsec'],
stokes='XX',
gain=0.05,
weighting='briggs',
mode='mfs',
imagermode='csclean',
psfmode='clark',
robust=0.0,
restoringbeam=['10.0arcsec'],
mask='',
pbcor=True)
print 'fits name =' + str(cur_eovsaFits)
hf.imreg(vis=vis,
imagefile=imagename + '.image',
fitsfile=imagename + '.fits',
timerange=timerange)
#plot eovsa
cur_emap = smap.Map(cur_eovsaFits)
cur_axname = plt.subplot(gs1[cur_spwCol + 1])
(npol, nf, nx, ny) = cur_emap.data.shape
print 'shape = ' + str(cur_emap.data.shape)
if npol != 1:
print 'To be determined'
else:
cur_emap.data = cur_emap.data.reshape((512, 512))
cur_emap.plot_settings['cmap'] = plt.get_cmap('jet')
cur_emap.plot(axes=cur_axname)
def calc_phasecenter_from_solxy(vis,
timerange='',
xycen=None,
usemsphacenter=True):
'''
return the phase center in RA and DEC of a given solar coordinates
:param vis: input measurement sets file
:param timerange: can be a string or astropy.time.core.Time object, or a 2-element list of string or Time object
:param xycen: solar x-pos and y-pos in arcsec
:param usemsphacenter:
:return:
phasecenter
midtim: mid time of the given timerange
'''
tb.open(vis + '/POINTING')
tst = Time(tb.getcell('TIME_ORIGIN', 0) / 24. / 3600., format='mjd')
ted = Time(tb.getcell('TIME_ORIGIN',
tb.nrows() - 1) / 24. / 3600.,
format='mjd')
tb.close()
datstr = tst.iso[:10]
if isinstance(timerange, Time):
try:
(sttim, edtim) = timerange
except:
sttim = timerange
edtim = sttim
else:
if timerange == '':
sttim = tst
edtim = ted
else:
try:
(tstart, tend) = timerange.split('~')
if tstart[2] == ':':
sttim = Time(datstr + 'T' + tstart)
edtim = Time(datstr + 'T' + tend)
# timerange = '{0}/{1}~{0}/{2}'.format(datstr.replace('-', '/'), tstart, tend)
else:
sttim = Time(qa.quantity(tstart, 'd')['value'],
format='mjd')
edtim = Time(qa.quantity(tend, 'd')['value'], format='mjd')
except:
try:
if timerange[2] == ':':
sttim = Time(datstr + 'T' + timerange)
edtim = sttim
else:
sttim = Time(qa.quantity(timerange, 'd')['value'],
format='mjd')
edtim = sttim
except ValueError:
print("keyword 'timerange' in wrong format")
ms.open(vis)
metadata = ms.metadata()
observatory = metadata.observatorynames()[0]
ms.close()
midtim_mjd = (sttim.mjd + edtim.mjd) / 2.
midtim = Time(midtim_mjd, format='mjd')
eph = read_horizons(t0=midtim)
if observatory == 'EOVSA' or (not usemsphacenter):
print('This is EOVSA data')
# use RA and DEC from FIELD ID 0
tb.open(vis + '/FIELD')
phadir = tb.getcol('PHASE_DIR').flatten()
tb.close()
ra0 = phadir[0]
dec0 = phadir[1]
else:
ra0 = eph['ra'][0]
dec0 = eph['dec'][0]
if not xycen:
# use solar disk center as default
phasecenter = 'J2000 ' + str(ra0) + 'rad ' + str(dec0) + 'rad'
else:
x0 = np.radians(xycen[0] / 3600.)
y0 = np.radians(xycen[1] / 3600.)
p0 = np.radians(eph['p0'][0]) # p angle in radians
raoff = -((x0) * np.cos(p0) - y0 * np.sin(p0)) / np.cos(eph['dec'][0])
decoff = (x0) * np.sin(p0) + y0 * np.cos(p0)
newra = ra0 + raoff
newdec = dec0 + decoff
phasecenter = 'J2000 ' + str(newra) + 'rad ' + str(newdec) + 'rad'
return phasecenter, midtim
def read_msinfo(vis=None, msinfofile=None, use_scan_time=True):
import glob
# read MS information #
msinfo = dict.fromkeys([
'vis', 'scans', 'fieldids', 'btimes', 'btimestr', 'inttimes', 'ras',
'decs', 'observatory'
])
ms.open(vis)
metadata = ms.metadata()
observatory = metadata.observatorynames()[0]
scans = ms.getscansummary()
scanids = sorted(scans.keys(), key=lambda x: int(x))
nscanid = len(scanids)
btimes = []
btimestr = []
etimes = []
fieldids = []
inttimes = []
dirs = []
ras = []
decs = []
ephem_file = glob.glob(vis + '/FIELD/EPHEM*SUN.tab')
if ephem_file:
print('Loading ephemeris info from {}'.format(ephem_file[0]))
tb.open(ephem_file[0])
col_ra = tb.getcol('RA')
col_dec = tb.getcol('DEC')
col_mjd = tb.getcol('MJD')
if use_scan_time:
from scipy.interpolate import interp1d
f_ra = interp1d(col_mjd, col_ra)
f_dec = interp1d(col_mjd, col_dec)
for idx, scanid in enumerate(scanids):
btimes.append(scans[scanid]['0']['BeginTime'])
etimes.append(scans[scanid]['0']['EndTime'])
fieldid = scans[scanid]['0']['FieldId']
fieldids.append(fieldid)
inttimes.append(scans[scanid]['0']['IntegrationTime'])
ras = f_ra(np.array(btimes))
decs = f_dec(np.array(btimes))
ras = qa.convert(qa.quantity(ras, 'deg'), 'rad')
decs = qa.convert(qa.quantity(decs, 'deg'), 'rad')
else:
ras = qa.convert(qa.quantity(col_ra, 'deg'), 'rad')
decs = qa.convert(qa.quantity(col_dec, 'deg'), 'rad')
else:
for idx, scanid in enumerate(scanids):
btimes.append(scans[scanid]['0']['BeginTime'])
etimes.append(scans[scanid]['0']['EndTime'])
fieldid = scans[scanid]['0']['FieldId']
fieldids.append(fieldid)
inttimes.append(scans[scanid]['0']['IntegrationTime'])
dir = ms.getfielddirmeas('PHASE_DIR', fieldid)
dirs.append(dir)
ras.append(dir['m0'])
decs.append(dir['m1'])
ms.close()
btimestr = [
qa.time(qa.quantity(btimes[idx], 'd'), form='fits', prec=10)[0]
for idx in range(nscanid)
]
msinfo['vis'] = vis
msinfo['scans'] = scans
msinfo['fieldids'] = fieldids
msinfo['btimes'] = btimes
msinfo['btimestr'] = btimestr
msinfo['inttimes'] = inttimes
msinfo['ras'] = ras
msinfo['decs'] = decs
msinfo['observatory'] = observatory
if msinfofile:
np.savez(msinfofile,
vis=vis,
scans=scans,
fieldids=fieldids,
btimes=btimes,
btimestr=btimestr,
inttimes=inttimes,
ras=ras,
decs=decs,
observatory=observatory)
return msinfo
def subvs2(vis=None,
outputvis=None,
timerange='',
spw='',
mode=None,
subtime1=None,
subtime2=None,
smoothaxis=None,
smoothtype=None,
smoothwidth=None,
splitsel=None,
reverse=None,
overwrite=None):
"""Perform vector subtraction for visibilities
Keyword arguments:
vis -- Name of input visibility file (MS)
default: none; example: vis='ngc5921.ms'
outputvis -- Name of output uv-subtracted visibility file (MS)
default: none; example: outputvis='ngc5921_src.ms'
timerange -- Time range of performing the UV subtraction:
default='' means all times. examples:
timerange = 'YYYY/MM/DD/hh:mm:ss~YYYY/MM/DD/hh:mm:ss'
timerange = 'hh:mm:ss~hh:mm:ss'
spw -- Select spectral window/channel.
default = '' all the spectral channels. Example: spw='0:1~20'
mode -- operation mode
default 'linear'
mode = 'linear': use a linear fit for the background to be subtracted
mode = 'lowpass': act as a lowpass filter---smooth the data using different
smooth types and window sizes. Can be performed along either time
or frequency axis
mode = 'highpass': act as a highpass filter---smooth the data first, and
subtract the smoothed data from the original. Can be performed along
either time or frequency axis
mode = 'linear' expandable parameters:
subtime1 -- Time range 1 of the background to be subtracted from the data
default='' means all times. format:
timerange = 'YYYY/MM/DD/hh:mm:ss~YYYY/MM/DD/hh:mm:ss'
timerange = 'hh:mm:ss~hh:mm:ss'
subtime2 -- Time range 2 of the backgroud to be subtracted from the data
default='' means all times. examples:
timerange = 'YYYY/MM/DD/hh:mm:ss~YYYY/MM/DD/hh:mm:ss'
timerange = 'hh:mm:ss~hh:mm:ss'
mode = 'lowpass' or 'highpass' expandable parameters:
smoothaxis -- axis of smooth
Default: 'time'
smoothaxis = 'time': smooth is along the time axis
smoothaxis = 'freq': smooth is along the frequency axis
smoothtype -- type of the smooth depending on the convolving kernel
Default: 'flat'
smoothtype = 'flat': convolving kernel is a flat rectangle,
equivalent to a boxcar moving smooth
smoothtype = 'hanning': Hanning smooth kernel. See numpy.hanning
smoothtype = 'hamming': Hamming smooth kernel. See numpy.hamming
smoothtype = 'bartlett': Bartlett smooth kernel. See numpy.bartlett
smoothtype = 'blackman': Blackman smooth kernel. See numpy.blackman
smoothwidth -- width of the smooth kernel
Default: 5
Examples: smoothwidth=5, meaning the width is 5 pixels
splitsel -- True or False. default = False. If splitsel = False, then the entire input
measurement set is copied as the output measurement set (outputvis), with
background subtracted at selected timerange and spectral channels.
If splitsel = True,then only the selected timerange and spectral channels
are copied into the output measurement set (outputvis).
reverse -- True or False. default = False. If reverse = False, then the times indicated
by subtime1 and/or subtime2 are treated as background and subtracted; If reverse
= True, then reverse the sign of the background-subtracted data. The option can
be used for mapping absorptive structure.
overwrite -- True or False. default = False. If overwrite = True and
outputvis already exists, the selected subtime and spw in the
output measurment set will be replaced with background subtracted
visibilities
"""
# check the visbility ms
casalog.post('input parameters:')
casalog.post('vis: ' + vis)
casalog.post('outputvis: ' + outputvis)
casalog.post('smoothaxis: ' + smoothaxis)
casalog.post('smoothtype: ' + smoothtype)
casalog.post('smoothwidth: ' + str(smoothwidth))
if not outputvis or outputvis.isspace():
raise (ValueError, 'Please specify outputvis')
if os.path.exists(outputvis):
if overwrite:
print(
"The already existing output measurement set will be updated.")
else:
raise (ValueError,
"Output MS %s already exists - will not overwrite." %
outputvis)
else:
if not splitsel:
shutil.copytree(vis, outputvis)
else:
ms.open(vis, nomodify=True)
ms.split(outputvis, spw=spw, time=timerange, whichcol='DATA')
ms.close()
if timerange and (type(timerange) == str):
[btimeo, etimeo] = timerange.split('~')
btimeosec = qa.getvalue(qa.convert(qa.totime(btimeo), 's'))
etimeosec = qa.getvalue(qa.convert(qa.totime(etimeo), 's'))
timebinosec = etimeosec - btimeosec
if timebinosec < 0:
raise Exception(
'Negative timebin! Please check the "timerange" parameter.')
casalog.post('Selected timerange: ' + timerange +
' as the time for UV subtraction.')
else:
casalog.post(
'Output timerange not specified, using the entire timerange')
if spw and (type(spw) == str):
spwlist = spw.split(';')
else:
casalog.post('spw not specified, use all frequency channels')
# read the output data
datams = mstool()
datams.open(outputvis, nomodify=False)
datamsmd = msmdtool()
datamsmd.open(outputvis)
spwinfod = datams.getspectralwindowinfo()
spwinfok = spwinfod.keys()
spwinfok.sort(key=int)
spwinfol = [spwinfod[k] for k in spwinfok]
for s, spi in enumerate(spwinfol):
print('processing spectral window {}'.format(spi['SpectralWindowId']))
datams.selectinit(reset=True)
staql = {'time': '', 'spw': ''}
if not splitsel:
# outputvis is identical to input visibility, do the selection
if timerange and (type(timerange == str)):
staql['time'] = timerange
if spw and (type(spw) == str):
staql['spw'] = spwlist[s]
if not spw and not timerange:
# data selection is not made
print('selecting all spws and times')
staql['spw'] = str(spi['SpectralWindowId'])
else:
# outputvis is splitted, selections have already applied, select all the data
print('split the selected spws and times')
staql['spw'] = str(spi['SpectralWindowId'])
datams.msselect(staql)
orec = datams.getdata(['data', 'time', 'axis_info'], ifraxis=True)
npol, nchan, nbl, ntim = orec['data'].shape
print('dimension of output data', orec['data'].shape)
casalog.post('Number of baselines: ' + str(nbl))
casalog.post('Number of spectral channels: ' + str(nchan))
casalog.post('Number of time pixels: ' + str(ntim))
try:
if mode == 'linear':
# define and check the background time ranges
if subtime1 and (type(subtime1) == str):
[bsubtime1, esubtime1] = subtime1.split('~')
bsubtime1sec = qa.getvalue(
qa.convert(qa.totime(bsubtime1), 's'))
esubtime1sec = qa.getvalue(
qa.convert(qa.totime(esubtime1), 's'))
timebin1sec = esubtime1sec - bsubtime1sec
if timebin1sec < 0:
raise Exception(
'Negative timebin! Please check the "subtime1" parameter.'
)
casalog.post('Selected timerange 1: ' + subtime1 +
' as background for uv subtraction.')
else:
raise Exception(
'Please enter at least one timerange as the background'
)
if subtime2 and (type(subtime2) == str):
[bsubtime2, esubtime2] = subtime2.split('~')
bsubtime2sec = qa.getvalue(
qa.convert(qa.totime(bsubtime2), 's'))
esubtime2sec = qa.getvalue(
qa.convert(qa.totime(esubtime2), 's'))
timebin2sec = esubtime2sec - bsubtime2sec
if timebin2sec < 0:
raise Exception(
'Negative timebin! Please check the "subtime2" parameter.'
)
timebin2 = str(timebin2sec) + 's'
casalog.post('Selected timerange 2: ' + subtime2 +
' as background for uv subtraction.')
# plus 1s is to ensure averaging over the entire timerange
else:
casalog.post(
'Timerange 2 not selected, using only timerange 1 as background'
)
# Select the background indicated by subtime1
ms.open(vis, nomodify=True)
# Select the spw id
# ms.msselect({'time': subtime1})
staql0 = {'time': subtime1, 'spw': ''}
if spw and (type(spw) == str):
staql0['spw'] = spwlist[s]
else:
staql0['spw'] = staql['spw']
ms.msselect(staql0)
rec1 = ms.getdata(['data', 'time', 'axis_info'], ifraxis=True)
# print('shape of the frequency matrix ',rec1['axis_info']['freq_axis']['chan_freq'].shape)
sz1 = rec1['data'].shape
print('dimension of selected background 1', rec1['data'].shape)
# the data shape is (n_pol,n_channel,n_baseline,n_time), no need to reshape
# rec1['data']=rec1['data'].reshape(sz1[0],sz1[1],sz1[2],nspw,sz1[3]/nspw,order='F')
# print('reshaped rec1 ', rec1['data'].shape)
rec1avg = np.average(rec1['data'], axis=3)
casalog.post('Averaging the visibilities in subtime1: ' +
subtime1)
ms.close()
if subtime2 and (type(subtime2) == str):
ms.open(vis, nomodify=True)
# Select the spw id
staql0 = {'time': subtime2, 'spw': ''}
if spw and (type(spw) == str):
staql0['spw'] = spwlist[s]
else:
staql0['spw'] = staql['spw']
ms.msselect(staql0)
rec2 = ms.getdata(['data', 'time', 'axis_info'],
ifraxis=True)
sz2 = rec2['data'].shape
print('dimension of selected background 2',
rec2['data'].shape)
# rec2['data']=rec2['data'].reshape(sz2[0],sz2[1],sz2[2],nspw,sz2[3]/nspw,order='F')
# print('reshaped rec1 ', rec2['data'].shape)
rec2avg = np.average(rec2['data'], axis=3)
ms.close()
casalog.post('Averaged the visibilities in subtime2: ' +
subtime2)
if subtime1 and (not subtime2):
casalog.post(
'Only "subtime1" is defined, subtracting background defined in subtime1: '
+ subtime1)
t1 = (np.amax(rec1['time']) + np.amin(rec1['time'])) / 2.
print('t1: ',
qa.time(qa.quantity(t1, 's'), form='ymd', prec=10))
for i in range(ntim):
orec['data'][:, :, :, i] -= rec1avg
if reverse:
orec['data'][:, :, :,
i] = -orec['data'][:, :, :, i]
if subtime1 and subtime2 and (type(subtime2) == str):
casalog.post(
'Both subtime1 and subtime2 are specified, doing linear interpolation between "subtime1" and "subtime2"'
)
t1 = (np.amax(rec1['time']) + np.amin(rec1['time'])) / 2.
t2 = (np.amax(rec2['time']) + np.amin(rec2['time'])) / 2.
touts = orec['time']
print('t1: ',
qa.time(qa.quantity(t1, 's'), form='ymd', prec=10))
print('t2: ',
qa.time(qa.quantity(t2, 's'), form='ymd', prec=10))
for i in range(ntim):
tout = touts[i]
if tout > np.amax([t1, t2]):
tout = np.amax([t1, t2])
elif tout < np.amin([t1, t2]):
tout = np.amin([t1, t2])
orec['data'][:, :, :, i] -= (rec2avg - rec1avg) * (
tout - t1) / (t2 - t1) + rec1avg
if reverse:
orec['data'][:, :, :,
i] = -orec['data'][:, :, :, i]
elif mode == 'highpass':
if smoothtype != 'flat' and smoothtype != 'hanning' and smoothtype != 'hamming' and smoothtype != 'bartlett' and smoothtype != 'blackman':
raise Exception('Unknown smoothtype ' + str(smoothtype))
if smoothaxis == 'time':
if smoothwidth <= 0 or smoothwidth >= ntim:
raise Exception(
'Specified smooth width is <=0 or >= the total number of '
+ smoothaxis)
else:
for i in range(orec['data'].shape[0]):
for j in range(nchan):
for k in range(nbl):
orec['data'][i, j,
k, :] -= signalsmooth.smooth(
orec['data'][i, j, k, :],
smoothwidth, smoothtype)
if smoothaxis == 'freq':
if smoothwidth <= 0 or smoothwidth >= nchan:
raise Exception(
'Specified smooth width is <=0 or >= the total number of '
+ smoothaxis)
else:
for i in range(orec['data'].shape[0]):
for j in range(nbl):
for k in range(ntim):
orec['data'][i, :, j,
k] -= signalsmooth.smooth(
orec['data'][i, :, j, k],
smoothwidth, smoothtype)
elif mode == 'lowpass':
if smoothtype != 'flat' and smoothtype != 'hanning' and smoothtype != 'hamming' and smoothtype != 'bartlett' and smoothtype != 'blackman':
raise Exception('Unknown smoothtype ' + str(smoothtype))
if smoothaxis == 'time':
if smoothwidth <= 0 or smoothwidth >= ntim:
raise Exception(
'Specified smooth width is <=0 or >= the total number of '
+ smoothaxis)
else:
for i in range(orec['data'].shape[0]):
for j in range(nchan):
for k in range(nbl):
orec['data'][i, j,
k, :] = signalsmooth.smooth(
orec['data'][i, j, k, :],
smoothwidth, smoothtype)
if smoothaxis == 'freq':
if smoothwidth <= 0 or smoothwidth >= nchan:
raise Exception(
'Specified smooth width is <=0 or >= the total number of '
+ smoothaxis)
else:
for i in range(orec['data'].shape[0]):
for j in range(nbl):
for k in range(ntim):
orec['data'][i, :, j,
k] = signalsmooth.smooth(
orec['data'][i, :, j, k],
smoothwidth, smoothtype)
else:
raise Exception('Unknown mode' + str(mode))
except Exception as instance:
print('*** Error ***', instance)
# orec['data']=orec['data'].reshape(szo[0],szo[1],szo[2],szo[3],order='F')
# put the modified data back into the output visibility set
del orec['time']
del orec['axis_info']
# ms.open(outputvis,nomodify=False)
# if not splitsel:
# outputvis is identical to input visibility, do the selection
# if timerange and (type(timerange==str)):
# datams.msselect({'time':timerange})
# if spw and (type(spw)==str):
# datams.selectinit(datadescid=int(spwid))
# nchan=int(echan)-int(bchan)+1
# datams.selectchannel(nchan,int(bchan),1,1)
# if not spw and not timerange:
# data selection is not made
# datams.selectinit(datadescid=0)
# else:
# outputvis is splitted, selections have already applied, select all the data
# datams.selectinit(datadescid=0)
datams.putdata(orec)
datams.close()
datamsmd.done()
def clean_iter(
tim, vis, imageprefix, imagesuffix, twidth, doreg, usephacenter,
reftime, ephem, msinfo, toTb, overwrite, selectdata, field, spw,
uvrange, antenna, scan, observation, intent, datacolumn, imsize, cell,
phasecenter, stokes, projection, startmodel, specmode, reffreq, nchan,
start, width, outframe, veltype, restfreq, interpolation, gridder,
facets, chanchunks, wprojplanes, vptable, usepointing, mosweight,
aterm, psterm, wbawp, conjbeams, cfcache, computepastep, rotatepastep,
pblimit, normtype, deconvolver, scales, nterms, smallscalebias,
restoration, restoringbeam, pbcor, outlierfile, weighting, robust,
npixels, uvtaper, niter, gain, threshold, nsigma, cycleniter,
cyclefactor, minpsffraction, maxpsffraction, interactive, usemask,
mask, pbmask, sidelobethreshold, noisethreshold, lownoisethreshold,
negativethreshold, smoothfactor, minbeamfrac, cutthreshold,
growiterations, dogrowprune, minpercentchange, verbose, restart,
savemodel, calcres, calcpsf, parallel, tmpdir, btidx):
from tclean_cli import tclean_cli as tclean
from split_cli import split_cli as split
bt = btidx # 0
if bt + twidth < len(tim) - 1:
et = btidx + twidth - 1
else:
et = len(tim) - 1
if bt == 0:
bt_d = tim[bt] - ((tim[bt + 1] - tim[bt]) / 2)
else:
bt_d = tim[bt] - ((tim[bt] - tim[bt - 1]) / 2)
if et == (len(tim) - 1) or et == -1:
et_d = tim[et] + ((tim[et] - tim[et - 1]) / 2)
else:
et_d = tim[et] + ((tim[et + 1] - tim[et]) / 2)
timerange = qa.time(qa.quantity(bt_d, 's'), prec=9, form='ymd')[0] + '~' + \
qa.time(qa.quantity(et_d, 's'), prec=9, form='ymd')[0]
btstr = qa.time(qa.quantity(bt_d, 's'), prec=9, form='fits')[0]
etstr = qa.time(qa.quantity(et_d, 's'), prec=9, form='fits')[0]
print 'cleaning timerange: ' + timerange
image0 = btstr.replace(':', '').replace('-', '')
imname = imageprefix + image0 + imagesuffix
ms_tmp = tmpdir + image0 + '.ms'
print('checkpoint 1')
# split(vis=vis, outputvis=ms_tmp, field=field, scan=scan, antenna=antenna, timerange=timerange,
# datacolumn=datacolumn)
ms.open(vis)
print('checkpoint 1-1')
ms.split(ms_tmp,
field=field,
scan=scan,
baseline=antenna,
time=timerange,
whichcol=datacolumn)
print('checkpoint 1-2')
ms.close()
print('checkpoint 2')
if overwrite or (len(glob.glob(imname + '*')) == 0):
os.system('rm -rf {}*'.format(imname))
try:
tclean(vis=ms_tmp,
selectdata=selectdata,
field=field,
spw=spw,
timerange=timerange,
uvrange=uvrange,
antenna=antenna,
scan=scan,
observation=observation,
intent=intent,
datacolumn=datacolumn,
imagename=imname,
imsize=imsize,
cell=cell,
phasecenter=phasecenter,
stokes=stokes,
projection=projection,
startmodel=startmodel,
specmode=specmode,
reffreq=reffreq,
nchan=nchan,
start=start,
width=width,
outframe=outframe,
veltype=veltype,
restfreq=restfreq,
interpolation=interpolation,
gridder=gridder,
facets=facets,
chanchunks=chanchunks,
wprojplanes=wprojplanes,
vptable=vptable,
usepointing=usepointing,
mosweight=mosweight,
aterm=aterm,
psterm=psterm,
wbawp=wbawp,
conjbeams=conjbeams,
cfcache=cfcache,
computepastep=computepastep,
rotatepastep=rotatepastep,
pblimit=pblimit,
normtype=normtype,
deconvolver=deconvolver,
scales=scales,
nterms=nterms,
smallscalebias=smallscalebias,
restoration=restoration,
restoringbeam=restoringbeam,
pbcor=pbcor,
outlierfile=outlierfile,
weighting=weighting,
robust=robust,
npixels=npixels,
uvtaper=uvtaper,
niter=niter,
gain=gain,
threshold=threshold,
nsigma=nsigma,
cycleniter=cycleniter,
cyclefactor=cyclefactor,
minpsffraction=minpsffraction,
maxpsffraction=maxpsffraction,
interactive=interactive,
usemask=usemask,
mask=mask,
pbmask=pbmask,
sidelobethreshold=sidelobethreshold,
noisethreshold=noisethreshold,
lownoisethreshold=lownoisethreshold,
negativethreshold=negativethreshold,
smoothfactor=smoothfactor,
minbeamfrac=minbeamfrac,
cutthreshold=cutthreshold,
growiterations=growiterations,
dogrowprune=dogrowprune,
minpercentchange=minpercentchange,
verbose=verbose,
restart=restart,
savemodel=savemodel,
calcres=calcres,
calcpsf=calcpsf,
parallel=parallel)
print('checkpoint 3')
clnjunks = [
'.flux', '.mask', '.model', '.psf', '.residual', '.pb',
'.sumwt', '.image.pbcor'
]
for clnjunk in clnjunks:
if os.path.exists(imname + clnjunk):
shutil.rmtree(imname + clnjunk)
except:
print('error in cleaning image: ' + btstr)
return [False, btstr, etstr, '']
else:
print imname + ' exists. Clean task aborted.'
if doreg and not os.path.isfile(imname + '.fits'):
# ephem.keys()
# msinfo.keys()
try:
# check if ephemfile and msinfofile exist
if not ephem:
print(
"ephemeris info does not exist, querying from JPL Horizons on the fly"
)
ephem = hf.read_horizons(vis=vis)
if not msinfo:
print("ms info not provided, generating one on the fly")
msinfo = hf.read_msinfo(vis)
hf.imreg(vis=vis,
ephem=ephem,
msinfo=msinfo,
timerange=timerange,
reftime=reftime,
imagefile=imname + '.image',
fitsfile=imname + '.fits',
toTb=toTb,
scl100=False,
usephacenter=usephacenter)
if os.path.exists(imname + '.fits'):
shutil.rmtree(imname + '.image')
return [True, btstr, etstr, imname + '.fits']
else:
return [False, btstr, etstr, '']
except:
print('error in registering image: ' + btstr)
return [False, btstr, etstr, imname + '.image']
else:
if os.path.exists(imname + '.image'):
return [True, btstr, etstr, imname + '.image']
else:
return [False, btstr, etstr, '']
def subvs(vis='',
outputvis='',
timerange='',
spw='',
timoffset=4,
windowlen=5,
windowtype='hamming',
splitsel=True,
reverse=False,
overwrite=False):
"""Perform vector subtraction for visibilities
Keyword arguments:
vis -- Name of input visibility file (MS)
default: none; example: vis='ngc5921.ms'
outputvis -- Name of output uv-subtracted visibility file (MS)
default: none; example: outputvis='ngc5921_src.ms'
timerange -- Time range of performing the UV subtraction:
default='' means all times. examples:
timerange = 'YYYY/MM/DD/hh:mm:ss~YYYY/MM/DD/hh:mm:ss'
timerange = 'hh:mm:ss~hh:mm:ss'
spw -- Select spectral window/channel.
windowlen -- Specify the width of window for smoothing
windowtype --The type of window from 'flat', 'hanning', 'hamming', 'bartlett', 'blackman'
flat window will produce a moving average smoothing.
splitsel -- True or False. default = False. If splitsel = False, then the entire input
measurement set is copied as the output measurement set (outputvis), with
background subtracted at selected timerange and spectral channels.
If splitsel = True,then only the selected timerange and spectral channels
are copied into the output measurement set (outputvis).
reverse -- True or False. default = False. If reverse = False, then the times indicated
by subtime1 and/or subtime2 are treated as background and subtracted; If reverse
= True, then reverse the sign of the background-subtracted data. The option can
be used for mapping absorptive structure.
overwrite -- True or False. default = False. If overwrite = True and
outputvis already exists, the selected subtime and spw in the
output measurment set will be replaced with background subtracted
visibilities
"""
# Get the time and frequency axis of the input ms
# Open the ms and plot dynamic spectrum
print 'using window length: ', windowlen
print 'using window type: ', windowtype
ms.open(vis, nomodify=True)
# ms.selectinit(datadescid=0)
timfreq = ms.getdata(['time', 'axis_info'], ifraxis=True)
tim = timfreq['time']
# check timerange input; default: entire timerange
if timerange and (type(timerange) == str):
[btimeo, etimeo] = timerange.split('~')
btimeosec = qa.getvalue(qa.convert(qa.totime(btimeo), 's'))
etimeosec = qa.getvalue(qa.convert(qa.totime(etimeo), 's'))
timebinosec = etimeosec - btimeosec
if timebinosec < 0:
raise Exception, 'Negative timebin! Please check the "timerange" parameter.'
else:
casalog.post('Selected timerange: ' + timerange +
' as the time for UV subtraction.')
else:
casalog.post(
'Output timerange not specified, using the entire timerange')
timerange = str(qa.time(qa.quantity(
tim[0], 's'), prec=8)[0]) + '~' + str(
qa.time(qa.quantity(tim[-1], 's'), prec=8)[0])
print 'Output timerange not specified, using the entire timerange', timerange
# check spectral window input; default: entire channels of spectral window 0
if spw and (type(spw) == str):
[spwid, chanran] = spw.split(':')
[bchan, echan] = chanran.split('~')
nchan = int(echan) - int(bchan) + 1
else:
casalog.post('spw not specified, use all frequency channels')
freq = timfreq['axis_info']['freq_axis']['chan_freq'].flatten()
nchan = len(freq)
spwid = '0'
bchan = '0'
echan = str(nchan - 1)
print 'spw not specified, use all frequency channels', spwid + ':' + bchan + '~' + str(
nchan - 1)
ntimergn = len(timerange)
# To avoid memory error, split the channel into smaller segements for smoothing
cellstep = 2
chancell = int(nchan / cellstep)
l = range(nchan)
chunks = [l[x:x + cellstep] for x in xrange(0, len(l), cellstep)]
#spwrange='0:0~'+str(chancell)
ms.close()
if not (timoffset and (type(timoffset) == int)):
timoffset = int(4)
for i in range(len(chunks)):
spwrange = spwid + ':' + str(int(bchan) + min(chunks[i])) + '~' + str(
int(bchan) + max(chunks[i]))
print 'Subtracting visibility from spectral range: ', spwrange
result2 = task_subvs_lv1.subvs(vis, outputvis, timerange, spwrange,
timoffset, windowlen, windowtype,
splitsel, False, True)
def mk_qlook_image(vis, ncpu=10, twidth=12, stokes='I,V', antenna='', imagedir=None, spws=[], toTb=True, overwrite=True, doslfcal=False,
phasecenter='', c_external=True):
vis = [vis]
subdir = ['/']
for idx, f in enumerate(vis):
if f[-1] == '/':
vis[idx] = f[:-1]
if not imagedir:
imagedir = './'
imres = {'Succeeded': [], 'BeginTime': [], 'EndTime': [], 'ImageName': [], 'Spw': [], 'Vis': [], 'Freq': []}
msfile = vis[0]
ms.open(msfile)
metadata = ms.metadata()
observatory = metadata.observatorynames()[0]
# axisInfo = ms.getdata(["axis_info"], ifraxis=True)
spwInfo = ms.getspectralwindowinfo()
# freqInfo = axisInfo["axis_info"]["freq_axis"]["chan_freq"].swapaxes(0, 1) / 1e9
# freqInfo_ravel = freqInfo.ravel()
ms.close()
if not spws:
if observatory == 'EVLA':
spws = ['0', '1', '2', '3', '4', '5', '6', '7']
if observatory == 'EOVSA':
spws = ['1~5', '6~10', '11~15', '16~25']
if observatory == 'EOVSA':
print 'Provide stokes: ' + str(stokes) + '. However EOVSA has linear feeds. Force stokes to be IV'
stokes = 'I,V'
msfilebs = os.path.basename(msfile)
imdir = imagedir + subdir[0]
if not os.path.exists(imdir):
os.makedirs(imdir)
if doslfcal:
slfcalms = './' + msfilebs + '.rr'
split(msfile, outputvis=slfcalms, datacolumn='corrected', correlation='RR')
for spw in spws:
spwran = [s.zfill(2) for s in spw.split('~')]
# freqran = [
# (int(s) * spwInfo['0']['TotalWidth'] + spwInfo['0']['RefFreq'] + spwInfo['0']['TotalWidth'] / 2.0) / 1.0e9
# for s in spw.split('~')]
spw_ = spw.split('~')
if len(spw_) == 2:
freqran = [(spwInfo['{}'.format(s)]['RefFreq'] + spwInfo['{}'.format(s)]['TotalWidth'] / 2.0) / 1.0e9 for s in spw.split('~')]
elif len(spw_) == 1:
s = spw_[0]
freqran = np.array([0, spwInfo['{}'.format(s)]['TotalWidth']]) + spwInfo['{}'.format(s)]['RefFreq']
freqran = freqran / 1.0e9
freqran = list(freqran)
else:
raise ValueError("Keyword 'spw' in wrong format")
cfreq = np.mean(freqran)
bmsz = max(30. / cfreq, 30.)
uvrange = '<3km'
if cfreq < 10.:
imsize = 512
cell = ['5arcsec']
else:
imsize = 1024
cell = ['2.5arcsec']
if len(spwran) == 2:
spwstr = spwran[0] + '~' + spwran[1]
else:
spwstr = spwran[0]
restoringbeam = ['{0:.1f}arcsec'.format(bmsz)]
imagesuffix = '.spw' + spwstr.replace('~', '-')
# if cfreq > 10.:
# antenna = antenna + ';!0&1;!0&2' # deselect the shortest baselines
sto = stokes.replace(',', '')
if c_external:
cleanscript = os.path.join(imdir, 'ptclean_external.py')
resfile = os.path.join(imdir, os.path.basename(msfile) + '.res.npz')
os.system('rm -rf {}'.format(cleanscript))
inpdict = {'vis': msfile, 'imageprefix': imdir, 'imagesuffix': imagesuffix, 'twidth': twidth, 'uvrange': uvrange, 'spw': spw,
'ncpu': ncpu, 'niter': 1000, 'gain': 0.05, 'antenna': antenna, 'imsize': imsize, 'cell': cell, 'stokes': sto, 'doreg': True,
'overwrite': overwrite, 'toTb': toTb, 'restoringbeam': restoringbeam, 'uvtaper': True, 'outertaper': ['30arcsec'],
'phasecenter': phasecenter}
for key, val in inpdict.items():
if type(val) is str:
inpdict[key] = '"{}"'.format(val)
fi = open(cleanscript, 'wb')
fi.write('from ptclean_cli import ptclean_cli as ptclean \n')
fi.write('import numpy as np \n')
fi.write(
'res = ptclean(vis={i[vis]},imageprefix={i[imageprefix]},imagesuffix={i[imagesuffix]},twidth={i[twidth]},uvrange={i[uvrange]},spw={i[spw]},ncpu={i[ncpu]},niter={i[niter]},gain={i[gain]},antenna={i[antenna]},imsize={i[imsize]},cell={i[cell]},stokes={i[stokes]},doreg={i[doreg]},overwrite={i[overwrite]},toTb={i[toTb]},restoringbeam={i[restoringbeam]},uvtaper={i[uvtaper]},outertaper={i[outertaper]},phasecenter={i[phasecenter]}) \n'.format(
i=inpdict))
fi.write('np.savez("{}",res=res) \n'.format(resfile))
fi.close()
os.system('casa --nologger -c {}'.format(cleanscript))
res = np.load(resfile)
res = res['res'].item()
else:
res = ptclean(vis=msfile, imageprefix=imdir, imagesuffix=imagesuffix, twidth=twidth, uvrange=uvrange, spw=spw, ncpu=ncpu, niter=1000,
gain=0.05, antenna=antenna, imsize=imsize, cell=cell, stokes=sto, doreg=True, overwrite=overwrite, toTb=toTb,
restoringbeam=restoringbeam, uvtaper=True, outertaper=['30arcsec'], phasecenter=phasecenter)
if res:
imres['Succeeded'] += res['Succeeded']
imres['BeginTime'] += res['BeginTime']
imres['EndTime'] += res['EndTime']
imres['ImageName'] += res['ImageName']
imres['Spw'] += [spwstr] * len(res['ImageName'])
imres['Vis'] += [msfile] * len(res['ImageName'])
imres['Freq'] += [freqran] * len(res['ImageName'])
else:
return None
# save it for debugging purposes
np.savez(os.path.join(imagedir, '{}.imres.npz'.format(os.path.basename(msfile))), imres=imres)
return imres
def read_horizons(t0=None,
dur=None,
vis=None,
observatory=None,
verbose=False):
import urllib2
import ssl
if not t0 and not vis:
t0 = Time.now()
if not dur:
dur = 1. / 60. / 24. # default to 2 minutes
if t0:
try:
btime = Time(t0)
except:
print('input time ' + str(t0) + ' not recognized')
return -1
if vis:
if not os.path.exists(vis):
print 'Input ms data ' + vis + ' does not exist! '
return -1
try:
# ms.open(vis)
# summary = ms.summary()
# ms.close()
# btime = Time(summary['BeginTime'], format='mjd')
# etime = Time(summary['EndTime'], format='mjd')
## alternative way to avoid conflicts with importeovsa, if needed -- more time consuming
if observatory == 'geocentric':
observatory = '500'
else:
ms.open(vis)
metadata = ms.metadata()
if metadata.observatorynames()[0] == 'EVLA':
observatory = '-5'
elif metadata.observatorynames()[0] == 'EOVSA':
observatory = '-81'
elif metadata.observatorynames()[0] == 'ALMA':
observatory = '-7'
ms.close()
tb.open(vis)
btime_vis = Time(tb.getcell('TIME', 0) / 24. / 3600., format='mjd')
etime_vis = Time(tb.getcell('TIME',
tb.nrows() - 1) / 24. / 3600.,
format='mjd')
tb.close()
if verbose:
print "Beginning time of this scan " + btime_vis.iso
print "End time of this scan " + etime_vis.iso
# extend the start and end time for jpl horizons by 0.5 hr on each end
btime = Time(btime_vis.mjd - 0.5 / 24., format='mjd')
dur = etime_vis.mjd - btime_vis.mjd + 1.0 / 24.
except:
print 'error in reading ms file: ' + vis + ' to obtain the ephemeris!'
return -1
# default the observatory to VLA, if none provided
if not observatory:
observatory = '-5'
etime = Time(btime.mjd + dur, format='mjd')
try:
cmdstr = "https://ssd.jpl.nasa.gov/horizons_batch.cgi?batch=1&TABLE_TYPE='OBSERVER'&QUANTITIES='1,17,20'&CSV_FORMAT='YES'&ANG_FORMAT='DEG'&CAL_FORMAT='BOTH'&SOLAR_ELONG='0,180'&CENTER='{}@399'&COMMAND='10'&START_TIME='".format(
observatory
) + btime.iso.replace(
' ', ','
) + "'&STOP_TIME='" + etime.iso[:-4].replace(
' ', ','
) + "'&STEP_SIZE='1m'&SKIP_DAYLT='NO'&EXTRA_PREC='YES'&APPARENT='REFRACTED'"
cmdstr = cmdstr.replace("'", "%27")
try:
context = ssl._create_unverified_context()
f = urllib2.urlopen(cmdstr, context=context)
except:
f = urllib2.urlopen(cmdstr)
lines = f.readlines()
f.close()
except:
#todo use geocentric coordinate for the new VLA data
import requests, collections
params = collections.OrderedDict()
params['batch'] = '1'
params['TABLE_TYPE'] = "'OBSERVER'"
params['QUANTITIES'] = "'1,17,20'"
params['CSV_FORMAT'] = "'YES'"
params['ANG_FORMAT'] = "'DEG'"
params['CAL_FORMAT'] = "'BOTH'"
params['SOLAR_ELONG'] = "'0,180'"
if observatory == '500':
params['CENTER'] = "'500'"
else:
params['CENTER'] = "'{}@399'".format(observatory)
params['COMMAND'] = "'10'"
params['START_TIME'] = "'{}'".format(btime.iso[:-4].replace(' ', ','))
params['STOP_TIME'] = "'{}'".format(etime.iso[:-4].replace(' ', ','))
params['STEP_SIZE'] = "'1m'"
params['SKIP_DAYLT'] = "'NO'"
params['EXTRA_PREC'] = "'YES'"
params['APPAENT'] = "'REFRACTED'"
results = requests.get("https://ssd.jpl.nasa.gov/horizons_batch.cgi",
params=params)
lines = [ll for ll in results.iter_lines()]
nline = len(lines)
istart = 0
for i in range(nline):
line = lines[i]
if line[0:5] == '$$SOE': # start recording
istart = i + 1
if line[0:5] == '$$EOE': # end recording
iend = i
newlines = lines[istart:iend]
nrec = len(newlines)
ephem_ = []
t = []
ra = []
dec = []
p0 = []
delta = []
for line in newlines:
items = line.split(',')
t.append(Time(float(items[1]), format='jd').mjd)
ra.append(np.radians(float(items[4])))
dec.append(np.radians(float(items[5])))
p0.append(float(items[6]))
delta.append(float(items[8]))
# convert list of dictionary to a dictionary of arrays
ephem = {'time': t, 'ra': ra, 'dec': dec, 'p0': p0, 'delta': delta}
return ephem
def ptclean3(vis, imageprefix, imagesuffix, ncpu, twidth, doreg, usephacenter,
reftime, toTb, overwrite, selectdata, field, spw, timerange,
uvrange, antenna, scan, observation, intent, datacolumn, imsize,
cell, phasecenter, stokes, projection, startmodel, specmode,
reffreq, nchan, start, width, outframe, veltype, restfreq,
interpolation, gridder, facets, chanchunks, wprojplanes, vptable,
usepointing, mosweight, aterm, psterm, wbawp, conjbeams, cfcache,
computepastep, rotatepastep, pblimit, normtype, deconvolver,
scales, nterms, smallscalebias, restoration, restoringbeam, pbcor,
outlierfile, weighting, robust, npixels, uvtaper, niter, gain,
threshold, nsigma, cycleniter, cyclefactor, minpsffraction,
maxpsffraction, interactive, usemask, mask, pbmask,
sidelobethreshold, noisethreshold, lownoisethreshold,
negativethreshold, smoothfactor, minbeamfrac, cutthreshold,
growiterations, dogrowprune, minpercentchange, verbose, restart,
savemodel, calcres, calcpsf, parallel, subregion):
if not (type(ncpu) is int):
casalog.post('ncpu should be an integer')
ncpu = 8
if doreg:
# check if ephem and msinfo exist. If not, generate one on the fly
try:
ephem = hf.read_horizons(vis=vis)
except ValueError:
print("error in obtaining ephemeris")
try:
msinfo = hf.read_msinfo(vis)
except ValueError:
print("error in getting ms info")
else:
ephem = None
msinfo = None
if imageprefix:
workdir = os.path.dirname(imageprefix)
else:
workdir = './'
tmpdir = workdir + '/tmp/'
if not os.path.exists(tmpdir):
os.makedirs(tmpdir)
# get number of time pixels
ms.open(vis)
ms.selectinit()
timfreq = ms.getdata(['time', 'axis_info'], ifraxis=True)
tim = timfreq['time']
ms.close()
if twidth < 1:
casalog.post('twidth less than 1. Change to 1')
twidth = 1
if twidth > len(tim):
casalog.post(
'twidth greater than # of time pixels in the dataset. Change to the timerange of the entire dateset'
)
twidth = len(tim)
# find out the start and end time index according to the parameter timerange
# if not defined (empty string), use start and end from the entire time of the ms
if not timerange:
btidx = 0
etidx = len(tim) - 1
else:
try:
(tstart, tend) = timerange.split('~')
bt_s = qa.convert(qa.quantity(tstart, 's'), 's')['value']
et_s = qa.convert(qa.quantity(tend, 's'), 's')['value']
# only time is given but not date, add the date (at 0 UT) from the first record
if bt_s < 86400. or et_s < 86400.:
bt_s += np.fix(
qa.convert(qa.quantity(tim[0], 's'),
'd')['value']) * 86400.
et_s += np.fix(
qa.convert(qa.quantity(tim[0], 's'),
'd')['value']) * 86400.
btidx = np.argmin(np.abs(tim - bt_s))
etidx = np.argmin(np.abs(tim - et_s))
# make the indice back to those bracket by the timerange
if tim[btidx] < bt_s:
btidx += 1
if tim[etidx] > et_s:
etidx -= 1
if etidx <= btidx:
print("ending time must be greater than starting time")
print("reinitiating to the entire time range")
btidx = 0
etidx = len(tim) - 1
except ValueError:
print("keyword 'timerange' has a wrong format")
btstr = qa.time(qa.quantity(tim[btidx], 's'), prec=9, form='fits')[0]
etstr = qa.time(qa.quantity(tim[etidx], 's'), prec=9, form='fits')[0]
iterable = range(btidx, etidx + 1, twidth)
print('First time pixel: ' + btstr)
print('Last time pixel: ' + etstr)
print(str(len(iterable)) + ' images to clean...')
res = []
# partition
clnpart = partial(
clean_iter, tim, vis, imageprefix, imagesuffix, twidth, doreg,
usephacenter, reftime, ephem, msinfo, toTb, overwrite, selectdata,
field, spw, uvrange, antenna, scan, observation, intent, datacolumn,
imsize, cell, phasecenter, stokes, projection, startmodel, specmode,
reffreq, nchan, start, width, outframe, veltype, restfreq,
interpolation, gridder, facets, chanchunks, wprojplanes, vptable,
usepointing, mosweight, aterm, psterm, wbawp, conjbeams, cfcache,
computepastep, rotatepastep, pblimit, normtype, deconvolver, scales,
nterms, smallscalebias, restoration, restoringbeam, pbcor, outlierfile,
weighting, robust, npixels, uvtaper, niter, gain, threshold, nsigma,
cycleniter, cyclefactor, minpsffraction, maxpsffraction, interactive,
usemask, mask, pbmask, sidelobethreshold, noisethreshold,
lownoisethreshold, negativethreshold, smoothfactor, minbeamfrac,
cutthreshold, growiterations, dogrowprune, minpercentchange, verbose,
restart, savemodel, calcres, calcpsf, parallel, subregion, tmpdir)
timelapse = 0
t0 = time()
# parallelization
if ncpu > 1:
import multiprocessing as mprocs
casalog.post('Perform clean in parallel ...')
print('Perform clean in parallel ...')
pool = mprocs.Pool(ncpu)
res = pool.map(clnpart, iterable)
pool.close()
pool.join()
else:
casalog.post('Perform clean in single process ...')
print('Perform clean in single process ...')
for i in iterable:
res.append(clnpart(i))
t1 = time()
timelapse = t1 - t0
print('It took %f secs to complete' % timelapse)
# repackage this into a single dictionary
results = {
'Succeeded': [],
'BeginTime': [],
'EndTime': [],
'ImageName': []
}
for r in res:
results['Succeeded'].append(r[0])
results['BeginTime'].append(r[1])
results['EndTime'].append(r[2])
results['ImageName'].append(r[3])
if os.path.exists(tmpdir):
os.system('rm -rf ' + tmpdir)
return results
def read_horizons(t0=None,
dur=None,
vis=None,
observatory=None,
verbose=False):
'''
This function visits JPL Horizons to retrieve J2000 topocentric RA and DEC of the solar disk center
as a function of time.
Keyword arguments:
t0: Referece time in astropy.Time format
dur: duration of the returned coordinates. Default to 2 minutes
vis: CASA visibility dataset (in measurement set format). If provided, use entire duration from
the visibility data
observatory: observatory code (from JPL Horizons). If not provided, use information from visibility.
if no visibility found, use earth center (code=500)
verbose: True to provide extra information
Usage:
>>> from astropy.time import Time
>>> out = read_horizons(t0=Time('2017-09-10 16:00:00'), observatory='-81')
>>> out = read_horizons(vis = 'mydata.ms')
History:
BC (sometime in 2014): function was first wrote, followed by a number of edits by BC and SY
BC (2019-07-16): Added docstring documentation
'''
try:
# For Python 3.0 and later
from urllib.request import urlopen
except ImportError:
# Fall back to Python 2's urllib2
from urllib2 import urlopen
import ssl
if not t0 and not vis:
t0 = Time.now()
if not dur:
dur = 1. / 60. / 24. # default to 2 minutes
if t0:
try:
btime = Time(t0)
except:
print('input time ' + str(t0) + ' not recognized')
return -1
if vis:
if not os.path.exists(vis):
print('Input ms data ' + vis + ' does not exist! ')
return -1
try:
# ms.open(vis)
# summary = ms.summary()
# ms.close()
# btime = Time(summary['BeginTime'], format='mjd')
# etime = Time(summary['EndTime'], format='mjd')
## alternative way to avoid conflicts with importeovsa, if needed -- more time consuming
if observatory == 'geocentric':
observatory = '500'
else:
ms.open(vis)
metadata = ms.metadata()
if metadata.observatorynames()[0] == 'EVLA':
observatory = '-5'
elif metadata.observatorynames()[0] == 'EOVSA':
observatory = '-81'
elif metadata.observatorynames()[0] == 'ALMA':
observatory = '-7'
ms.close()
tb.open(vis)
btime_vis = Time(tb.getcell('TIME', 0) / 24. / 3600., format='mjd')
etime_vis = Time(tb.getcell('TIME',
tb.nrows() - 1) / 24. / 3600.,
format='mjd')
tb.close()
if verbose:
print("Beginning time of this scan " + btime_vis.iso)
print("End time of this scan " + etime_vis.iso)
# extend the start and end time for jpl horizons by 0.5 hr on each end
btime = Time(btime_vis.mjd - 0.5 / 24., format='mjd')
dur = etime_vis.mjd - btime_vis.mjd + 1.0 / 24.
except:
print('error in reading ms file: ' + vis +
' to obtain the ephemeris!')
return -1
# default the observatory to geocentric, if none provided
if not observatory:
observatory = '500'
etime = Time(btime.mjd + dur, format='mjd')
try:
cmdstr = "https://ssd.jpl.nasa.gov/horizons_batch.cgi?batch=1&TABLE_TYPE='OBSERVER'&QUANTITIES='1,17,20'&CSV_FORMAT='YES'&ANG_FORMAT='DEG'&CAL_FORMAT='BOTH'&SOLAR_ELONG='0,180'&CENTER='{}@399'&COMMAND='10'&START_TIME='".format(
observatory
) + btime.iso.replace(
' ', ','
) + "'&STOP_TIME='" + etime.iso[:-4].replace(
' ', ','
) + "'&STEP_SIZE='1m'&SKIP_DAYLT='NO'&EXTRA_PREC='YES'&APPARENT='REFRACTED'"
cmdstr = cmdstr.replace("'", "%27")
try:
context = ssl._create_unverified_context()
f = urlopen(cmdstr, context=context)
except:
f = urlopen(cmdstr)
lines = f.readlines()
f.close()
except:
# todo use geocentric coordinate for the new VLA data
import requests, collections
params = collections.OrderedDict()
params['batch'] = '1'
params['TABLE_TYPE'] = "'OBSERVER'"
params['QUANTITIES'] = "'1,17,20'"
params['CSV_FORMAT'] = "'YES'"
params['ANG_FORMAT'] = "'DEG'"
params['CAL_FORMAT'] = "'BOTH'"
params['SOLAR_ELONG'] = "'0,180'"
if observatory == '500':
params['CENTER'] = "'500'"
else:
params['CENTER'] = "'{}@399'".format(observatory)
params['COMMAND'] = "'10'"
params['START_TIME'] = "'{}'".format(btime.iso[:-4].replace(' ', ','))
params['STOP_TIME'] = "'{}'".format(etime.iso[:-4].replace(' ', ','))
params['STEP_SIZE'] = "'1m'"
params['SKIP_DAYLT'] = "'NO'"
params['EXTRA_PREC'] = "'YES'"
params['APPAENT'] = "'REFRACTED'"
results = requests.get("https://ssd.jpl.nasa.gov/horizons_batch.cgi",
params=params)
lines = [ll for ll in results.iter_lines()]
# add a check for python 3
if py3:
lines = [l.decode('utf-8', 'backslashreplace') for l in lines]
nline = len(lines)
istart = 0
for i in range(nline):
if lines[i][0:5] == '$$SOE': # start recording
istart = i + 1
if lines[i][0:5] == '$$EOE': # end recording
iend = i
newlines = lines[istart:iend]
nrec = len(newlines)
ephem_ = []
t = []
ra = []
dec = []
p0 = []
delta = []
for line in newlines:
items = line.split(',')
t.append(Time(float(items[1]), format='jd').mjd)
ra.append(np.radians(float(items[4])))
dec.append(np.radians(float(items[5])))
p0.append(float(items[6]))
delta.append(float(items[8]))
# convert list of dictionary to a dictionary of arrays
ephem = {'time': t, 'ra': ra, 'dec': dec, 'p0': p0, 'delta': delta}
return ephem
def mk_qlook_image(trange,
doimport=False,
docalib=False,
ncpu=10,
twidth=12,
stokes=None,
antenna='0~12',
lowcutoff_freq=3.7,
imagedir=None,
spws=['1~5', '6~10', '11~15', '16~25'],
toTb=True,
overwrite=True,
doslfcal=False,
verbose=False):
'''
trange: can be 1) a single Time() object: use the entire day
2) a range of Time(), e.g., Time(['2017-08-01 00:00','2017-08-01 23:00'])
3) a single or a list of UDBms file(s)
4) None -- use current date Time.now()
'''
antenna0 = antenna
if type(trange) == Time:
mslist = trange2ms(trange=trange, doimport=doimport)
vis = mslist['ms']
tsts = [l.to_datetime() for l in mslist['tstlist']]
if type(trange) == str:
try:
date = Time(trange)
mslist = trange2ms(trange=trange, doimport=doimport)
vis = mslist['ms']
tsts = [l.to_datetime() for l in mslist['tstlist']]
except:
vis = [trange]
tsts = []
for v in vis:
tb.open(v + '/OBSERVATION')
tsts.append(
Time(tb.getcell('TIME_RANGE')[0] / 24 / 3600,
format='mjd').datetime)
tb.close()
subdir = [tst.strftime("%Y/%m/%d/") for tst in tsts]
for idx, f in enumerate(vis):
if f[-1] == '/':
vis[idx] = f[:-1]
if not stokes:
stokes = 'XX'
if not imagedir:
imagedir = './'
imres = {
'Succeeded': [],
'BeginTime': [],
'EndTime': [],
'ImageName': [],
'Spw': [],
'Vis': [],
'Synoptic': {
'Succeeded': [],
'BeginTime': [],
'EndTime': [],
'ImageName': [],
'Spw': [],
'Vis': []
}
}
for n, msfile in enumerate(vis):
msfilebs = os.path.basename(msfile)
imdir = imagedir + subdir[n]
if not os.path.exists(imdir):
os.makedirs(imdir)
if doslfcal:
slfcalms = './' + msfilebs + '.xx'
split(msfile,
outputvis=slfcalms,
datacolumn='corrected',
correlation='XX')
cfreqs = getspwfreq(msfile)
for spw in spws:
antenna = antenna0
if spw == '':
continue
spwran = [s.zfill(2) for s in spw.split('~')]
freqran = [cfreqs[int(s)] for s in spw.split('~')]
cfreq = np.mean(freqran)
bmsz = max(150. / cfreq, 20.)
uvrange = '<10klambda'
if doslfcal:
slfcal_img = './' + msfilebs + '.slf.spw' + spw.replace(
'~', '-') + '.slfimg'
slfcal_tb = './' + msfilebs + '.slf.spw' + spw.replace(
'~', '-') + '.slftb'
try:
clean(vis=slfcalms,
antenna=antenna,
imagename=slfcal_img,
spw=spw,
mode='mfs',
timerange='',
imagermode='csclean',
psfmode='clark',
imsize=[512, 512],
cell=['5arcsec'],
niter=100,
gain=0.05,
stokes='I',
weighting='natural',
restoringbeam=[str(bmsz) + 'arcsec'],
pbcor=False,
interactive=False,
usescratch=True)
except:
print('error in cleaning spw: ' + spw)
break
gaincal(vis=slfcalms,
refant='0',
antenna=antenna,
caltable=slfcal_tb,
spw=spw,
uvrange='',
gaintable=[],
selectdata=True,
timerange='',
solint='600s',
gaintype='G',
calmode='p',
combine='',
minblperant=3,
minsnr=2,
append=False)
if not os.path.exists(slfcal_tb):
print('No solution found in spw: ' + spw)
break
else:
clearcal(slfcalms)
delmod(slfcalms)
applycal(vis=slfcalms,
gaintable=[slfcal_tb],
spw=spw,
selectdata=True,
antenna=antenna,
interp='nearest',
flagbackup=False,
applymode='calonly',
calwt=False)
msfile = slfcalms
imsize = 512
cell = ['5arcsec']
if len(spwran) == 2:
spwstr = spwran[0] + '~' + spwran[1]
else:
spwstr = spwran[0]
restoringbeam = ['{0:.1f}arcsec'.format(bmsz)]
imagesuffix = '.spw' + spwstr.replace('~', '-')
if cfreq > 10.:
antenna = antenna + ';!0&1;!0&2' # deselect the shortest baselines
# else:
# antenna = antenna + ';!0&1' # deselect the shortest baselines
res = ptclean3(vis=msfile,
imageprefix=imdir,
imagesuffix=imagesuffix,
twidth=twidth,
uvrange=uvrange,
spw=spw,
ncpu=ncpu,
niter=1000,
gain=0.05,
antenna=antenna,
imsize=imsize,
cell=cell,
stokes=stokes,
doreg=True,
usephacenter=False,
overwrite=overwrite,
toTb=toTb,
restoringbeam=restoringbeam,
specmode="mfs",
deconvolver="hogbom",
datacolumn='data',
pbcor=True)
if res:
imres['Succeeded'] += res['Succeeded']
imres['BeginTime'] += res['BeginTime']
imres['EndTime'] += res['EndTime']
imres['ImageName'] += res['ImageName']
imres['Spw'] += [spwstr] * len(res['ImageName'])
imres['Vis'] += [msfile] * len(res['ImageName'])
else:
continue
if len(vis) == 1:
# produce the band-by-band whole-day images
ms.open(msfile)
ms.selectinit()
timfreq = ms.getdata(['time', 'axis_info'], ifraxis=True)
tim = timfreq['time']
ms.close()
cfreqs = getspwfreq(msfile)
imdir = imagedir + subdir[0]
if not os.path.exists(imdir):
os.makedirs(imdir)
for spw in spws:
antenna = antenna0
if spw == '':
spw = '{:d}~{:d}'.format(
next(x[0] for x in enumerate(cfreqs)
if x[1] > lowcutoff_freq),
len(cfreqs) - 1)
spwran = [s.zfill(2) for s in spw.split('~')]
freqran = [cfreqs[int(s)] for s in spw.split('~')]
cfreq = np.mean(freqran)
bmsz = max(150. / cfreq, 20.)
uvrange = ''
imsize = 512
cell = ['5arcsec']
if len(spwran) == 2:
spwstr = spwran[0] + '~' + spwran[1]
else:
spwstr = spwran[0]
restoringbeam = ['{0:.1f}arcsec'.format(bmsz)]
imagesuffix = '.synoptic.spw' + spwstr.replace('~', '-')
antenna = antenna + ';!0&1' # deselect the shortest baselines
res = ptclean3(vis=msfile,
imageprefix=imdir,
imagesuffix=imagesuffix,
twidth=len(tim),
uvrange=uvrange,
spw=spw,
ncpu=1,
niter=0,
gain=0.05,
antenna=antenna,
imsize=imsize,
cell=cell,
stokes=stokes,
doreg=True,
usephacenter=False,
overwrite=overwrite,
toTb=toTb,
restoringbeam=restoringbeam,
specmode="mfs",
deconvolver="hogbom",
datacolumn='data',
pbcor=True)
if res:
imres['Synoptic']['Succeeded'] += res['Succeeded']
imres['Synoptic']['BeginTime'] += res['BeginTime']
imres['Synoptic']['EndTime'] += res['EndTime']
imres['Synoptic']['ImageName'] += res['ImageName']
imres['Synoptic']['Spw'] += [spwstr] * len(res['ImageName'])
imres['Synoptic']['Vis'] += [msfile] * len(res['ImageName'])
else:
continue
# save it for debugging purposes
np.savez('imres.npz', imres=imres)
return imres
def get_dspec(vis=None,
savespec=True,
specfile=None,
bl='',
uvrange='',
field='',
scan='',
datacolumn='data',
domedian=False,
timeran=None,
spw=None,
timebin='0s',
verbose=False):
from split_cli import split_cli as split
msfile = vis
if not spw:
spw = ''
if not timeran:
timeran = ''
if not bl:
bl = ''
if domedian:
if not uvrange:
uvrange = '0.2~0.8km'
else:
uvrange = ''
# Open the ms and plot dynamic spectrum
if verbose:
print 'Splitting selected data...'
vis_spl = './tmpms.splitted'
if os.path.exists(vis_spl):
os.system('rm -rf ' + vis_spl)
ms.open(msfile, nomodify=False)
ms.split(outputms=vis_spl,
whichcol=datacolumn,
time=timeran,
spw=spw,
baseline=bl,
field=field,
scan=scan,
uvrange=uvrange,
timebin=timebin)
ms.close()
# split(vis=msfile, outputvis=vis_spl, timerange=timeran, antenna=bl, field=field, scan=scan, spw=spw,
# uvrange=uvrange, timebin=timebin, datacolumn=datacolumn)
ms.open(vis_spl, nomodify=False)
if verbose:
print 'Regridding into a single spectral window...'
# print 'Reading data spw by spw'
ms.cvel(outframe='LSRK', mode='frequency', interp='nearest')
ms.selectinit(datadescid=0, reset=True)
data = ms.getdata(['amplitude', 'time', 'axis_info'], ifraxis=True)
ms.close()
os.system('rm -rf ' + vis_spl)
specamp = data['amplitude']
(npol, nfreq, nbl, ntim) = specamp.shape
if verbose:
print 'npol, nfreq, nbl, ntime:', data['amplitude'].shape
spec = np.swapaxes(specamp, 2, 1)
freq = data['axis_info']['freq_axis']['chan_freq'].reshape(nfreq)
tim = data['time']
if domedian:
if verbose:
print('doing median of all the baselines')
# mask zero values before median
spec_masked = np.ma.masked_where(spec < 1e-9, spec)
spec_med = np.ma.filled(np.ma.median(spec_masked, axis=1),
fill_value=0.)
nbl = 1
ospec = spec_med.reshape((npol, nbl, nfreq, ntim))
else:
ospec = spec
# Save the dynamic spectral data
if savespec:
if not specfile:
specfile = msfile + '.dspec.npz'
if os.path.exists(specfile):
os.system('rm -rf ' + specfile)
np.savez(specfile,
spec=ospec,
tim=tim,
freq=freq,
timeran=timeran,
spw=spw,
bl=bl,
uvrange=uvrange)
if verbose:
print 'Median dynamic spectrum saved as: ' + specfile
return {
'spec': ospec,
'tim': tim,
'freq': freq,
'timeran': timeran,
'spw': spw,
'bl': bl,
'uvrange': uvrange
}
def ptclean(vis, imageprefix, ncpu, twidth, doreg, overwrite, ephemfile,
msinfofile, outlierfile, field, spw, selectdata, timerange,
uvrange, antenna, scan, observation, intent, mode, resmooth,
gridmode, wprojplanes, facets, cfcache, rotpainc, painc, aterm,
psterm, mterm, wbawp, conjbeams, epjtable, interpolation, niter,
gain, threshold, psfmode, imagermode, ftmachine, mosweight,
scaletype, multiscale, negcomponent, smallscalebias, interactive,
mask, nchan, start, width, outframe, veltype, imsize, cell,
phasecenter, restfreq, stokes, weighting, robust, uvtaper,
outertaper, innertaper, modelimage, restoringbeam, pbcor, minpb,
usescratch, noise, npixels, npercycle, cyclefactor, cyclespeedup,
nterms, reffreq, chaniter, flatnoise, allowchunk):
if not (type(ncpu) is int):
casalog.post('ncpu should be an integer')
ncpu = 8
if doreg:
# check if ephemfile and msinfofile exist
try:
ephem = vla_prep.read_horizons(ephemfile=ephemfile)
except ValueError:
print("error in reading ephemeris file")
if not os.path.isfile(msinfofile):
print("msinfofile does not exist!")
else:
ephem = None
# get number of time pixels
ms.open(vis)
ms.selectinit()
timfreq = ms.getdata(['time', 'axis_info'], ifraxis=True)
tim = timfreq['time']
# dt = tim[1]-tim[0] #need to change to median of all time intervals
dt = np.median(np.diff(tim))
freq = timfreq['axis_info']['freq_axis']['chan_freq'].flatten()
ms.close()
if twidth < 1 or twidth > len(tim):
casalog.post(
'twidth not between 1 and # of time pixels in the dataset. Change to 1'
)
twidth = 1
# find out the start and end time index according to the parameter timerange
# if not defined (empty string), use start and end from the entire time of the ms
if not timerange:
btidx = 0
etidx = len(tim) - 1
else:
try:
(tstart, tend) = timerange.split('~')
bt_s = qa.convert(qa.quantity(tstart, 's'), 's')['value']
et_s = qa.convert(qa.quantity(tend, 's'), 's')['value']
# only time is given but not date, add the date (at 0 UT) from the first record
if bt_s < 86400. or et_s < 86400.:
bt_s += np.fix(
qa.convert(qa.quantity(tim[0], 's'),
'd')['value']) * 86400.
et_s += np.fix(
qa.convert(qa.quantity(tim[0], 's'),
'd')['value']) * 86400.
btidx = np.argmin(np.abs(tim - bt_s))
etidx = np.argmin(np.abs(tim - et_s))
# make the indice back to those bracket by the timerange
if tim[btidx] < bt_s:
btidx += 1
if tim[etidx] > et_s:
etidx -= 1
if etidx <= btidx:
print "ending time must be greater than starting time"
print "reinitiating to the entire time range"
btidx = 0
etidx = len(tim) - 1
except ValueError:
print "keyword 'timerange' has a wrong format"
btstr = qa.time(qa.quantity(tim[btidx], 's'), prec=9, form='fits')[0]
etstr = qa.time(qa.quantity(tim[etidx], 's'), prec=9, form='fits')[0]
iterable = range(btidx, etidx + 1, twidth)
print 'First time pixel: ' + btstr
print 'Last time pixel: ' + etstr
print str(len(iterable)) + ' images to clean...'
res = []
# partition
clnpart = partial(
clean_iter, tim, freq, vis, imageprefix, ncpu, twidth, doreg,
overwrite, ephemfile, ephem, msinfofile, outlierfile, field, spw,
selectdata, uvrange, antenna, scan, observation, intent, mode,
resmooth, gridmode, wprojplanes, facets, cfcache, rotpainc, painc,
aterm, psterm, mterm, wbawp, conjbeams, epjtable, interpolation, niter,
gain, threshold, psfmode, imagermode, ftmachine, mosweight, scaletype,
multiscale, negcomponent, smallscalebias, interactive, mask, nchan,
start, width, outframe, veltype, imsize, cell, phasecenter, restfreq,
stokes, weighting, robust, uvtaper, outertaper, innertaper, modelimage,
restoringbeam, pbcor, minpb, usescratch, noise, npixels, npercycle,
cyclefactor, cyclespeedup, nterms, reffreq, chaniter, flatnoise,
allowchunk)
timelapse = 0
t0 = time()
# parallelization
para = 1
if para:
casalog.post('Perform clean in parallel ...')
pool = mp.Pool(ncpu)
# res = pool.map_async(clnpart, iterable)
res = pool.map(clnpart, iterable)
pool.close()
pool.join()
else:
for i in iterable:
res.append(clnpart(i))
t1 = time()
timelapse = t1 - t0
print 'It took %f secs to complete' % timelapse
# repackage this into a single dictionary
results = {'succeeded': [], 'timestamps': [], 'imagenames': []}
for r in res:
results['succeeded'].append(r[0])
results['timestamps'].append(r[1])
results['imagenames'].append(r[2])
return results
def pltvs(vis='',
outputvis='',
plttimerange='',
pltspw1='',
pltspw2='',
pltspw3='',
timoffset=4,
windowlen=10,
windowtype='hamming',
pol='LL',
bl='19&22'):
ms.open(vis, nomodify=True)
""" This function can do two steps: (1) plot the uv amplitude vs. time
on three selected channel and compare the original and smoothed signals.
You can select appropriate window length and type, and the offset time
to match the peaks and valleys for the three channel curves. (2) Confirm
the use of specified window length, type and offset time and smooth the
data channel by channel. The background-subtracted measurement set is
then generated.
"""
timfreq = ms.getdata(['time', 'axis_info'], ifraxis=True)
tim = timfreq['time']
timerange = str(qa.time(qa.quantity(tim[0], 's'), prec=8)[0]) + '~' + str(
qa.time(qa.quantity(tim[-1], 's'), prec=8)[0])
# check plotting timerange
if plttimerange and (type(plttimerange) == str):
print 'plotting the specified timerange: ', plttimerange
else:
plttimerange = timerange
print 'plotting the entire timerange: ', plttimerange
if pltspw1 and (type(pltspw1) == str):
print 'Using the specified channel 1:', pltspw1
else:
pltspw1 = '0:400'
if pltspw2 and (type(pltspw2) == str):
print 'Using the specified channel 2:', pltspw2
else:
pltspw2 = '0:500'
if pltspw3 and (type(pltspw3) == str):
print 'Using the specified channel 3:', pltspw3
else:
pltspw3 = '0:600'
[spwid1, chan1] = pltspw1.split(':')
[spwid2, chan2] = pltspw2.split(':')
[spwid3, chan3] = pltspw3.split(':')
chanid = [chan1, chan2, chan3]
if not (spwid1 == spwid2 and spwid1 == spwid3):
print 'Please use the same spectral window'
exit()
if not (timoffset and (type(timoffset) == int)):
timoffset = int(4)
# initialize the loop-out status
status = 'n'
while True:
timoffset = int(
raw_input("Please specify the offset after smoothing:"))
windowlen = int(
raw_input("Please specify window width for smoothing:"))
windowtype = str(
raw_input(
"Please specify window type for smoothing: (e.g. 'flat', 'hanning', 'hamming', 'bartlett', 'blackman')"
))
pol = str(raw_input("Please specify polarization: (e.g. RR/LL)"))
bl = str(raw_input("Please specify baseline: (e.g. '19&22')"))
j = 331
for i in range(len(chanid)):
ms.selectinit(datadescid=int(spwid1))
if bl and (type(bl) == str):
ms.msselect({'baseline': bl})
if timerange and (type(timerange) == str):
ms.msselect({'time': timerange})
if chanid[i] and (type(chanid[i]) == str):
ms.selectchannel(1, int(chanid[i]), 1, 1)
specdata = ms.getdata(['data', 'time', 'axis_info'], ifraxis=True)
if pol == 'RR':
spec = specdata['data'][0, 0, 0, :]
if pol == 'LL':
spec = specdata['data'][1, 0, 0, :]
ms.selectinit(datadescid=int(spwid1))
if bl and (type(bl) == str):
ms.msselect({'baseline': bl})
if plttimerange and (type(plttimerange) == str):
ms.msselect({'time': plttimerange})
if chanid[i] and (type(chanid[i]) == str):
ms.selectchannel(1, int(chanid[i]), 1, 1)
specdata_plt = ms.getdata(['data', 'time', 'axis_info'],
ifraxis=True)
if pol == 'RR':
spec_plt = specdata_plt['data'][0, 0, 0, :]
if pol == 'LL':
spec_plt = specdata_plt['data'][1, 0, 0, :]
spec_plt_smooth = task_smooth.smooth(spec_plt, timoffset,
windowlen, windowtype)
spec_smooth = task_smooth.smooth(spec, timoffset, windowlen,
windowtype)
spec_plt_amp = np.absolute(spec_plt)
spec_plt_smooth_amp = np.absolute(spec_plt_smooth)
#spec_plt_amp=sqrt(spec_plt.real**2+spec_plt.imag**2)
#spec_plt_smooth_amp=sqrt(spec_plt_smooth.real**2+spec_plt_smooth.imag**2)
#print len(spec)
#print len(spec_smooth)
#print type(spec)
#print spec[0]
sp1 = fft(spec)
sp2 = fft(spec_smooth)
sp3 = sp1 - sp2
freq1 = fftfreq(len(sp1), d=0.001)
freq2 = fftfreq(len(sp2), d=0.001)
freq3 = fftfreq(len(sp3), d=0.001)
freq1_index = np.argsort(freq1)
freq2_index = np.argsort(freq2)
freq3_index = np.argsort(freq3)
#print min(freq1),max(freq1)
subplot(j)
plot(spec_plt_amp)
plot(spec_plt_smooth_amp)
title("Signal vs Time")
j = j + 1
subplot(j)
plot(freq1[freq1_index], log10(sp1[freq1_index]))
plot(freq2[freq2_index], log10(sp2[freq2_index]))
ylim([0, 6])
title("FFT signal vs Frequency")
j = j + 1
subplot(j)
#plot(subspec)
plot(freq3[freq3_index], log10(sp3[freq3_index]))
ylim([0, 6])
title("FFT smoothed signal vs Frequency")
j = j + 1
#print "number of original data points: ",len(spec)
#print "number of smoothed data points: ",len(spec_smooth)
status = str(
raw_input("Confirm to use current parameters? (y/n/abort) "))
if status == 'y':
flag1 = str(
raw_input("Smooth all the channels and time range? (y/n) "))
if flag1 == 'y':
smtimerange = ''
smspw = ''
splitsel = False
else:
print 'confirm using window width: ', windowlen
print 'confirm using window type: ', windowtype
smtimerange = str(
raw_input(
"Please specify the time range for smoothing (HH:MM:SS) :"
))
smspw = str(
raw_input(
"Please specify spectral window and channel (e.g. 0:0~1032) :"
))
splitsel = True
break
elif status == 'abort':
print 'Abort background subtraction.'
sys.exit()
if not outputvis:
outputvis = str(timoffset) + '_' + str(windowlen) + '_' + str(
windowtype) + '.ms'
print "Generating output: ", outputvis
ms.close()
result1 = subvs(vis, outputvis, smtimerange, smspw, timoffset, windowlen,
windowtype, '', '', '')
