def convert_hdf_to_blockvisibility(f):
""" Convert HDF root to blockvisibility
:param f:
:return:
"""
assert f.attrs[
'ARL_data_model'] == "BlockVisibility", "Not a BlockVisibility"
s = f.attrs['phasecentre_coords'].split()
ss = [float(s[0]), float(s[1])] * u.deg
phasecentre = SkyCoord(ra=ss[0],
dec=ss[1],
frame=f.attrs['phasecentre_frame'])
polarisation_frame = PolarisationFrame(f.attrs['polarisation_frame'])
frequency = f.attrs['frequency']
channel_bandwidth = f.attrs['channel_bandwidth']
data = numpy.array(f['data'])
source = f.attrs['source']
meta = ast.literal_eval(f.attrs['meta'])
vis = BlockVisibility(data=data,
polarisation_frame=polarisation_frame,
phasecentre=phasecentre,
frequency=frequency,
channel_bandwidth=channel_bandwidth,
source=source,
meta=meta)
vis.configuration = convert_configuration_from_hdf(f)
return vis
def integrate_visibility_by_channel(vis: BlockVisibility) -> BlockVisibility:
""" Integrate visibility across channels, returning new visibility
:param vis:
:return: BlockVisibility
"""
assert isinstance(vis, Visibility) or isinstance(vis, BlockVisibility), vis
vis_shape = list(vis.vis.shape)
ntimes, nants, _, nchan, npol = vis_shape
vis_shape[-2] = 1
newvis = BlockVisibility(
data=None,
frequency=numpy.ones([1]) * numpy.average(vis.frequency),
channel_bandwidth=numpy.ones([1]) * numpy.sum(vis.channel_bandwidth),
phasecentre=vis.phasecentre,
configuration=vis.configuration,
uvw=vis.uvw,
time=vis.time,
vis=numpy.zeros(vis_shape, dtype='complex'),
weight=numpy.ones(vis_shape, dtype='float'),
integration_time=vis.integration_time,
polarisation_frame=vis.polarisation_frame)
newvis.data['vis'][...,
0, :] = numpy.sum(vis.data['vis'] * vis.data['weight'],
axis=-2)
newvis.data['weight'][..., 0, :] = numpy.sum(vis.data['weight'], axis=-2)
mask = newvis.data['weight'] > 0.0
newvis.data['vis'][
mask] = newvis.data['vis'][mask] / newvis.data['weight'][mask]
return newvis
def visibility_gather_channel(vis_list: List[BlockVisibility],
vis: BlockVisibility = None):
""" Gather a visibility by channel
:param vis_list:
:param vis:
:return:
"""
cols = ['vis', 'weight']
if vis is None:
vis_shape = numpy.array(vis_list[0].vis.shape)
vis_shape[-2] = len(vis_list)
for v in vis_list:
assert len(v.frequency) == 1
assert len(v.channel_bandwidth) == 1
vis = BlockVisibility(
data=None,
frequency=numpy.array([v.frequency[0] for v in vis_list]),
channel_bandwidth=numpy.array(
[v.channel_bandwidth[0] for v in vis_list]),
phasecentre=vis_list[0].phasecentre,
configuration=vis_list[0].configuration,
uvw=vis_list[0].uvw,
time=vis_list[0].time,
vis=numpy.zeros(vis_shape, dtype=vis_list[0].vis.dtype),
weight=numpy.ones(vis_shape, dtype=vis_list[0].weight.dtype),
imaging_weight=numpy.ones(vis_shape,
dtype=vis_list[0].weight.dtype),
integration_time=vis_list[0].integration_time,
polarisation_frame=vis_list[0].polarisation_frame,
source=vis_list[0].source,
meta=vis_list[0].meta)
assert len(vis.frequency) == len(vis_list)
for chan, _ in enumerate(vis_list):
subvis = vis_list[chan]
assert abs(subvis.frequency[0] - vis.frequency[chan]) < 1e-15
for col in cols:
vis.data[col][..., chan, :] = subvis.data[col][..., 0, :]
vis.frequency[chan] = subvis.frequency[0]
nchan = vis.vis.shape[-2]
assert nchan == len(vis.frequency)
return vis
def divide_visibility(vis: BlockVisibility, modelvis: BlockVisibility):
""" Divide visibility by model forming visibility for equivalent point source
This is a useful intermediate product for calibration. Variation of the visibility in time and
frequency due to the model structure is removed and the data can be averaged to a limit determined
by the instrumental stability. The weight is adjusted to compensate for the division.
Zero divisions are avoided and the corresponding weight set to zero.
:param vis:
:param modelvis:
:return:
"""
assert isinstance(vis, Visibility) or isinstance(vis, BlockVisibility), vis
# Different for scalar and vector/matrix cases
isscalar = vis.polarisation_frame.npol == 1
if isscalar:
# Scalar case is straightforward
x = numpy.zeros_like(vis.vis)
xwt = numpy.abs(modelvis.vis)**2 * vis.weight
mask = xwt > 0.0
x[mask] = vis.vis[mask] / modelvis.vis[mask]
else:
nrows, nants, _, nchan, npol = vis.vis.shape
nrec = 2
assert nrec * nrec == npol
xshape = (nrows, nants, nants, nchan, nrec, nrec)
x = numpy.zeros(xshape, dtype='complex')
xwt = numpy.zeros(xshape)
for row in range(nrows):
for ant1 in range(nants):
for ant2 in range(ant1 + 1, nants):
for chan in range(nchan):
ovis = numpy.matrix(vis.vis[row, ant2, ant1,
chan].reshape([2, 2]))
mvis = numpy.matrix(modelvis.vis[row, ant2, ant1,
chan].reshape([2, 2]))
wt = numpy.matrix(vis.weight[row, ant2, ant1,
chan].reshape([2, 2]))
x[row, ant2, ant1,
chan] = numpy.matmul(numpy.linalg.inv(mvis), ovis)
xwt[row, ant2, ant1,
chan] = numpy.dot(mvis,
numpy.multiply(wt, mvis.H)).real
x = x.reshape((nrows, nants, nants, nchan, nrec * nrec))
xwt = xwt.reshape((nrows, nants, nants, nchan, nrec * nrec))
pointsource_vis = BlockVisibility(data=None,
frequency=vis.frequency,
channel_bandwidth=vis.channel_bandwidth,
phasecentre=vis.phasecentre,
configuration=vis.configuration,
uvw=vis.uvw,
time=vis.time,
integration_time=vis.integration_time,
vis=x,
weight=xwt)
return pointsource_vis
def helper_create_blockvisibility_object(c_vis, freqs, chan_b, config):
# This may be incorrect
# especially the data field...
tvis = BlockVisibility(data=c_vis,
frequency=freqs,
channel_bandwidth=chan_b,
configuration=config,
integration_time=c_vis['integration_time'],
weight=c_vis['weight'],
uvw=c_vis['uvw'],
time=c_vis['time'])
return tvis
def extract_channel(v, chan):
vis_shape = numpy.array(v.data['vis'].shape)
vis_shape[3] = 1
vis = BlockVisibility(data=None,
frequency=numpy.array([v.frequency[chan]]),
channel_bandwidth=numpy.array([v.channel_bandwidth[chan]]),
phasecentre=v.phasecentre,
configuration=v.configuration,
uvw=v.uvw,
time=v.time,
vis=v.vis[..., chan, :][..., numpy.newaxis, :],
weight=v.weight[..., chan, :][..., numpy.newaxis, :],
integration_time=v.integration_time,
polarisation_frame=v.polarisation_frame)
return vis
def concatenate_blockvisibility_frequency(bvis_list):
"""Concatenate a list of BlockVisibility's in frequency
:param bvis_list:
:return: BlockVisibility
"""
assert len(bvis_list) > 0
nvis = bvis_list[0].nvis
time = bvis_list[0].time
frequency = numpy.array(
numpy.array([bvis.frequency for bvis in bvis_list]).flat)
channel_bandwidth = numpy.array(
numpy.array([bvis.channel_bandwidth for bvis in bvis_list]).flat)
nchan = len(frequency)
ntimes, nants, _, _, npol = bvis_list[0].vis.shape
uvw = bvis_list[0].uvw
integration_time = bvis_list[0].integration_time
vis = numpy.zeros([nvis, nants, nants, nchan, npol], dtype='complex')
weight = numpy.ones([nvis, nants, nants, nchan, npol])
imaging_weight = numpy.ones([nvis, nants, nants, nchan, npol])
echan = 0
for ibv, bvis in enumerate(bvis_list):
schan = echan
echan = schan + len(bvis.frequency)
vis[..., schan:echan, :] = bvis.vis[...]
weight[..., schan:echan, :] = bvis.weight[...]
imaging_weight[..., schan:echan, :] = bvis.imaging_weight[...]
vis[..., schan:echan, :] = bvis.vis[...]
return BlockVisibility(vis=vis,
weight=weight,
imaging_weight=imaging_weight,
uvw=uvw,
time=time,
integration_time=integration_time,
frequency=frequency,
channel_bandwidth=channel_bandwidth,
polarisation_frame=bvis_list[0].polarisation_frame,
source=bvis_list[0].source,
configuration=bvis_list[0].configuration,
phasecentre=bvis_list[0].phasecentre,
meta=None)
def convert_blockvisibility_to_stokesI(vis):
"""Convert the polarisation frame data into Stokes I dropping other polarisations, return new Visibility
Args:
vis (obj): ARL visibility data.
Returns:
vis: New, converted visibility data.
"""
polarisation_frame = PolarisationFrame('stokesI')
poldef = vis.polarisation_frame
if poldef == PolarisationFrame('linear'):
vis_data = convert_linear_to_stokesI(vis.data['vis'])
vis_weight = (vis.weight[..., 0] + vis.weight[..., 3])[...,
numpy.newaxis]
vis_imaging_weight = (vis.imaging_weight[..., 0] +
vis.imaging_weight[..., 3])[..., numpy.newaxis]
elif poldef == PolarisationFrame('circular'):
vis_data = convert_circular_to_stokesI(vis.data['vis'])
vis_weight = (vis.weight[..., 0] + vis.weight[..., 3])[...,
numpy.newaxis]
vis_imaging_weight = (vis.imaging_weight[..., 0] +
vis.imaging_weight[..., 3])[..., numpy.newaxis]
else:
raise NameError("Polarisation frame %s unknown" % poldef)
return BlockVisibility(frequency=vis.frequency,
channel_bandwidth=vis.channel_bandwidth,
phasecentre=vis.phasecentre,
configuration=vis.configuration,
uvw=vis.uvw,
time=vis.time,
vis=vis_data,
weight=vis_weight,
imaging_weight=vis_imaging_weight,
integration_time=vis.integration_time,
polarisation_frame=polarisation_frame,
source=vis.source,
meta=vis.meta)
def create_blockvisibility(config: Configuration,
times: numpy.array,
frequency: numpy.array,
phasecentre: SkyCoord,
weight: float = 1.0,
polarisation_frame: PolarisationFrame = None,
integration_time=1.0,
channel_bandwidth=1e6,
zerow=False,
**kwargs) -> BlockVisibility:
""" Create a BlockVisibility from Configuration, hour angles, and direction of source
Note that we keep track of the integration time for BDA purposes
:param config: Configuration of antennas
:param times: hour angles in radians
:param frequency: frequencies (Hz] [nchan]
:param weight: weight of a single sample
:param phasecentre: phasecentre of observation
:param channel_bandwidth: channel bandwidths: (Hz] [nchan]
:param integration_time: Integration time ('auto' or value in s)
:param polarisation_frame:
:return: BlockVisibility
"""
assert phasecentre is not None, "Must specify phase centre"
if polarisation_frame is None:
polarisation_frame = correlate_polarisation(config.receptor_frame)
nch = len(frequency)
ants_xyz = config.data['xyz']
nants = len(config.data['names'])
ntimes = len(times)
npol = polarisation_frame.npol
visshape = [ntimes, nants, nants, nch, npol]
rvis = numpy.zeros(visshape, dtype='complex')
rweight = weight * numpy.ones(visshape)
rtimes = numpy.zeros([ntimes])
ruvw = numpy.zeros([ntimes, nants, nants, 3])
# Do each hour angle in turn
for iha, ha in enumerate(times):
# Calculate the positions of the antennas as seen for this hour angle
# and declination
ant_pos = xyz_to_uvw(ants_xyz, ha, phasecentre.dec.rad)
rtimes[iha] = ha * 43200.0 / numpy.pi
# Loop over all pairs of antennas. Note that a2>a1
for a1 in range(nants):
for a2 in range(a1 + 1, nants):
ruvw[iha, a2, a1, :] = (ant_pos[a2, :] - ant_pos[a1, :])
ruvw[iha, a1, a2, :] = (ant_pos[a1, :] - ant_pos[a2, :])
rintegration_time = numpy.full_like(rtimes, integration_time)
rchannel_bandwidth = numpy.full_like(frequency, channel_bandwidth)
if zerow:
ruvw[..., 2] = 0.0
vis = BlockVisibility(uvw=ruvw,
time=rtimes,
frequency=frequency,
vis=rvis,
weight=rweight,
integration_time=rintegration_time,
channel_bandwidth=rchannel_bandwidth,
polarisation_frame=polarisation_frame)
vis.phasecentre = phasecentre
vis.configuration = config
log.info("create_blockvisibility: %s" % (vis_summary(vis)))
assert isinstance(
vis, BlockVisibility), "vis is not a BlockVisibility: %r" % vis
return vis
def create_blockvisibility_from_ms(msname, channum=None, ack=False):
""" Minimal MS to BlockVisibility converter
The MS format is much more general than the ARL BlockVisibility so we cut many corners. This requires casacore to be
installed. If not an exception ModuleNotFoundError is raised.
Creates a list of BlockVisibility's, split by field and spectral window
:param msname: File name of MS
:param channum: range of channels e.g. range(17,32), default is None meaning all
:return:
"""
try:
from casacore.tables import table # pylint: disable=import-error
except ModuleNotFoundError:
raise ModuleNotFoundError("casacore is not installed")
tab = table(msname, ack=ack)
log.debug("create_blockvisibility_from_ms: %s" % str(tab.info()))
fields = numpy.unique(tab.getcol('FIELD_ID'))
dds = numpy.unique(tab.getcol('DATA_DESC_ID'))
log.debug(
"create_blockvisibility_from_ms: Found unique fields %s, unique data descriptions %s"
% (str(fields), str(dds)))
vis_list = list()
for dd in dds:
dtab = table(msname, ack=ack).query('DATA_DESC_ID==%d' % dd, style='')
for field in fields:
ms = dtab.query('FIELD_ID==%d' % field, style='')
assert ms.nrows(
) > 0, "Empty selection for FIELD_ID=%d and DATA_DESC_ID=%d" % (
field, dd)
log.debug("create_blockvisibility_from_ms: Found %d rows" %
(ms.nrows()))
time = ms.getcol('TIME')
channels = ms.getcol('DATA').shape[-2]
log.debug("create_visibility_from_ms: Found %d channels" %
(channels))
if channum is None:
channum = range(channels)
try:
ms_vis = ms.getcol('DATA')[:, channum, :]
ms_weight = ms.getcol('WEIGHT')[:, :]
except IndexError:
raise IndexError("channel number exceeds max. within ms")
uvw = -1 * ms.getcol('UVW')
antenna1 = ms.getcol('ANTENNA1')
antenna2 = ms.getcol('ANTENNA2')
integration_time = ms.getcol('INTERVAL')
# Now get info from the subtables
spwtab = table('%s/SPECTRAL_WINDOW' % msname, ack=False)
cfrequency = spwtab.getcol('CHAN_FREQ')[dd][channum]
cchannel_bandwidth = spwtab.getcol('CHAN_WIDTH')[dd][channum]
nchan = cfrequency.shape[0]
# Get polarisation info
poltab = table('%s/POLARIZATION' % msname, ack=False)
corr_type = poltab.getcol('CORR_TYPE')
# These correspond to the CASA Stokes enumerations
if numpy.array_equal(corr_type[0], [1, 2, 3, 4]):
polarisation_frame = PolarisationFrame('stokesIQUV')
elif numpy.array_equal(corr_type[0], [5, 6, 7, 8]):
polarisation_frame = PolarisationFrame('circular')
elif numpy.array_equal(corr_type[0], [9, 10, 11, 12]):
polarisation_frame = PolarisationFrame('linear')
else:
raise KeyError("Polarisation not understood: %s" %
str(corr_type))
npol = 4
# Get configuration
anttab = table('%s/ANTENNA' % msname, ack=False)
nants = anttab.nrows()
mount = anttab.getcol('MOUNT')
names = anttab.getcol('NAME')
diameter = anttab.getcol('DISH_DIAMETER')
xyz = anttab.getcol('POSITION')
configuration = Configuration(
name='',
data=None,
location=None,
names=names,
xyz=xyz,
mount=mount,
frame=None,
receptor_frame=ReceptorFrame("linear"),
diameter=diameter)
# Get phasecentres
fieldtab = table('%s/FIELD' % msname, ack=False)
pc = fieldtab.getcol('PHASE_DIR')[field, 0, :]
phasecentre = SkyCoord(ra=[pc[0]] * u.rad,
dec=pc[1] * u.rad,
frame='icrs',
equinox='J2000')
bv_times = numpy.unique(time)
ntimes = len(bv_times)
bv_vis = numpy.zeros([ntimes, nants, nants, nchan,
npol]).astype('complex')
bv_weight = numpy.zeros([ntimes, nants, nants, nchan, npol])
bv_uvw = numpy.zeros([ntimes, nants, nants, 3])
time_last = time[0]
time_index = 0
for row, _ in enumerate(ms_vis):
# MS has shape [row, npol, nchan]
# BV has shape [ntimes, nants, nants, nchan, npol]
if time[row] != time_last:
assert time[
row] > time_last, "MS is not time-sorted - cannot convert"
time_index += 1
time_last = time[row]
bv_vis[time_index, antenna2[row], antenna1[row],
...] = ms_vis[row, ...]
bv_weight[time_index, antenna2[row], antenna1[row], :,
...] = ms_weight[row, numpy.newaxis, ...]
bv_uvw[time_index, antenna2[row],
antenna1[row], :] = uvw[row, :]
vis_list.append(
BlockVisibility(uvw=bv_uvw,
time=bv_times,
frequency=cfrequency,
channel_bandwidth=cchannel_bandwidth,
vis=bv_vis,
weight=bv_weight,
configuration=configuration,
phasecentre=phasecentre,
polarisation_frame=polarisation_frame))
tab.close()
return vis_list
def create_blockvisibility_from_uvfits(fitsname, channum=None, ack=False, antnum=None):
""" Minimal UVFIT to BlockVisibility converter
The UVFITS format is much more general than the ARL BlockVisibility so we cut many corners.
Creates a list of BlockVisibility's, split by field and spectral window
:param fitsname: File name of UVFITS
:param channum: range of channels e.g. range(17,32), default is None meaning all
:param antnum: the number of antenna
:return:
"""
def ParamDict(hdul):
"Return the dictionary of the random parameters"
"""
The keys of the dictionary are the parameter names uppercased for
consistency. The values are the column numbers.
If multiple parameters have the same name (e.g., DATE) their
columns are entered as a list.
"""
pre=re.compile(r"PTYPE(?P<i>\d+)")
res={}
for k,v in hdul.header.items():
m=pre.match(k)
if m :
vu=v.upper()
if vu in res:
res[ vu ] = [ res[vu], int(m.group("i")) ]
else:
res[ vu ] = int(m.group("i"))
return res
# Open the file
with fits.open(fitsname) as hdul:
# Read Spectral Window
nspw = hdul[0].header['NAXIS5']
# Read Channel and Frequency Interval
freq_ref = hdul[0].header['CRVAL4']
mid_chan_freq = hdul[0].header['CRPIX4']
delt_freq = hdul[0].header['CDELT4']
# Real the number of channels in one spectral window
channels = hdul[0].header['NAXIS4']
freq = numpy.zeros([nspw, channels])
# Read Frequency or IF
freqhdulname="AIPS FQ"
sdhu = hdul.index_of(freqhdulname)
if_freq = hdul[sdhu].data['IF FREQ'].ravel()
for i in range(nspw):
temp = numpy.array([if_freq[i] + freq_ref+delt_freq* ff for ff in range(channels)])
freq[i,:] = temp[:]
freq_delt = numpy.ones(channels) * delt_freq
if channum is None:
channum = range(channels)
primary = hdul[0].data
# Read time
bvtimes = Time(hdul[0].data['DATE'], hdul[0].data['_DATE'], format='jd')
bv_times = numpy.unique(bvtimes.jd)
ntimes = len(bv_times)
# # Get Antenna
# blin = hdul[0].data['BASELINE']
antennahdulname="AIPS AN"
adhu = hdul.index_of(antennahdulname)
try:
antenna_name = hdul[adhu].data['ANNAME']
antenna_name = antenna_name.encode('ascii','ignore')
except:
antenna_name = None
antenna_xyz = hdul[adhu].data['STABXYZ']
antenna_mount = hdul[adhu].data['MNTSTA']
try:
antenna_diameter = hdul[adhu].data['DIAMETER']
except:
antenna_diameter = None
# To reading some UVFITS with wrong numbers of antenna
if antnum is not None:
if antenna_name is not None:
antenna_name = antenna_name[:antnum]
antenna_xyz = antenna_xyz[:antnum]
antenna_mount = antenna_mount[:antnum]
if antenna_diameter is not None:
antenna_diameter = antenna_diameter[:antnum]
nants = len(antenna_xyz)
# res= {}
# for i,row in enumerate(fin[ahdul].data):
# res[row.field("ANNAME") ] = i +1
# Get polarisation info
npol = hdul[0].header['NAXIS3']
corr_type = numpy.arange(hdul[0].header['NAXIS3']) - (hdul[0].header['CRPIX3'] - 1)
corr_type *= hdul[0].header['CDELT3']
corr_type += hdul[0].header['CRVAL3']
# xx yy xy yx
# These correspond to the CASA Stokes enumerations
if numpy.array_equal(corr_type, [1, 2, 3, 4]):
polarisation_frame = PolarisationFrame('stokesIQUV')
elif numpy.array_equal(corr_type, [-1, -2, -3, -4]):
polarisation_frame = PolarisationFrame('circular')
elif numpy.array_equal(corr_type, [-5, -6, -7, -8]):
polarisation_frame = PolarisationFrame('linear')
else:
raise KeyError("Polarisation not understood: %s" % str(corr_type))
configuration = Configuration(name='', data=None, location=None,
names=antenna_name, xyz=antenna_xyz, mount=antenna_mount, frame=None,
receptor_frame=polarisation_frame,
diameter=antenna_diameter)
# Get RA and DEC
phase_center_ra_degrees = numpy.float(hdul[0].header['CRVAL6'])
phase_center_dec_degrees = numpy.float(hdul[0].header['CRVAL7'])
# Get phasecentres
phasecentre = SkyCoord(ra=phase_center_ra_degrees * u.deg, dec=phase_center_dec_degrees * u.deg, frame='icrs', equinox='J2000')
# Get UVW
d=ParamDict(hdul[0])
if "UU" in d:
uu = hdul[0].data['UU']
vv = hdul[0].data['VV']
ww = hdul[0].data['WW']
else:
uu = hdul[0].data['UU---SIN']
vv = hdul[0].data['VV---SIN']
ww = hdul[0].data['WW---SIN']
_vis = hdul[0].data['DATA']
#_vis.shape = (nchan, ntimes, (nants*(nants-1)//2 ), npol, -1)
#self.vis = -(_vis[...,0] * 1.j + _vis[...,1])
row = 0
nchan = len(channum)
vis_list = list()
for spw_index in range(nspw):
bv_vis = numpy.zeros([ntimes, nants, nants, nchan, npol]).astype('complex')
bv_weight = numpy.zeros([ntimes, nants, nants, nchan, npol])
bv_uvw = numpy.zeros([ntimes, nants, nants, 3])
for time_index , time in enumerate(bv_times):
#restfreq = freq[channel_index]
for antenna1 in range(nants-1):
for antenna2 in range(antenna1 + 1, nants):
for channel_no, channel_index in enumerate(channum):
for pol_index in range(npol):
bv_vis[time_index, antenna2, antenna1, channel_no,pol_index] = complex(_vis[row,:,:,spw_index,channel_index, pol_index ,0],_vis[row,:,:,spw_index,channel_index,pol_index ,1])
bv_weight[time_index, antenna2, antenna1, channel_no, pol_index] = _vis[row,:,:,spw_index,channel_index,pol_index ,2]
bv_uvw[time_index, antenna2, antenna1, 0] = uu[row]* constants.c.value
bv_uvw[time_index, antenna2, antenna1, 1] = vv[row]* constants.c.value
bv_uvw[time_index, antenna2, antenna1, 2] = ww[row]* constants.c.value
row += 1
vis_list.append(BlockVisibility(uvw=bv_uvw,
time=bv_times,
frequency=freq[spw_index][channum],
channel_bandwidth=freq_delt[channum],
vis=bv_vis,
weight=bv_weight,
imaging_weight= bv_weight,
configuration=configuration,
phasecentre=phasecentre,
polarisation_frame=polarisation_frame))
return vis_list
def apply_gaintable(vis: BlockVisibility,
gt: GainTable,
inverse=False,
vis_slices=None,
**kwargs) -> BlockVisibility:
"""Apply a gain table to a block visibility
The corrected visibility is::
V_corrected = {g_i * g_j^*}^-1 V_obs
If the visibility data are polarised e.g. polarisation_frame("linear") then the inverse operator
represents an actual inverse of the gains.
:param vis: Visibility to have gains applied
:param gt: Gaintable to be applied
:param inverse: Apply the inverse (default=False)
:return: input vis with gains applied
"""
assert isinstance(
vis, BlockVisibility), "vis is not a BlockVisibility: %r" % vis
assert isinstance(gt, GainTable), "gt is not a GainTable: %r" % gt
assert_vis_gt_compatible(vis, gt)
if inverse:
log.debug('apply_gaintable: Apply inverse gaintable')
else:
log.debug('apply_gaintable: Apply gaintable')
is_scalar = gt.gain.shape[-2:] == (1, 1)
if is_scalar:
log.debug('apply_gaintable: scalar gains')
for chunk, rows in enumerate(vis_timeslice_iter(vis,
vis_slices=vis_slices)):
if numpy.sum(rows) > 0:
vistime = numpy.average(vis.time[rows])
gaintable_rows = abs(gt.time - vistime) < gt.interval / 2.0
# Lookup the gain for this set of visibilities
gain = gt.data['gain'][gaintable_rows]
gainwt = gt.data['weight'][gaintable_rows]
# The shape of the mueller matrix is
ntimes, nant, nchan, nrec, _ = gain.shape
original = vis.vis[rows]
originalwt = vis.weight[rows]
applied = copy.deepcopy(original)
appliedwt = copy.deepcopy(originalwt)
for time in range(ntimes):
antantwt = numpy.outer(gainwt, gainwt)
if is_scalar:
if inverse:
lgain = numpy.ones_like(gain)
lgain[numpy.abs(gain) >
0.0] = 1.0 / gain[numpy.abs(gain) > 0.0]
else:
lgain = gain
clgain = numpy.conjugate(lgain)
for chan in range(nchan):
smueller = numpy.ma.outer(
lgain[time, :, chan, 0],
clgain[time, :, chan, 0]).reshape([nant, nant])
applied[time, :, :, chan,
0] = original[time, :, :, chan, 0] * smueller
antantwt = numpy.outer(gainwt[time, :, chan, 0, 0],
gainwt[time, :, chan, 0, 0])
applied[time, :, :, chan, 0][antantwt == 0.0] = 0.0
appliedwt[time, :, :, chan, 0][antantwt == 0.0] = 0.0
else:
for a1 in range(vis.nants - 1):
for a2 in range(a1 + 1, vis.nants):
for chan in range(nchan):
mueller = numpy.kron(
gain[time, a1, chan, :, :],
numpy.conjugate(gain[time, a2,
chan, :, :]))
if inverse:
# If the Mueller is singular, ignore it
try:
mueller = numpy.linalg.inv(mueller)
applied[time, a2, a1, chan, :] = \
numpy.matmul(mueller, original[time, a2, a1, chan, :])
except numpy.linalg.linalg.LinAlgError:
applied[time, a2, a1, chan, :] = \
original[time, a2, a1, chan, :]
else:
applied[time, a2, a1, chan, :] = \
numpy.matmul(mueller, original[time, a2, a1, chan, :])
if (gainwt[time, a1, chan, 0, 0] <= 0.0) or (
gainwt[time, a1, chan, 0, 0] <= 0.0):
applied[time, a2, a1, chan, 0] = 0.0
appliedwt[time, a2, a1, chan, 0] = 0.0
vis.data['vis'][rows] = applied
return vis
def create_blockvisibility_from_ms(msname, channum=None, start_chan=None, end_chan=None, ack=False,
datacolumn='DATA', selected_sources=None, selected_dds=None):
""" Minimal MS to BlockVisibility converter
The MS format is much more general than the ARL BlockVisibility so we cut many corners. This requires casacore to be
installed. If not an exception ModuleNotFoundError is raised.
Creates a list of BlockVisibility's, split by field and spectral window
Reading of a subset of channels is possible using either start_chan and end_chan or channnum. Using start_chan
and end_chan is preferred since it only reads the channels required. Channum is more flexible and can be used to
read a random list of channels.
:param msname: File name of MS
:param channum: range of channels e.g. range(17,32), default is None meaning all
:param start_chan: Starting channel to read
:param end_chan: End channel to read
:return:
"""
try:
from casacore.tables import table # pylint: disable=import-error
except ModuleNotFoundError:
raise ModuleNotFoundError("casacore is not installed")
try:
from processing_components.visibility import msv2
except ModuleNotFoundError:
raise ModuleNotFoundError("cannot import msv2")
tab = table(msname, ack=ack)
log.debug("create_blockvisibility_from_ms: %s" % str(tab.info()))
if selected_sources is None:
fields = numpy.unique(tab.getcol('FIELD_ID'))
else:
fieldtab = table('%s/FIELD' % msname, ack=False)
sources = fieldtab.getcol('NAME')
fields = list()
for field, source in enumerate(sources):
if source in selected_sources: fields.append(field)
assert len(fields) > 0, "No sources selected"
if selected_dds is None:
dds = numpy.unique(tab.getcol('DATA_DESC_ID'))
else:
dds = selected_dds
log.debug("create_blockvisibility_from_ms: Reading unique fields %s, unique data descriptions %s" % (
str(fields), str(dds)))
vis_list = list()
for field in fields:
ftab = table(msname, ack=ack).query('FIELD_ID==%d' % field, style='')
for dd in dds:
meta = {'MSV2':{'FIELD_ID': field, 'DATA_DESC_ID':dd}}
ms = ftab.query('DATA_DESC_ID==%d' % dd, style='')
assert ms.nrows() > 0, "Empty selection for FIELD_ID=%d and DATA_DESC_ID=%d" % (field, dd)
log.debug("create_blockvisibility_from_ms: Found %d rows" % (ms.nrows()))
# The TIME column has descriptor:
# {'valueType': 'double', 'dataManagerType': 'IncrementalStMan', 'dataManagerGroup': 'TIME',
# 'option': 0, 'maxlen': 0, 'comment': 'Modified Julian Day',
# 'keywords': {'QuantumUnits': ['s'], 'MEASINFO': {'type': 'epoch', 'Ref': 'UTC'}}}
otime = ms.getcol('TIME')
datacol = ms.getcol(datacolumn, nrow=1)
datacol_shape = list(datacol.shape)
channels = datacol.shape[-2]
log.debug("create_blockvisibility_from_ms: Found %d channels" % (channels))
if channum is None:
if start_chan is not None and end_chan is not None:
try:
log.debug("create_blockvisibility_from_ms: Reading channels from %d to %d" %
(start_chan, end_chan))
print("create_blockvisibility_from_ms: Reading channels from %d to %d (inclusive)" %
(start_chan, end_chan))
blc = [start_chan, 0]
trc = [end_chan, datacol_shape[-1] - 1]
channum = range(start_chan, end_chan+1)
ms_vis = ms.getcolslice(datacolumn, blc=blc, trc=trc)
ms_weight = ms.getcol('WEIGHT')
except IndexError:
raise IndexError("channel number exceeds max. within ms")
else:
log.debug("create_blockvisibility_from_ms: Reading all %d channels" % (channels))
try:
channum = range(channels)
ms_vis = ms.getcol(datacolumn)[:, channum, :]
ms_weight = ms.getcol('WEIGHT')
channum = range(channels)
except IndexError:
raise IndexError("channel number exceeds max. within ms")
else:
log.debug("create_blockvisibility_from_ms: Reading channels %s " % (channum))
channum = range(channels)
try:
ms_vis = ms.getcol(datacolumn)[:, channum, :]
ms_weight = ms.getcol('WEIGHT')[:, :]
except IndexError:
raise IndexError("channel number exceeds max. within ms")
uvw = -1 * ms.getcol('UVW')
antenna1 = ms.getcol('ANTENNA1')
antenna2 = ms.getcol('ANTENNA2')
integration_time = ms.getcol('INTERVAL')
# time = Time((time-integration_time/2.0)/86400+ 2400000.5,format='jd',scale='utc').utc.value
time = (otime - integration_time / 2.0)
start_time = numpy.min(time)/86400.0
end_time = numpy.max(time)/86400.0
log.debug("create_blockvisibility_from_ms: Observation from %s to %s" %
(Time(start_time, format='mjd').iso, Time(end_time, format='mjd').iso))
# Now get info from the subtables
spwtab = table('%s/SPECTRAL_WINDOW' % msname, ack=False)
cfrequency = spwtab.getcol('CHAN_FREQ')[dd][channum]
cchannel_bandwidth = spwtab.getcol('CHAN_WIDTH')[dd][channum]
nchan = cfrequency.shape[0]
# Get polarisation info
npol = 4
poltab = table('%s/POLARIZATION' % msname, ack=False)
corr_type = poltab.getcol('CORR_TYPE')
# These correspond to the CASA Stokes enumerations
if numpy.array_equal(corr_type[0], [1, 2, 3, 4]):
polarisation_frame = PolarisationFrame('stokesIQUV')
elif numpy.array_equal(corr_type[0], [5, 6, 7, 8]):
polarisation_frame = PolarisationFrame('circular')
elif numpy.array_equal(corr_type[0], [9, 10, 11, 12]):
polarisation_frame = PolarisationFrame('linear')
elif numpy.array_equal(corr_type[0], [9]):
npol = 1
polarisation_frame = PolarisationFrame('stokesI')
else:
raise KeyError("Polarisation not understood: %s" % str(corr_type))
# Get configuration
anttab = table('%s/ANTENNA' % msname, ack=False)
nants = anttab.nrows()
mount = anttab.getcol('MOUNT')
names = anttab.getcol('NAME')
diameter = anttab.getcol('DISH_DIAMETER')
xyz = anttab.getcol('POSITION')
configuration = Configuration(name='', data=None, location=None,
names=names, xyz=xyz, mount=mount, frame=None,
receptor_frame=ReceptorFrame("linear"),
diameter=diameter)
# Get phasecentres
fieldtab = table('%s/FIELD' % msname, ack=False)
pc = fieldtab.getcol('PHASE_DIR')[field, 0, :]
source = fieldtab.getcol('NAME')[field]
phasecentre = SkyCoord(ra=pc[0] * u.rad, dec=pc[1] * u.rad, frame='icrs', equinox='J2000')
time_index_row = numpy.zeros_like(time, dtype='int')
time_last = time[0]
time_index = 0
for row, _ in enumerate(time):
if time[row] > time_last + integration_time[row]:
assert time[row] > time_last, "MS is not time-sorted - cannot convert"
time_index += 1
time_last = time[row]
time_index_row[row] = time_index
ntimes = time_index + 1
bv_times = numpy.zeros([ntimes])
bv_vis = numpy.zeros([ntimes, nants, nants, nchan, npol]).astype('complex')
bv_weight = numpy.zeros([ntimes, nants, nants, nchan, npol])
bv_imaging_weight = numpy.zeros([ntimes, nants, nants, nchan, npol])
bv_uvw = numpy.zeros([ntimes, nants, nants, 3])
bv_integration_time = numpy.zeros([ntimes])
for row, _ in enumerate(time):
time_index = time_index_row[row]
bv_times[time_index] = time[row]
bv_vis[time_index, antenna2[row], antenna1[row], ...] = ms_vis[row, ...]
bv_weight[time_index, antenna2[row], antenna1[row], :, ...] = ms_weight[row, numpy.newaxis, ...]
bv_imaging_weight[time_index, antenna2[row], antenna1[row], :, ...] = ms_weight[row, numpy.newaxis, ...]
bv_uvw[time_index, antenna2[row], antenna1[row], :] = uvw[row, :]
bv_integration_time[time_index] = integration_time[row]
vis_list.append(BlockVisibility(uvw=bv_uvw,
time=bv_times,
frequency=cfrequency,
channel_bandwidth=cchannel_bandwidth,
vis=bv_vis,
weight=bv_weight,
integration_time = bv_integration_time,
imaging_weight=bv_imaging_weight,
configuration=configuration,
phasecentre=phasecentre,
polarisation_frame=polarisation_frame,
source=source, meta=meta))
tab.close()
return vis_list
def create_blockvisibility(config: Configuration,
times: numpy.array,
frequency: numpy.array,
phasecentre: SkyCoord,
weight: float = 1.0,
polarisation_frame: PolarisationFrame = None,
integration_time=1.0,
channel_bandwidth=1e6,
zerow=False,
elevation_limit=None,
source='unknown',
meta=None,
**kwargs) -> BlockVisibility:
""" Create a BlockVisibility from Configuration, hour angles, and direction of source
Note that we keep track of the integration time for BDA purposes
:param config: Configuration of antennas
:param times: hour angles in radians
:param frequency: frequencies (Hz] [nchan]
:param weight: weight of a single sample
:param phasecentre: phasecentre of observation
:param channel_bandwidth: channel bandwidths: (Hz] [nchan]
:param integration_time: Integration time ('auto' or value in s)
:param polarisation_frame:
:return: BlockVisibility
"""
assert phasecentre is not None, "Must specify phase centre"
if polarisation_frame is None:
polarisation_frame = correlate_polarisation(config.receptor_frame)
latitude = config.location.geodetic[1].to('rad').value
nch = len(frequency)
ants_xyz = config.data['xyz']
nants = len(config.data['names'])
ntimes = 0
n_flagged = 0
for iha, ha in enumerate(times):
# Calculate the positions of the antennas as seen for this hour angle
# and declination
_, elevation = hadec_to_azel(ha, phasecentre.dec.rad, latitude)
if elevation_limit is None or (elevation > elevation_limit):
ntimes +=1
else:
n_flagged += 1
assert ntimes > 0, "No unflagged points"
if elevation_limit is not None:
log.info('create_visibility: flagged %d/%d times below elevation limit %f (rad)' %
(n_flagged, ntimes, elevation_limit))
else:
log.info('create_visibility: created %d times' % (ntimes))
npol = polarisation_frame.npol
visshape = [ntimes, nants, nants, nch, npol]
rvis = numpy.zeros(visshape, dtype='complex')
rweight = weight * numpy.ones(visshape)
rimaging_weight = numpy.ones(visshape)
rtimes = numpy.zeros([ntimes])
ruvw = numpy.zeros([ntimes, nants, nants, 3])
# Do each hour angle in turn
itime = 0
for iha, ha in enumerate(times):
# Calculate the positions of the antennas as seen for this hour angle
# and declination
ant_pos = xyz_to_uvw(ants_xyz, ha, phasecentre.dec.rad)
_, elevation = hadec_to_azel(ha, phasecentre.dec.rad, latitude)
if elevation_limit is None or (elevation > elevation_limit):
rtimes[itime] = ha * 43200.0 / numpy.pi
rweight[itime, ...] = 1.0
# Loop over all pairs of antennas. Note that a2>a1
for a1 in range(nants):
for a2 in range(a1 + 1, nants):
ruvw[itime, a2, a1, :] = (ant_pos[a2, :] - ant_pos[a1, :])
ruvw[itime, a1, a2, :] = (ant_pos[a1, :] - ant_pos[a2, :])
itime += 1
rintegration_time = numpy.full_like(rtimes, integration_time)
rchannel_bandwidth = channel_bandwidth
if zerow:
ruvw[..., 2] = 0.0
vis = BlockVisibility(uvw=ruvw, time=rtimes, frequency=frequency, vis=rvis, weight=rweight,
imaging_weight=rimaging_weight,
integration_time=rintegration_time, channel_bandwidth=rchannel_bandwidth,
polarisation_frame=polarisation_frame, source=source, meta=meta)
vis.phasecentre = phasecentre
vis.configuration = config
log.info("create_blockvisibility: %s" % (vis_summary(vis)))
assert isinstance(vis, BlockVisibility), "vis is not a BlockVisibility: %r" % vis
return vis