def average_blockvisibility_by_channel(vis: BlockVisibility, channel_average=None) \
-> List[BlockVisibility]:
""" Average visibility by groups of channels, returning list of new visibility
:param vis: BlockVisibility
:param channel_average: Number of channels to average
:return: List[BlockVisibility]
"""
assert isinstance(vis, BlockVisibility), vis
vis_shape = list(vis.vis.shape)
ntimes, nants, _, nchan, npol = vis_shape
newvis_list = list()
ochannels = range(nchan)
channels = []
for i in range(0, nchan, channel_average):
channels.append([ochannels[i], ochannels[i + channel_average - 1] + 1])
for group in channels:
vis_shape[-2] = 1
freq = numpy.array([numpy.average(vis.frequency[group[0]:group[1]])])
cb = numpy.array([numpy.sum(vis.channel_bandwidth[group[0]:group[1]])])
newvis = \
BlockVisibility(data=None,
frequency=freq,
channel_bandwidth=cb,
phasecentre=vis.phasecentre,
configuration=vis.configuration,
uvw=vis.uvw,
time=vis.time,
vis=numpy.zeros(vis_shape, dtype='complex'),
flags=numpy.zeros(vis_shape, dtype='int'),
weight=numpy.zeros(vis_shape, dtype='float'),
imaging_weight=numpy.zeros(vis_shape, dtype='float'),
integration_time=vis.integration_time,
polarisation_frame=vis.polarisation_frame,
source=vis.source,
meta=vis.meta)
vf = vis.flags[..., group[0]:group[1], :]
vfvw = vis.flagged_vis[..., group[0]:group[1], :] * vis.weight[
..., group[0]:group[1], :]
vfw = vis.flagged_weight[..., group[0]:group[1], :]
vfiw = vis.flagged_imaging_weight[..., group[0]:group[1], :]
newvis.data['flags'][..., 0, :] = numpy.sum(vf, axis=-2)
newvis.data['flags'][newvis.data['flags'] < nchan] = 0
newvis.data['flags'][newvis.data['flags'] > 1] = 1
newvis.data['vis'][..., 0, :] = numpy.sum(vfvw, axis=-2)
newvis.data['weight'][..., 0, :] = numpy.sum(vfw, axis=-2)
newvis.data['imaging_weight'][..., 0, :] = numpy.sum(vfiw, axis=-2)
mask = newvis.flagged_weight > 0.0
newvis.data['vis'][
mask] = newvis.data['vis'][mask] / newvis.flagged_weight[mask]
newvis_list.append(newvis)
return newvis_list
def convert_hdf_to_blockvisibility(f):
""" Convert HDF root to blockvisibility
:param f:
:return:
"""
assert f.attrs[
'RASCIL_data_model'] == "BlockVisibility", "Not a BlockVisibility"
s = f.attrs['phasecentre_coords'].split()
ss = [float(s[0]), float(s[1])] * u.deg
phasecentre = SkyCoord(ra=ss[0],
dec=ss[1],
frame=f.attrs['phasecentre_frame'])
polarisation_frame = PolarisationFrame(f.attrs['polarisation_frame'])
frequency = f.attrs['frequency']
channel_bandwidth = f.attrs['channel_bandwidth']
data = numpy.array(f['data'])
source = f.attrs['source']
meta = ast.literal_eval(f.attrs['meta'])
vis = BlockVisibility(data=data,
polarisation_frame=polarisation_frame,
phasecentre=phasecentre,
frequency=frequency,
channel_bandwidth=channel_bandwidth,
source=source,
meta=meta)
vis.configuration = convert_configuration_from_hdf(f)
return vis
def visibility_gather_channel(vis_list: List[BlockVisibility],
vis: BlockVisibility = None):
""" Gather a visibility by channel
:param vis_list:
:param vis:
:return:
"""
cols = ['vis', 'weight']
if vis is None:
vis_shape = numpy.array(vis_list[0].vis.shape)
vis_shape[-2] = len(vis_list)
for v in vis_list:
assert len(v.frequency) == 1
assert len(v.channel_bandwidth) == 1
vis = BlockVisibility(
data=None,
frequency=numpy.array([v.frequency[0] for v in vis_list]),
channel_bandwidth=numpy.array(
[v.channel_bandwidth[0] for v in vis_list]),
phasecentre=vis_list[0].phasecentre,
configuration=vis_list[0].configuration,
uvw=vis_list[0].uvw,
time=vis_list[0].time,
vis=numpy.zeros(vis_shape, dtype=vis_list[0].vis.dtype),
weight=numpy.ones(vis_shape, dtype=vis_list[0].weight.dtype),
imaging_weight=numpy.ones(vis_shape,
dtype=vis_list[0].weight.dtype),
integration_time=vis_list[0].integration_time,
polarisation_frame=vis_list[0].polarisation_frame,
source=vis_list[0].source,
meta=vis_list[0].meta)
assert len(vis.frequency) == len(vis_list)
for chan, _ in enumerate(vis_list):
subvis = vis_list[chan]
assert abs(subvis.frequency[0] - vis.frequency[chan]) < 1e-15
for col in cols:
vis.data[col][..., chan, :] = subvis.data[col][..., 0, :]
vis.frequency[chan] = subvis.frequency[0]
nchan = vis.vis.shape[-2]
assert nchan == len(vis.frequency)
return vis
def divide_visibility(vis: BlockVisibility, modelvis: BlockVisibility):
""" Divide visibility by model forming visibility for equivalent point source
This is a useful intermediate product for calibration. Variation of the visibility in time and
frequency due to the model structure is removed and the data can be averaged to a limit determined
by the instrumental stability. The weight is adjusted to compensate for the division.
Zero divisions are avoided and the corresponding weight set to zero.
:param vis:
:param modelvis:
:return:
"""
assert isinstance(vis, Visibility) or isinstance(vis, BlockVisibility), vis
x = numpy.zeros_like(vis.vis)
xwt = numpy.abs(modelvis.vis)**2 * vis.flagged_weight
mask = xwt > 0.0
x[mask] = vis.vis[mask] / modelvis.vis[mask]
pointsource_vis = BlockVisibility(data=None,
flags=vis.flags,
frequency=vis.frequency,
channel_bandwidth=vis.channel_bandwidth,
phasecentre=vis.phasecentre,
configuration=vis.configuration,
uvw=vis.uvw,
time=vis.time,
integration_time=vis.integration_time,
vis=x,
weight=xwt,
source=vis.source,
meta=vis.meta)
return pointsource_vis
def integrate_visibility_by_channel(vis: BlockVisibility) -> BlockVisibility:
""" Integrate visibility across all channels, returning new visibility
:param vis: BlockVisibility
:return: BlockVisibility
"""
assert isinstance(vis, BlockVisibility), vis
vis_shape = list(vis.vis.shape)
ntimes, nants, _, nchan, npol = vis_shape
vis_shape[-2] = 1
newvis = BlockVisibility(
data=None,
frequency=numpy.ones([1]) * numpy.average(vis.frequency),
channel_bandwidth=numpy.ones([1]) * numpy.sum(vis.channel_bandwidth),
phasecentre=vis.phasecentre,
configuration=vis.configuration,
uvw=vis.uvw,
time=vis.time,
vis=numpy.zeros(vis_shape, dtype='complex'),
flags=numpy.zeros(vis_shape, dtype='int'),
weight=numpy.zeros(vis_shape, dtype='float'),
imaging_weight=numpy.zeros(vis_shape, dtype='float'),
integration_time=vis.integration_time,
polarisation_frame=vis.polarisation_frame,
source=vis.source,
meta=vis.meta)
newvis.data['flags'][..., 0, :] = numpy.sum(vis.flags, axis=-2)
newvis.data['flags'][newvis.data['flags'] < nchan] = 0
newvis.data['flags'][newvis.data['flags'] > 1] = 1
newvis.data['vis'][...,
0, :] = numpy.sum(vis.data['vis'] * vis.flagged_weight,
axis=-2)
newvis.data['weight'][..., 0, :] = numpy.sum(vis.flagged_weight, axis=-2)
newvis.data['imaging_weight'][...,
0, :] = numpy.sum(vis.flagged_imaging_weight,
axis=-2)
mask = newvis.flagged_weight > 0.0
newvis.data['vis'][
mask] = newvis.data['vis'][mask] / newvis.flagged_weight[mask]
return newvis
def visibility_gather_channel(vis_list: List[BlockVisibility],
vis: BlockVisibility = None):
""" Gather a visibility by channel
:param vis_list: List of Visibility
:param vis: Template output Visibility
:return:
"""
if vis is None:
vis_shape = numpy.array(vis_list[0].vis.shape)
vis_shape[-2] = len(vis_list) * vis_shape[-2]
gathered_frequency = numpy.concatenate([v.frequency for v in vis_list])
gathered_channel_bandwidth = numpy.concatenate([v.channel_bandwidth for v in vis_list])
vis = BlockVisibility(data=None,
frequency=gathered_frequency,
channel_bandwidth=gathered_channel_bandwidth,
phasecentre=vis_list[0].phasecentre,
configuration=vis_list[0].configuration,
uvw=vis_list[0].uvw,
time=vis_list[0].time,
vis=numpy.zeros(vis_shape, dtype=vis_list[0].vis.dtype),
flags=numpy.zeros(vis_shape, dtype=vis_list[0].flags.dtype),
weight=numpy.ones(vis_shape, dtype=vis_list[0].weight.dtype),
imaging_weight=numpy.ones(vis_shape, dtype=vis_list[0].weight.dtype),
integration_time=vis_list[0].integration_time,
polarisation_frame=vis_list[0].polarisation_frame,
source=vis_list[0].source,
meta=vis_list[0].meta)
cols = ['vis', 'weight', 'imaging_weight', 'flags']
for col in cols:
vis.data[col] = numpy.concatenate([v.data[col] for v in vis_list], axis=-2)
nchan = vis.vis.shape[-2]
assert nchan == len(vis.frequency)
return vis
def concatenate_blockvisibility_frequency(bvis_list):
"""Concatenate a list of BlockVisibility's in frequency
The list should be in sequence of channels
:param bvis_list: List of BlockVisibility
:return: BlockVisibility
"""
assert len(bvis_list) > 0
nvis = bvis_list[0].nvis
time = bvis_list[0].time
frequency = numpy.array(
numpy.array([bvis.frequency for bvis in bvis_list]).flat)
channel_bandwidth = numpy.array(
numpy.array([bvis.channel_bandwidth for bvis in bvis_list]).flat)
nchan = len(frequency)
ntimes, nants, _, _, npol = bvis_list[0].vis.shape
uvw = bvis_list[0].uvw
integration_time = bvis_list[0].integration_time
vis = numpy.zeros([nvis, nants, nants, nchan, npol], dtype='complex')
flags = numpy.zeros([nvis, nants, nants, nchan, npol], dtype='int')
weight = numpy.ones([nvis, nants, nants, nchan, npol])
imaging_weight = numpy.ones([nvis, nants, nants, nchan, npol])
echan = 0
for ibv, bvis in enumerate(bvis_list):
schan = echan
echan = schan + len(bvis.frequency)
flags[..., schan:echan, :] = bvis.flags[...]
vis[..., schan:echan, :] = bvis.flagged_vis[...]
weight[..., schan:echan, :] = bvis.flagged_weight[...]
imaging_weight[..., schan:echan, :] = bvis.flagged_imaging_weight[...]
result = BlockVisibility(
vis=vis,
flags=flags,
weight=weight,
imaging_weight=imaging_weight,
uvw=uvw,
time=time,
integration_time=integration_time,
frequency=frequency,
channel_bandwidth=channel_bandwidth,
polarisation_frame=bvis_list[0].polarisation_frame,
source=bvis_list[0].source,
configuration=bvis_list[0].configuration,
phasecentre=bvis_list[0].phasecentre,
meta=None)
assert result.nchan == nchan
return result
def extract_channel(v, chan):
vis_shape = numpy.array(v.data['vis'].shape)
vis_shape[3] = 1
vis = BlockVisibility(data=None,
frequency=numpy.array([v.frequency[chan]]),
channel_bandwidth=numpy.array([v.channel_bandwidth[chan]]),
phasecentre=v.phasecentre,
configuration=v.configuration,
uvw=v.uvw,
time=v.time,
vis=v.vis[..., chan, :][..., numpy.newaxis, :],
flags=v.flags[..., chan, :][..., numpy.newaxis, :],
weight=v.weight[..., chan, :][..., numpy.newaxis, :],
imaging_weight=v.imaging_weight[..., chan, :][..., numpy.newaxis, :],
integration_time=v.integration_time,
polarisation_frame=v.polarisation_frame,
source=v.source,
meta=v.meta)
return vis
def convert_blockvisibility_to_stokesI(vis):
"""Convert the polarisation frame data into Stokes I dropping other polarisations, return new Visibility
Args:
vis (obj): RASCIL visibility data.
Returns:
vis: New, converted visibility data.
"""
polarisation_frame = PolarisationFrame('stokesI')
poldef = vis.polarisation_frame
if poldef == PolarisationFrame('linear'):
vis_data = convert_linear_to_stokesI(vis.data['vis'])
vis_weight = (vis.weight[..., 0] + vis.weight[..., 3])[...,
numpy.newaxis]
vis_imaging_weight = (vis.imaging_weight[..., 0] +
vis.imaging_weight[..., 3])[..., numpy.newaxis]
elif poldef == PolarisationFrame('circular'):
vis_data = convert_circular_to_stokesI(vis.data['vis'])
vis_weight = (vis.weight[..., 0] + vis.weight[..., 3])[...,
numpy.newaxis]
vis_imaging_weight = (vis.imaging_weight[..., 0] +
vis.imaging_weight[..., 3])[..., numpy.newaxis]
else:
raise NameError("Polarisation frame %s unknown" % poldef)
return BlockVisibility(frequency=vis.frequency,
channel_bandwidth=vis.channel_bandwidth,
phasecentre=vis.phasecentre,
configuration=vis.configuration,
uvw=vis.uvw,
time=vis.time,
vis=vis_data,
weight=vis_weight,
imaging_weight=vis_imaging_weight,
integration_time=vis.integration_time,
polarisation_frame=polarisation_frame,
source=vis.source,
meta=vis.meta)
def apply_gaintable(vis: BlockVisibility,
gt: GainTable,
inverse=False,
**kwargs) -> BlockVisibility:
"""Apply a gain table to a block visibility
The corrected visibility is::
V_corrected = {g_i * g_j^*}^-1 V_obs
If the visibility data are polarised e.g. polarisation_frame("linear") then the inverse operator
represents an actual inverse of the gains.
:param vis: Visibility to have gains applied
:param gt: Gaintable to be applied
:param inverse: Apply the inverse (default=False)
:return: input vis with gains applied
"""
assert isinstance(
vis, BlockVisibility), "vis is not a BlockVisibility: %r" % vis
assert isinstance(gt, GainTable), "gt is not a GainTable: %r" % gt
assert_vis_gt_compatible(vis, gt)
if inverse:
log.debug('apply_gaintable: Apply inverse gaintable')
else:
log.debug('apply_gaintable: Apply gaintable')
is_scalar = gt.gain.shape[-2:] == (1, 1)
if vis.npol == 1:
log.debug('apply_gaintable: scalar gains')
# row_numbers = numpy.array(list(range(len(vis.time))), dtype='int')
row_numbers = numpy.arange(len(vis.time))
done = numpy.zeros(len(row_numbers), dtype='int')
for row in range(gt.ntimes):
vis_rows = numpy.abs(vis.time - gt.time[row]) < gt.interval[row] / 2.0
vis_rows = row_numbers[vis_rows]
if len(vis_rows) > 0:
# Lookup the gain for this set of visibilities
gain = gt.data['gain'][row]
cgain = numpy.conjugate(gt.data['gain'][row])
gainwt = gt.data['weight'][row]
# The shape of the mueller matrix is
nant, nchan, nrec, _ = gain.shape
original = vis.vis[vis_rows]
applied = copy.copy(vis.vis[vis_rows])
appliedwt = copy.copy(vis.weight[vis_rows])
if vis.npol == 1:
if inverse:
# lgain = numpy.ones_like(gain)
# lgain[numpy.abs(gain) > 0.0] = 1.0 / gain[numpy.abs(gain) > 0.0]
lgain = numpy.ones_like(gain)
numpy.putmask(lgain, numpy.abs(gain) > 0.0, 1.0 / gain)
else:
lgain = gain
# tlgain = lgain.T
# tclgain = numpy.conjugate(tlgain)
# smueller = numpy.ones([nchan, nant, nant], dtype='complex')
# for chan in range(nchan):
# smueller[chan, :, :] = numpy.ma.outer(tlgain[0, 0, chan, :],
# tclgain[0, 0, chan, :]).reshape([nant, nant])
# numpy.testing.assert_allclose(smueller,smueller1,rtol=1e-5)
# Original Code with Loop
# for sub_vis_row in range(original.shape[0]):
# for chan in range(nchan):
# applied[sub_vis_row, :, :, chan, 0] = \
# original[sub_vis_row, :, :, chan, 0] * smueller[chan, :, :]
# antantwt = numpy.outer(gainwt[:, chan, 0, 0], gainwt[:, chan, 0, 0])
# appliedwt[sub_vis_row, :, :, chan, 0] = antantwt
# applied[sub_vis_row, :, :, chan, 0][antantwt == 0.0] = 0.0
# Optimized (SIM-423)
# smueller1 = numpy.ones([nchan, nant, nant], dtype='complex')
smueller1 = numpy.einsum('ijlm,kjlm->jik', lgain,
numpy.conjugate(lgain))
for sub_vis_row in range(original.shape[0]):
for chan in range(nchan):
applied[sub_vis_row, :, :, chan, 0] = \
original[sub_vis_row, :, :, chan, 0] * smueller1[chan, :, :]
antantwt = numpy.einsum('i,j->ij', gainwt[:, chan, 0,
0],
gainwt[:, chan, 0, 0])
appliedwt[sub_vis_row, :, :, chan, 0] = antantwt
applied[sub_vis_row, :, :, chan,
0][antantwt == 0.0] = 0.0
# smueller1 = numpy.einsum('ijlm,kjlm->ikj', lgain, numpy.conjugate(lgain))
# for sub_vis_row in range(original.shape[0]):
# applied[sub_vis_row, :, :, :, 0] = \
# original[sub_vis_row, :, :, :, 0] * smueller1[:, :, :]
# antantwt = numpy.einsum('ik,jk->ijk',gainwt[:, :, 0, 0], gainwt[:, :, 0, 0])
# appliedwt[sub_vis_row, :, :, :, 0] = antantwt
# numpy.putmask(applied[sub_vis_row, :, :, :, 0], antantwt[:,:,:] == 0.0, 0.0)
elif vis.npol == 2:
has_inverse_ant = numpy.zeros([nant, nchan], dtype='bool')
if inverse:
igain = gain.copy()
cigain = cgain.copy()
for a1 in range(vis.nants):
for chan in range(nchan):
try:
igain[a1, chan, :, :] = numpy.linalg.inv(
gain[a1, chan, :, :])
cigain[a1, chan, :, :] = numpy.conjugate(
igain[a1, chan, :, :])
has_inverse_ant[a1, chan] = True
except numpy.linalg.linalg.LinAlgError:
has_inverse_ant[a1, chan] = False
for sub_vis_row in range(original.shape[0]):
for a1 in range(vis.nants - 1):
for a2 in range(a1 + 1, vis.nants):
for chan in range(nchan):
if has_inverse_ant[
a1,
chan] and has_inverse_ant[a2,
chan]:
cfs = numpy.diag(original[sub_vis_row,
a2, a1, chan,
...])
applied[sub_vis_row, a2, a1, chan, ...] = \
numpy.diag(igain[a1, chan, :, :] @ cfs @ cigain[a2, chan, :, :]).reshape([2])
else:
for sub_vis_row in range(original.shape[0]):
for a1 in range(vis.nants - 1):
for a2 in range(a1 + 1, vis.nants):
for chan in range(nchan):
cfs = numpy.diag(original[sub_vis_row, a2,
a1, chan, ...])
applied[sub_vis_row, a2, a1, chan, ...] = \
numpy.diag(gain[a1, chan, :, :] @ cfs @ cgain[a2, chan, :, :]).reshape([2])
elif vis.npol == 4:
has_inverse_ant = numpy.zeros([nant, nchan], dtype='bool')
if inverse:
igain = gain.copy()
cigain = cgain.copy()
for a1 in range(vis.nants):
for chan in range(nchan):
try:
igain[a1, chan, :, :] = numpy.linalg.inv(
gain[a1, chan, :, :])
cigain[a1, chan, :, :] = numpy.conjugate(
igain[a1, chan, :, :])
has_inverse_ant[a1, chan] = True
except numpy.linalg.linalg.LinAlgError:
has_inverse_ant[a1, chan] = False
for sub_vis_row in range(original.shape[0]):
for a1 in range(vis.nants - 1):
for a2 in range(a1 + 1, vis.nants):
for chan in range(nchan):
if has_inverse_ant[
a1,
chan] and has_inverse_ant[a2,
chan]:
cfs = original[sub_vis_row, a2, a1,
chan,
...].reshape([2, 2])
applied[sub_vis_row, a2, a1, chan, ...] = \
(igain[a1, chan, :, :] @ cfs @ cigain[a2, chan, :, :]).reshape([4])
else:
for sub_vis_row in range(original.shape[0]):
for a1 in range(vis.nants - 1):
for a2 in range(a1 + 1, vis.nants):
for chan in range(nchan):
cfs = original[sub_vis_row, a2, a1, chan,
...].reshape([2, 2])
applied[sub_vis_row, a2, a1, chan, ...] = \
(gain[a1, chan, :, :] @ cfs @ cgain[a2, chan, :, :]).reshape([4])
else:
times = Time(vis.time / 86400.0, format='mjd', scale='utc')
print(
"No row in gaintable for visibility time range {} to {}".
format(times[0].isot, times[-1].isot))
log.warning(
"No row in gaintable for visibility row, time range {} to {}"
.format(times[0].isot, times[-1].isot))
vis.data['vis'][vis_rows] = applied
for r in vis_rows:
done[r] = 1
assert done.all() == 1, "Some rows were not calibrated"
return vis
def create_blockvisibility_from_uvfits(fitsname, channum=None, ack=False, antnum=None):
""" Minimal UVFIT to BlockVisibility converter
The UVFITS format is much more general than the RASCIL BlockVisibility so we cut many corners.
Creates a list of BlockVisibility's, split by field and spectral window
:param fitsname: File name of UVFITS
:param channum: range of channels e.g. range(17,32), default is None meaning all
:param antnum: the number of antenna
:return:
"""
def ParamDict(hdul):
"Return the dictionary of the random parameters"
"""
The keys of the dictionary are the parameter names uppercased for
consistency. The values are the column numbers.
If multiple parameters have the same name (e.g., DATE) their
columns are entered as a list.
"""
pre=re.compile(r"PTYPE(?P<i>\d+)")
res={}
for k,v in hdul.header.items():
m=pre.match(k)
if m :
vu=v.upper()
if vu in res:
res[ vu ] = [ res[vu], int(m.group("i")) ]
else:
res[ vu ] = int(m.group("i"))
return res
# Open the file
with fits.open(fitsname) as hdul:
# Read Spectral Window
nspw = hdul[0].header['NAXIS5']
# Read Channel and Frequency Interval
freq_ref = hdul[0].header['CRVAL4']
mid_chan_freq = hdul[0].header['CRPIX4']
delt_freq = hdul[0].header['CDELT4']
# Real the number of channels in one spectral window
channels = hdul[0].header['NAXIS4']
freq = numpy.zeros([nspw, channels])
# Read Frequency or IF
freqhdulname="AIPS FQ"
sdhu = hdul.index_of(freqhdulname)
if_freq = hdul[sdhu].data['IF FREQ'].ravel()
for i in range(nspw):
temp = numpy.array([if_freq[i] + freq_ref+delt_freq* ff for ff in range(channels)])
freq[i,:] = temp[:]
freq_delt = numpy.ones(channels) * delt_freq
if channum is None:
channum = range(channels)
primary = hdul[0].data
# Read time
bvtimes = Time(hdul[0].data['DATE'], hdul[0].data['_DATE'], format='jd')
bv_times = numpy.unique(bvtimes.jd)
ntimes = len(bv_times)
# # Get Antenna
# blin = hdul[0].data['BASELINE']
antennahdulname="AIPS AN"
adhu = hdul.index_of(antennahdulname)
try:
antenna_name = hdul[adhu].data['ANNAME']
antenna_name = antenna_name.encode('ascii','ignore')
except:
antenna_name = None
antenna_xyz = hdul[adhu].data['STABXYZ']
antenna_mount = hdul[adhu].data['MNTSTA']
try:
antenna_diameter = hdul[adhu].data['DIAMETER']
except:
antenna_diameter = None
# To reading some UVFITS with wrong numbers of antenna
if antnum is not None:
if antenna_name is not None:
antenna_name = antenna_name[:antnum]
antenna_xyz = antenna_xyz[:antnum]
antenna_mount = antenna_mount[:antnum]
if antenna_diameter is not None:
antenna_diameter = antenna_diameter[:antnum]
nants = len(antenna_xyz)
# res= {}
# for i,row in enumerate(fin[ahdul].data):
# res[row.field("ANNAME") ] = i +1
# Get polarisation info
npol = hdul[0].header['NAXIS3']
corr_type = numpy.arange(hdul[0].header['NAXIS3']) - (hdul[0].header['CRPIX3'] - 1)
corr_type *= hdul[0].header['CDELT3']
corr_type += hdul[0].header['CRVAL3']
# xx yy xy yx
# These correspond to the CASA Stokes enumerations
if numpy.array_equal(corr_type, [1, 2, 3, 4]):
polarisation_frame = PolarisationFrame('stokesIQUV')
elif numpy.array_equal(corr_type, [-1, -2, -3, -4]):
polarisation_frame = PolarisationFrame('circular')
elif numpy.array_equal(corr_type, [-5, -6, -7, -8]):
polarisation_frame = PolarisationFrame('linear')
else:
raise KeyError("Polarisation not understood: %s" % str(corr_type))
configuration = Configuration(name='', data=None, location=None,
names=antenna_name, xyz=antenna_xyz, mount=antenna_mount, frame=None,
receptor_frame=polarisation_frame,
diameter=antenna_diameter)
# Get RA and DEC
phase_center_ra_degrees = numpy.float(hdul[0].header['CRVAL6'])
phase_center_dec_degrees = numpy.float(hdul[0].header['CRVAL7'])
# Get phasecentres
phasecentre = SkyCoord(ra=phase_center_ra_degrees * u.deg, dec=phase_center_dec_degrees * u.deg, frame='icrs', equinox='J2000')
# Get UVW
d=ParamDict(hdul[0])
if "UU" in d:
uu = hdul[0].data['UU']
vv = hdul[0].data['VV']
ww = hdul[0].data['WW']
else:
uu = hdul[0].data['UU---SIN']
vv = hdul[0].data['VV---SIN']
ww = hdul[0].data['WW---SIN']
_vis = hdul[0].data['DATA']
#_vis.shape = (nchan, ntimes, (nants*(nants-1)//2 ), npol, -1)
#self.vis = -(_vis[...,0] * 1.j + _vis[...,1])
row = 0
nchan = len(channum)
vis_list = list()
for spw_index in range(nspw):
bv_vis = numpy.zeros([ntimes, nants, nants, nchan, npol]).astype('complex')
bv_weight = numpy.zeros([ntimes, nants, nants, nchan, npol])
bv_uvw = numpy.zeros([ntimes, nants, nants, 3])
for time_index , time in enumerate(bv_times):
#restfreq = freq[channel_index]
for antenna1 in range(nants-1):
for antenna2 in range(antenna1 + 1, nants):
for channel_no, channel_index in enumerate(channum):
for pol_index in range(npol):
bv_vis[time_index, antenna2, antenna1, channel_no,pol_index] = complex(_vis[row,:,:,spw_index,channel_index, pol_index ,0],_vis[row,:,:,spw_index,channel_index,pol_index ,1])
bv_weight[time_index, antenna2, antenna1, channel_no, pol_index] = _vis[row,:,:,spw_index,channel_index,pol_index ,2]
bv_uvw[time_index, antenna2, antenna1, 0] = uu[row]* constants.c.value
bv_uvw[time_index, antenna2, antenna1, 1] = vv[row]* constants.c.value
bv_uvw[time_index, antenna2, antenna1, 2] = ww[row]* constants.c.value
row += 1
vis_list.append(BlockVisibility(uvw=bv_uvw,
time=bv_times,
frequency=freq[spw_index][channum],
channel_bandwidth=freq_delt[channum],
vis=bv_vis,
weight=bv_weight,
imaging_weight= bv_weight,
configuration=configuration,
phasecentre=phasecentre,
polarisation_frame=polarisation_frame))
return vis_list
def create_blockvisibility_from_ms(msname, channum=None, start_chan=None, end_chan=None, ack=False,
datacolumn='DATA', selected_sources=None, selected_dds=None):
""" Minimal MS to BlockVisibility converter
The MS format is much more general than the RASCIL BlockVisibility so we cut many corners. This requires casacore to be
installed. If not an exception ModuleNotFoundError is raised.
Creates a list of BlockVisibility's, split by field and spectral window
Reading of a subset of channels is possible using either start_chan and end_chan or channnum. Using start_chan
and end_chan is preferred since it only reads the channels required. Channum is more flexible and can be used to
read a random list of channels.
:param msname: File name of MS
:param channum: range of channels e.g. range(17,32), default is None meaning all
:param start_chan: Starting channel to read
:param end_chan: End channel to read
:return:
"""
try:
from casacore.tables import table # pylint: disable=import-error
except ModuleNotFoundError:
raise ModuleNotFoundError("casacore is not installed")
try:
from rascil.processing_components.visibility import msv2
except ModuleNotFoundError:
raise ModuleNotFoundError("cannot import msv2")
tab = table(msname, ack=ack)
log.debug("create_blockvisibility_from_ms: %s" % str(tab.info()))
if selected_sources is None:
fields = numpy.unique(tab.getcol('FIELD_ID'))
else:
fieldtab = table('%s/FIELD' % msname, ack=False)
sources = fieldtab.getcol('NAME')
fields = list()
for field, source in enumerate(sources):
if source in selected_sources: fields.append(field)
assert len(fields) > 0, "No sources selected"
if selected_dds is None:
dds = numpy.unique(tab.getcol('DATA_DESC_ID'))
else:
dds = selected_dds
log.debug("create_blockvisibility_from_ms: Reading unique fields %s, unique data descriptions %s" % (
str(fields), str(dds)))
vis_list = list()
for field in fields:
ftab = table(msname, ack=ack).query('FIELD_ID==%d' % field, style='')
for dd in dds:
meta = {'MSV2':{'FIELD_ID': field, 'DATA_DESC_ID':dd}}
ms = ftab.query('DATA_DESC_ID==%d' % dd, style='')
assert ms.nrows() > 0, "Empty selection for FIELD_ID=%d and DATA_DESC_ID=%d" % (field, dd)
log.debug("create_blockvisibility_from_ms: Found %d rows" % (ms.nrows()))
# The TIME column has descriptor:
# {'valueType': 'double', 'dataManagerType': 'IncrementalStMan', 'dataManagerGroup': 'TIME',
# 'option': 0, 'maxlen': 0, 'comment': 'Modified Julian Day',
# 'keywords': {'QuantumUnits': ['s'], 'MEASINFO': {'type': 'epoch', 'Ref': 'UTC'}}}
otime = ms.getcol('TIME')
datacol = ms.getcol(datacolumn, nrow=1)
datacol_shape = list(datacol.shape)
channels = datacol.shape[-2]
log.debug("create_blockvisibility_from_ms: Found %d channels" % (channels))
if channum is None:
if start_chan is not None and end_chan is not None:
try:
log.debug("create_blockvisibility_from_ms: Reading channels from %d to %d" %
(start_chan, end_chan))
blc = [start_chan, 0]
trc = [end_chan, datacol_shape[-1] - 1]
channum = range(start_chan, end_chan+1)
ms_vis = ms.getcolslice(datacolumn, blc=blc, trc=trc)
ms_weight = ms.getcol('WEIGHT')
except IndexError:
raise IndexError("channel number exceeds max. within ms")
else:
log.debug("create_blockvisibility_from_ms: Reading all %d channels" % (channels))
try:
channum = range(channels)
ms_vis = ms.getcol(datacolumn)[:, channum, :]
ms_weight = ms.getcol('WEIGHT')
channum = range(channels)
except IndexError:
raise IndexError("channel number exceeds max. within ms")
else:
log.debug("create_blockvisibility_from_ms: Reading channels %s " % (channum))
channum = range(channels)
try:
ms_vis = ms.getcol(datacolumn)[:, channum, :]
ms_weight = ms.getcol('WEIGHT')[:, :]
except IndexError:
raise IndexError("channel number exceeds max. within ms")
uvw = -1 * ms.getcol('UVW')
antenna1 = ms.getcol('ANTENNA1')
antenna2 = ms.getcol('ANTENNA2')
integration_time = ms.getcol('INTERVAL')
# time = Time((time-integration_time/2.0)/86400+ 2400000.5,format='jd',scale='utc').utc.value
time = (otime - integration_time / 2.0)
start_time = numpy.min(time)/86400.0
end_time = numpy.max(time)/86400.0
log.debug("create_blockvisibility_from_ms: Observation from %s to %s" %
(Time(start_time, format='mjd').iso, Time(end_time, format='mjd').iso))
# Now get info from the subtables
spwtab = table('%s/SPECTRAL_WINDOW' % msname, ack=False)
cfrequency = spwtab.getcol('CHAN_FREQ')[dd][channum]
cchannel_bandwidth = spwtab.getcol('CHAN_WIDTH')[dd][channum]
nchan = cfrequency.shape[0]
# Get polarisation info
npol = 4
poltab = table('%s/POLARIZATION' % msname, ack=False)
corr_type = poltab.getcol('CORR_TYPE')
# These correspond to the CASA Stokes enumerations
if numpy.array_equal(corr_type[0], [1, 2, 3, 4]):
polarisation_frame = PolarisationFrame('stokesIQUV')
elif numpy.array_equal(corr_type[0], [5, 6, 7, 8]):
polarisation_frame = PolarisationFrame('circular')
elif numpy.array_equal(corr_type[0], [9, 10, 11, 12]):
polarisation_frame = PolarisationFrame('linear')
elif numpy.array_equal(corr_type[0], [9]):
npol = 1
polarisation_frame = PolarisationFrame('stokesI')
else:
raise KeyError("Polarisation not understood: %s" % str(corr_type))
# Get configuration
anttab = table('%s/ANTENNA' % msname, ack=False)
nants = anttab.nrows()
mount = anttab.getcol('MOUNT')
names = anttab.getcol('NAME')
diameter = anttab.getcol('DISH_DIAMETER')
xyz = anttab.getcol('POSITION')
configuration = Configuration(name='', data=None, location=None,
names=names, xyz=xyz, mount=mount, frame=None,
receptor_frame=ReceptorFrame("linear"),
diameter=diameter)
# Get phasecentres
fieldtab = table('%s/FIELD' % msname, ack=False)
pc = fieldtab.getcol('PHASE_DIR')[field, 0, :]
source = fieldtab.getcol('NAME')[field]
phasecentre = SkyCoord(ra=pc[0] * u.rad, dec=pc[1] * u.rad, frame='icrs', equinox='J2000')
time_index_row = numpy.zeros_like(time, dtype='int')
time_last = time[0]
time_index = 0
for row, _ in enumerate(time):
if time[row] > time_last + integration_time[row]:
assert time[row] > time_last, "MS is not time-sorted - cannot convert"
time_index += 1
time_last = time[row]
time_index_row[row] = time_index
ntimes = time_index + 1
bv_times = numpy.zeros([ntimes])
bv_vis = numpy.zeros([ntimes, nants, nants, nchan, npol]).astype('complex')
bv_weight = numpy.zeros([ntimes, nants, nants, nchan, npol])
bv_imaging_weight = numpy.zeros([ntimes, nants, nants, nchan, npol])
bv_uvw = numpy.zeros([ntimes, nants, nants, 3])
bv_integration_time = numpy.zeros([ntimes])
for row, _ in enumerate(time):
time_index = time_index_row[row]
bv_times[time_index] = time[row]
bv_vis[time_index, antenna2[row], antenna1[row], ...] = ms_vis[row, ...]
bv_weight[time_index, antenna2[row], antenna1[row], :, ...] = ms_weight[row, numpy.newaxis, ...]
bv_imaging_weight[time_index, antenna2[row], antenna1[row], :, ...] = ms_weight[row, numpy.newaxis, ...]
bv_uvw[time_index, antenna2[row], antenna1[row], :] = uvw[row, :]
bv_integration_time[time_index] = integration_time[row]
vis_list.append(BlockVisibility(uvw=bv_uvw,
time=bv_times,
frequency=cfrequency,
channel_bandwidth=cchannel_bandwidth,
vis=bv_vis,
weight=bv_weight,
integration_time = bv_integration_time,
imaging_weight=bv_imaging_weight,
configuration=configuration,
phasecentre=phasecentre,
polarisation_frame=polarisation_frame,
source=source, meta=meta))
tab.close()
return vis_list
def create_blockvisibility(config: Configuration,
times: numpy.array,
frequency: numpy.array,
phasecentre: SkyCoord,
weight: float = 1.0,
polarisation_frame: PolarisationFrame = None,
integration_time=1.0,
channel_bandwidth=1e6,
zerow=False,
elevation_limit=None,
source='unknown',
meta=None,
**kwargs) -> BlockVisibility:
""" Create a BlockVisibility from Configuration, hour angles, and direction of source
Note that we keep track of the integration time for BDA purposes
:param config: Configuration of antennas
:param times: hour angles in radians
:param frequency: frequencies (Hz] [nchan]
:param weight: weight of a single sample
:param phasecentre: phasecentre of observation
:param channel_bandwidth: channel bandwidths: (Hz] [nchan]
:param integration_time: Integration time ('auto' or value in s)
:param polarisation_frame:
:return: BlockVisibility
"""
assert phasecentre is not None, "Must specify phase centre"
if polarisation_frame is None:
polarisation_frame = correlate_polarisation(config.receptor_frame)
latitude = config.location.geodetic[1].to('rad').value
nch = len(frequency)
ants_xyz = config.data['xyz']
nants = len(config.data['names'])
ntimes = 0
n_flagged = 0
for iha, ha in enumerate(times):
# Calculate the positions of the antennas as seen for this hour angle
# and declination
_, elevation = hadec_to_azel(ha, phasecentre.dec.rad, latitude)
if elevation_limit is None or (elevation > elevation_limit):
ntimes +=1
else:
n_flagged += 1
assert ntimes > 0, "No unflagged points"
if elevation_limit is not None:
log.info('create_visibility: flagged %d/%d times below elevation limit %f (rad)' %
(n_flagged, ntimes, elevation_limit))
else:
log.info('create_visibility: created %d times' % (ntimes))
npol = polarisation_frame.npol
visshape = [ntimes, nants, nants, nch, npol]
rvis = numpy.zeros(visshape, dtype='complex')
rweight = weight * numpy.ones(visshape)
rimaging_weight = numpy.ones(visshape)
rtimes = numpy.zeros([ntimes])
ruvw = numpy.zeros([ntimes, nants, nants, 3])
# Do each hour angle in turn
itime = 0
for iha, ha in enumerate(times):
# Calculate the positions of the antennas as seen for this hour angle
# and declination
ant_pos = xyz_to_uvw(ants_xyz, ha, phasecentre.dec.rad)
_, elevation = hadec_to_azel(ha, phasecentre.dec.rad, latitude)
if elevation_limit is None or (elevation > elevation_limit):
rtimes[itime] = ha * 43200.0 / numpy.pi
rweight[itime, ...] = 1.0
# Loop over all pairs of antennas. Note that a2>a1
for a1 in range(nants):
for a2 in range(a1 + 1, nants):
ruvw[itime, a2, a1, :] = (ant_pos[a2, :] - ant_pos[a1, :])
ruvw[itime, a1, a2, :] = (ant_pos[a1, :] - ant_pos[a2, :])
itime += 1
rintegration_time = numpy.full_like(rtimes, integration_time)
rchannel_bandwidth = channel_bandwidth
if zerow:
ruvw[..., 2] = 0.0
vis = BlockVisibility(uvw=ruvw, time=rtimes, frequency=frequency, vis=rvis, weight=rweight,
imaging_weight=rimaging_weight,
integration_time=rintegration_time, channel_bandwidth=rchannel_bandwidth,
polarisation_frame=polarisation_frame, source=source, meta=meta)
vis.phasecentre = phasecentre
vis.configuration = config
log.info("create_blockvisibility: %s" % (vis_summary(vis)))
assert isinstance(vis, BlockVisibility), "vis is not a BlockVisibility: %r" % vis
return vis
def divide_visibility(vis: BlockVisibility, modelvis: BlockVisibility):
""" Divide visibility by model forming visibility for equivalent point source
This is a useful intermediate product for calibration. Variation of the visibility in time and
frequency due to the model structure is removed and the data can be averaged to a limit determined
by the instrumental stability. The weight is adjusted to compensate for the division.
Zero divisions are avoided and the corresponding weight set to zero.
:param vis:
:param modelvis:
:return:
"""
assert isinstance(vis, Visibility) or isinstance(vis, BlockVisibility), vis
# Different for scalar and vector/matrix cases
isscalar = vis.polarisation_frame.npol == 1
if isscalar:
# Scalar case is straightforward
x = numpy.zeros_like(vis.vis)
xwt = numpy.abs(modelvis.vis)**2 * vis.weight
mask = xwt > 0.0
x[mask] = vis.vis[mask] / modelvis.vis[mask]
else:
nrows, nants, _, nchan, npol = vis.vis.shape
nrec = 2
assert nrec * nrec == npol
xshape = (nrows, nants, nants, nchan, nrec, nrec)
x = numpy.zeros(xshape, dtype='complex')
xwt = numpy.zeros(xshape)
# TODO: Remove filter when fixed to use ndarray
warnings.simplefilter("ignore", category=PendingDeprecationWarning)
# TODO: optimise loop
for row in range(nrows):
for ant1 in range(nants):
for ant2 in range(ant1 + 1, nants):
for chan in range(nchan):
ovis = numpy.matrix(vis.vis[row, ant2, ant1,
chan].reshape([2, 2]))
mvis = numpy.matrix(modelvis.vis[row, ant2, ant1,
chan].reshape([2, 2]))
wt = numpy.matrix(vis.weight[row, ant2, ant1,
chan].reshape([2, 2]))
x[row, ant2, ant1,
chan] = numpy.matmul(numpy.linalg.inv(mvis), ovis)
xwt[row, ant2, ant1,
chan] = numpy.dot(mvis,
numpy.multiply(wt, mvis.H)).real
x = x.reshape((nrows, nants, nants, nchan, nrec * nrec))
xwt = xwt.reshape((nrows, nants, nants, nchan, nrec * nrec))
pointsource_vis = BlockVisibility(data=None,
frequency=vis.frequency,
channel_bandwidth=vis.channel_bandwidth,
phasecentre=vis.phasecentre,
configuration=vis.configuration,
uvw=vis.uvw,
time=vis.time,
integration_time=vis.integration_time,
vis=x,
weight=xwt,
source=vis.source,
meta=vis.meta)
return pointsource_vis
def convert_blockvisibility_to_stokesI(vis):
"""Convert the polarisation frame data into Stokes I dropping other polarisations, return new Visibility
:param vis: Visibility
:return: Converted visibility data.
"""
if vis.polarisation_frame == PolarisationFrame('stokesI'):
return vis
polarisation_frame = PolarisationFrame('stokesI')
poldef = vis.polarisation_frame
if poldef == PolarisationFrame('linear'):
vis_data = convert_linear_to_stokesI(vis.data['vis'])
vis_flags = numpy.logical_or(vis.flags[..., 0],
vis.flags[..., 3])[..., numpy.newaxis]
vis_weight = (vis.flagged_weight[..., 0] +
vis.flagged_weight[..., 3])[..., numpy.newaxis]
vis_imaging_weight = (
vis.flagged_imaging_weight[..., 0] +
vis.flagged_imaging_weight[..., 3])[..., numpy.newaxis]
elif poldef == PolarisationFrame('linearnp'):
vis_data = convert_linear_to_stokesI(vis.data['vis'])
vis_flags = numpy.logical_or(vis.flags[..., 0],
vis.flags[..., 1])[..., numpy.newaxis]
vis_weight = (vis.flagged_weight[..., 0] +
vis.flagged_weight[..., 1])[..., numpy.newaxis]
vis_imaging_weight = (
vis.flagged_imaging_weight[..., 0] +
vis.flagged_imaging_weight[..., 1])[..., numpy.newaxis]
elif poldef == PolarisationFrame('circular'):
vis_data = convert_circular_to_stokesI(vis.data['vis'])
vis_flags = numpy.logical_or(vis.flags[..., 0],
vis.flags[..., 3])[..., numpy.newaxis]
vis_weight = (vis.flagged_weight[..., 0] +
vis.flagged_weight[..., 3])[..., numpy.newaxis]
vis_imaging_weight = (
vis.flagged_imaging_weight[..., 0] +
vis.flagged_imaging_weight[..., 3])[..., numpy.newaxis]
elif poldef == PolarisationFrame('circularnp'):
vis_data = convert_circular_to_stokesI(vis.data['vis'])
vis_flags = numpy.logical_or(vis.flags[..., 0],
vis.flags[..., 1])[..., numpy.newaxis]
vis_weight = (vis.flagged_weight[..., 0] +
vis.flagged_weight[..., 1])[..., numpy.newaxis]
vis_imaging_weight = (
vis.flagged_imaging_weight[..., 0] +
vis.flagged_imaging_weight[..., 1])[..., numpy.newaxis]
else:
raise NameError("Polarisation frame %s unknown" % poldef)
return BlockVisibility(frequency=vis.frequency,
channel_bandwidth=vis.channel_bandwidth,
phasecentre=vis.phasecentre,
configuration=vis.configuration,
uvw=vis.uvw,
time=vis.time,
vis=vis_data,
flags=vis_flags,
weight=vis_weight,
imaging_weight=vis_imaging_weight,
integration_time=vis.integration_time,
polarisation_frame=polarisation_frame,
source=vis.source,
meta=vis.meta)
