def test_rec_frame(self):
rec_frame = ReceptorFrame("linear")
assert rec_frame.nrec == 2
rec_frame = ReceptorFrame("circular")
assert rec_frame.nrec == 2
rec_frame = ReceptorFrame("stokesI")
assert rec_frame.nrec == 1
with self.assertRaises(ValueError):
rec_frame = ReceptorFrame("circuloid")
def convert_hdf_to_pointingtable(f):
""" Convert HDF root to a PointingTable
:param f:
:return:
"""
assert f.attrs[
'RASCIL_data_model'] == "PointingTable", "Not a PointingTable"
receptor_frame = ReceptorFrame(f.attrs['receptor_frame'])
frequency = numpy.array(f.attrs['frequency'])
data = numpy.array(f['data'])
s = f.attrs['pointingcentre_coords'].split()
ss = [float(s[0]), float(s[1])] * u.deg
pointingcentre = SkyCoord(ra=ss[0],
dec=ss[1],
frame=f.attrs['pointingcentre_frame'])
pointing_frame = f.attrs['pointing_frame']
configuration = convert_configuration_from_hdf(f)
pt = PointingTable(data=data,
frequency=frequency,
receptor_frame=receptor_frame,
pointing_frame=pointing_frame,
pointingcentre=pointingcentre,
configuration=configuration)
return pt
def convert_configuration_from_hdf(f):
""" Extract configuration from HDF
:param f:
:return: Configuration
"""
cf = f['configuration']
assert cf.attrs[
'RASCIL_data_model'] == "Configuration", "%s is a Configuration" % cf.attrs[
'RASCIL_data_model']
name = cf.attrs['name']
location = convert_earthlocation_from_string(cf.attrs['location'])
receptor_frame = ReceptorFrame(cf.attrs['receptor_frame'])
frame = cf.attrs['frame']
xyz = cf['configuration/xyz']
diameter = cf['configuration/diameter']
names = [str(n) for n in cf['configuration/names']]
mount = [str(m) for m in cf['configuration/mount']]
return Configuration(name=name,
location=location,
receptor_frame=receptor_frame,
xyz=xyz,
frame=frame,
diameter=diameter,
names=names,
mount=mount)
def __initData(self):
"""Private function to generate a random set of data for writing a UVFITS
file. The data is returned as a dictionary with keys:
* freq - frequency array in Hz
* site - Observatory object
* stands - array of stand numbers
* bl - list of baseline pairs in real stand numbers
* vis - array of visibility data in baseline x freq format
"""
if run_ms_tests == False:
return
# Frequency range
freq = numpy.arange(0,512)*20e6/512 + 40e6
channel_width = numpy.full_like(freq,20e6/512.)
# Site and stands
obs = EarthLocation(lon="116.76444824", lat="-26.824722084", height=300.0)
mount = numpy.array(['equat','equat','equat','equat','equat','equat','equat','equat','equat','equat'])
names = numpy.array(['ak02','ak04','ak05','ak12','ak13','ak14','ak16','ak24','ak28','ak30'])
diameter = numpy.array([12.,12.,12.,12.,12.,12.,12.,12.,12.,12.])
xyz = numpy.array([[-2556109.98244348,5097388.70050131,-2848440.1332423 ],
[-2556087.396082 , 5097423.589662 , -2848396.867933 ],
[-2556028.60254059, 5097451.46195695, -2848399.83113161],
[-2556496.23893101, 5097333.71466669, -2848187.33832738],
[-2556407.35299627, 5097064.98390756, -2848756.02069474],
[-2555972.78456557, 5097233.65481756, -2848839.88915184],
[-2555592.88867802, 5097835.02121109, -2848098.26409648],
[-2555959.34313275, 5096979.52802882, -2849303.57702486],
[-2556552.97431815, 5097767.23612874, -2847354.29540396],
[-2557348.40370367, 5097170.17682775, -2847716.21368966]])
site_config = Configuration(name='ASKAP', data=None, location= obs,
names=names, xyz=xyz, mount=mount, frame=None,
receptor_frame=ReceptorFrame("linear"),
diameter=diameter)
antennas = []
for i in range(len(names)):
antennas.append(Antenna(i,Stand(names[i],xyz[i,0],xyz[i,1],xyz[i,2])))
# Set baselines and data
blList = []
N = len(antennas)
antennas2 = antennas
for i in range(0, N - 1):
for j in range(i + 1, N):
blList.append((antennas[i], antennas2[j]))
visData = numpy.random.rand(len(blList), len(freq))
visData = visData.astype(numpy.complex64)
return {'freq': freq, 'channel_width':channel_width, 'site': site_config, 'antennas': antennas, 'bl': blList, 'vis': visData}
def __init__(self,
data=None,
pointing: numpy.array = None,
nominal: numpy.array = None,
time: numpy.array = None,
interval=None,
weight: numpy.array = None,
residual: numpy.array = None,
frequency: numpy.array = None,
receptor_frame: ReceptorFrame = ReceptorFrame("linear"),
pointing_frame: str = "local",
pointingcentre=None,
configuration=None):
""" Create a pointing table from arrays
:param data: Structured data (used in copying)
:param pointing: Pointing (rad) [:, nants, nchan, nrec, 2]
:param nominal: Nominal pointing (rad) [:, nants, nchan, nrec, 2]
:param time: Centroid of solution [:]
:param interval: Interval of validity
:param weight: Weight [: nants, nchan, nrec]
:param residual: Residual [: nants, nchan, nrec, 2]
:param frequency: [nchan]
:param receptor_frame: e.g. Receptor_frame("linear")
:param pointing_frame: Pointing frame
:param pointingcentre: SkyCoord
:param configuration: Configuration
"""
if data is None and pointing is not None:
nrec = receptor_frame.nrec
nrows = pointing.shape[0]
nants = pointing.shape[1]
nchan = pointing.shape[2]
assert len(frequency) == nchan, "Discrepancy in frequency channels"
desc = [('pointing', 'f16', (nants, nchan, nrec, 2)),
('nominal', 'f16', (nants, nchan, nrec, 2)),
('weight', 'f8', (nants, nchan, nrec, 2)),
('residual', 'f8', (nchan, nrec, 2)), ('time', 'f8'),
('interval', 'f8')]
data = numpy.zeros(shape=[nrows], dtype=desc)
data['pointing'] = pointing
data['weight'] = weight
data['time'] = time
data['interval'] = interval
data['residual'] = residual
data['nominal'] = nominal
self.data = data
self.frequency = frequency
self.receptor_frame = receptor_frame
self.pointing_frame = pointing_frame
self.pointingcentre = pointingcentre
self.configuration = configuration
def __init__(self,
data=None,
gain: numpy.array = None,
time: numpy.array = None,
interval=None,
weight: numpy.array = None,
residual: numpy.array = None,
frequency: numpy.array = None,
receptor_frame: ReceptorFrame = ReceptorFrame("linear"),
phasecentre=None,
configuration=None):
""" Create a gaintable from arrays
The definition of gain is:
Vobs = g_i g_j^* Vmodel
:param data: Structured data (used in copying)
:param gain: Complex gain [nrows, nants, nchan, nrec, nrec]
:param time: Centroid of solution [nrows]
:param interval: Interval of validity
:param weight: Weight of gain [nrows, nchan, nrec, nrec]
:param residual: Residual of fit [nchan, nrec, nrec]
:param frequency: Frequency [nchan]
:param receptor_frame: Receptor frame
:param phasecentre: Phasecentre (SkyCoord)
:param configuration: Configuration
"""
if data is None and gain is not None:
nrec = receptor_frame.nrec
nrows = gain.shape[0]
nants = gain.shape[1]
nchan = gain.shape[2]
assert len(frequency) == nchan, "Discrepancy in frequency channels"
desc = [('gain', 'c16', (nants, nchan, nrec, nrec)),
('weight', 'f8', (nants, nchan, nrec, nrec)),
('residual', 'f8', (nchan, nrec, nrec)), ('time', 'f8'),
('interval', 'f8')]
data = numpy.zeros(shape=[nrows], dtype=desc)
data['gain'] = gain
data['weight'] = weight
data['time'] = time
data['interval'] = interval
data['residual'] = residual
self.data = data
self.frequency = frequency
self.receptor_frame = receptor_frame
self.phasecentre = phasecentre
self.configuration = configuration
def __init__(self, name='', data=None, location=None,
names="%s", xyz=None, mount="alt-az", frame="",
receptor_frame=ReceptorFrame("linear"),
diameter=None, offset=None, stations="%s"):
"""Configuration object describing data for processing
:param name: Name of configuration e.g. 'LOWR3'
:param data: Data array (used in copying)
:param location: Location of array as an astropy EarthLocation
:param names: Names of the dishes/stations
:param xyz: Geocentric coordinates of dishes/stations
:param mount: Mount types of dishes/stations 'altaz' | 'xy' | 'equatorial'
:param frame: Reference frame of locations
:param receptor_frame: Receptor frame
:param diameter: Diameters of dishes/stations (m)
:param offset: Axis offset (m)
:param stations: Identifiers of the dishes/stations
"""
if data is None and xyz is not None:
desc = [('names', 'U12'),
('xyz', 'f8', (3,)),
('diameter', 'f8'),
('mount', 'U12'),
('offset', 'f8', (3,)),
('stations', 'U12')]
nants = xyz.shape[0]
if isinstance(names, str):
names = [names % ant for ant in range(nants)]
if isinstance(mount, str):
mount = numpy.repeat(mount, nants)
data = numpy.zeros(shape=[nants], dtype=desc)
data['names'] = names
data['xyz'] = xyz
data['mount'] = mount
data['diameter'] = diameter
if offset is not None:
data['offset'] = offset
if isinstance(stations, str):
stations = [stations % ant for ant in range(nants)]
data['stations'] = stations
self.name = name
self.data = data
self.location = location
self.frame = frame
self.receptor_frame = receptor_frame
def convert_hdf_to_gaintable(f):
""" Convert HDF root to a GainTable
:param f:
:return:
"""
assert f.attrs['RASCIL_data_model'] == "GainTable", "Not a GainTable"
receptor_frame = ReceptorFrame(f.attrs['receptor_frame'])
frequency = numpy.array(f.attrs['frequency'])
data = numpy.array(f['data'])
s = f.attrs['phasecentre_coords'].split()
ss = [float(s[0]), float(s[1])] * u.deg
phasecentre = SkyCoord(ra=ss[0],
dec=ss[1],
frame=f.attrs['phasecentre_frame'])
gt = GainTable(data=data,
receptor_frame=receptor_frame,
frequency=frequency,
phasecentre=phasecentre)
return gt
def test_congruent(self):
for frame in ["linear", "circular", "stokesI"]:
assert congruent_polarisation(ReceptorFrame(frame),
PolarisationFrame(frame))
assert not congruent_polarisation(ReceptorFrame(frame),
PolarisationFrame("stokesIQUV"))
def test_correlate(self):
for frame in ["linear", "circular", "stokesI"]:
rec_frame = ReceptorFrame(frame)
assert correlate_polarisation(rec_frame) == PolarisationFrame(
frame)
def create_gaintable_from_blockvisibility(vis: BlockVisibility,
timeslice=None,
frequencyslice: float = None,
**kwargs) -> GainTable:
""" Create gain table from visibility.
This makes an empty gain table consistent with the BlockVisibility.
:param vis: BlockVisibilty
:param timeslice: Time interval between solutions (s)
:param frequencyslice: Frequency solution width (Hz) (NYI)
:return: GainTable
"""
assert isinstance(
vis, BlockVisibility), "vis is not a BlockVisibility: %r" % vis
nants = vis.nants
if timeslice is None or timeslice == 'auto':
utimes = numpy.unique(vis.time)
gain_interval = vis.integration_time
else:
utimes = vis.time[0] + timeslice * numpy.unique(
numpy.round((vis.time - vis.time[0]) / timeslice))
gain_interval = timeslice * numpy.ones_like(utimes)
ntimes = len(utimes)
# log.debug('create_gaintable_from_blockvisibility: times are %s' % str(utimes))
# log.debug('create_gaintable_from_blockvisibility: intervals are %s' % str(gain_interval))
ntimes = len(utimes)
ufrequency = numpy.unique(vis.frequency)
nfrequency = len(ufrequency)
receptor_frame = ReceptorFrame(vis.polarisation_frame.type)
nrec = receptor_frame.nrec
gainshape = [ntimes, nants, nfrequency, nrec, nrec]
gain = numpy.ones(gainshape, dtype='complex')
if nrec > 1:
gain[..., 0, 1] = 0.0
gain[..., 1, 0] = 0.0
gain_weight = numpy.ones(gainshape)
gain_time = utimes
gain_frequency = ufrequency
gain_residual = numpy.zeros([ntimes, nfrequency, nrec, nrec])
gt = GainTable(gain=gain,
time=gain_time,
interval=gain_interval,
weight=gain_weight,
residual=gain_residual,
frequency=gain_frequency,
receptor_frame=receptor_frame,
phasecentre=vis.phasecentre,
configuration=vis.configuration)
assert isinstance(gt, GainTable), "gt is not a GainTable: %r" % gt
assert_vis_gt_compatible(vis, gt)
return gt
def create_blockvisibility_from_ms(msname, channum=None, start_chan=None, end_chan=None, ack=False,
datacolumn='DATA', selected_sources=None, selected_dds=None):
""" Minimal MS to BlockVisibility converter
The MS format is much more general than the RASCIL BlockVisibility so we cut many corners. This requires casacore to be
installed. If not an exception ModuleNotFoundError is raised.
Creates a list of BlockVisibility's, split by field and spectral window
Reading of a subset of channels is possible using either start_chan and end_chan or channnum. Using start_chan
and end_chan is preferred since it only reads the channels required. Channum is more flexible and can be used to
read a random list of channels.
:param msname: File name of MS
:param channum: range of channels e.g. range(17,32), default is None meaning all
:param start_chan: Starting channel to read
:param end_chan: End channel to read
:return:
"""
try:
from casacore.tables import table # pylint: disable=import-error
except ModuleNotFoundError:
raise ModuleNotFoundError("casacore is not installed")
try:
from rascil.processing_components.visibility import msv2
except ModuleNotFoundError:
raise ModuleNotFoundError("cannot import msv2")
tab = table(msname, ack=ack)
log.debug("create_blockvisibility_from_ms: %s" % str(tab.info()))
if selected_sources is None:
fields = numpy.unique(tab.getcol('FIELD_ID'))
else:
fieldtab = table('%s/FIELD' % msname, ack=False)
sources = fieldtab.getcol('NAME')
fields = list()
for field, source in enumerate(sources):
if source in selected_sources: fields.append(field)
assert len(fields) > 0, "No sources selected"
if selected_dds is None:
dds = numpy.unique(tab.getcol('DATA_DESC_ID'))
else:
dds = selected_dds
log.debug("create_blockvisibility_from_ms: Reading unique fields %s, unique data descriptions %s" % (
str(fields), str(dds)))
vis_list = list()
for field in fields:
ftab = table(msname, ack=ack).query('FIELD_ID==%d' % field, style='')
for dd in dds:
meta = {'MSV2':{'FIELD_ID': field, 'DATA_DESC_ID':dd}}
ms = ftab.query('DATA_DESC_ID==%d' % dd, style='')
assert ms.nrows() > 0, "Empty selection for FIELD_ID=%d and DATA_DESC_ID=%d" % (field, dd)
log.debug("create_blockvisibility_from_ms: Found %d rows" % (ms.nrows()))
# The TIME column has descriptor:
# {'valueType': 'double', 'dataManagerType': 'IncrementalStMan', 'dataManagerGroup': 'TIME',
# 'option': 0, 'maxlen': 0, 'comment': 'Modified Julian Day',
# 'keywords': {'QuantumUnits': ['s'], 'MEASINFO': {'type': 'epoch', 'Ref': 'UTC'}}}
otime = ms.getcol('TIME')
datacol = ms.getcol(datacolumn, nrow=1)
datacol_shape = list(datacol.shape)
channels = datacol.shape[-2]
log.debug("create_blockvisibility_from_ms: Found %d channels" % (channels))
if channum is None:
if start_chan is not None and end_chan is not None:
try:
log.debug("create_blockvisibility_from_ms: Reading channels from %d to %d" %
(start_chan, end_chan))
blc = [start_chan, 0]
trc = [end_chan, datacol_shape[-1] - 1]
channum = range(start_chan, end_chan+1)
ms_vis = ms.getcolslice(datacolumn, blc=blc, trc=trc)
ms_weight = ms.getcol('WEIGHT')
except IndexError:
raise IndexError("channel number exceeds max. within ms")
else:
log.debug("create_blockvisibility_from_ms: Reading all %d channels" % (channels))
try:
channum = range(channels)
ms_vis = ms.getcol(datacolumn)[:, channum, :]
ms_weight = ms.getcol('WEIGHT')
channum = range(channels)
except IndexError:
raise IndexError("channel number exceeds max. within ms")
else:
log.debug("create_blockvisibility_from_ms: Reading channels %s " % (channum))
channum = range(channels)
try:
ms_vis = ms.getcol(datacolumn)[:, channum, :]
ms_weight = ms.getcol('WEIGHT')[:, :]
except IndexError:
raise IndexError("channel number exceeds max. within ms")
uvw = -1 * ms.getcol('UVW')
antenna1 = ms.getcol('ANTENNA1')
antenna2 = ms.getcol('ANTENNA2')
integration_time = ms.getcol('INTERVAL')
# time = Time((time-integration_time/2.0)/86400+ 2400000.5,format='jd',scale='utc').utc.value
time = (otime - integration_time / 2.0)
start_time = numpy.min(time)/86400.0
end_time = numpy.max(time)/86400.0
log.debug("create_blockvisibility_from_ms: Observation from %s to %s" %
(Time(start_time, format='mjd').iso, Time(end_time, format='mjd').iso))
# Now get info from the subtables
spwtab = table('%s/SPECTRAL_WINDOW' % msname, ack=False)
cfrequency = spwtab.getcol('CHAN_FREQ')[dd][channum]
cchannel_bandwidth = spwtab.getcol('CHAN_WIDTH')[dd][channum]
nchan = cfrequency.shape[0]
# Get polarisation info
npol = 4
poltab = table('%s/POLARIZATION' % msname, ack=False)
corr_type = poltab.getcol('CORR_TYPE')
# These correspond to the CASA Stokes enumerations
if numpy.array_equal(corr_type[0], [1, 2, 3, 4]):
polarisation_frame = PolarisationFrame('stokesIQUV')
elif numpy.array_equal(corr_type[0], [5, 6, 7, 8]):
polarisation_frame = PolarisationFrame('circular')
elif numpy.array_equal(corr_type[0], [9, 10, 11, 12]):
polarisation_frame = PolarisationFrame('linear')
elif numpy.array_equal(corr_type[0], [9]):
npol = 1
polarisation_frame = PolarisationFrame('stokesI')
else:
raise KeyError("Polarisation not understood: %s" % str(corr_type))
# Get configuration
anttab = table('%s/ANTENNA' % msname, ack=False)
nants = anttab.nrows()
mount = anttab.getcol('MOUNT')
names = anttab.getcol('NAME')
diameter = anttab.getcol('DISH_DIAMETER')
xyz = anttab.getcol('POSITION')
configuration = Configuration(name='', data=None, location=None,
names=names, xyz=xyz, mount=mount, frame=None,
receptor_frame=ReceptorFrame("linear"),
diameter=diameter)
# Get phasecentres
fieldtab = table('%s/FIELD' % msname, ack=False)
pc = fieldtab.getcol('PHASE_DIR')[field, 0, :]
source = fieldtab.getcol('NAME')[field]
phasecentre = SkyCoord(ra=pc[0] * u.rad, dec=pc[1] * u.rad, frame='icrs', equinox='J2000')
time_index_row = numpy.zeros_like(time, dtype='int')
time_last = time[0]
time_index = 0
for row, _ in enumerate(time):
if time[row] > time_last + integration_time[row]:
assert time[row] > time_last, "MS is not time-sorted - cannot convert"
time_index += 1
time_last = time[row]
time_index_row[row] = time_index
ntimes = time_index + 1
bv_times = numpy.zeros([ntimes])
bv_vis = numpy.zeros([ntimes, nants, nants, nchan, npol]).astype('complex')
bv_weight = numpy.zeros([ntimes, nants, nants, nchan, npol])
bv_imaging_weight = numpy.zeros([ntimes, nants, nants, nchan, npol])
bv_uvw = numpy.zeros([ntimes, nants, nants, 3])
bv_integration_time = numpy.zeros([ntimes])
for row, _ in enumerate(time):
time_index = time_index_row[row]
bv_times[time_index] = time[row]
bv_vis[time_index, antenna2[row], antenna1[row], ...] = ms_vis[row, ...]
bv_weight[time_index, antenna2[row], antenna1[row], :, ...] = ms_weight[row, numpy.newaxis, ...]
bv_imaging_weight[time_index, antenna2[row], antenna1[row], :, ...] = ms_weight[row, numpy.newaxis, ...]
bv_uvw[time_index, antenna2[row], antenna1[row], :] = uvw[row, :]
bv_integration_time[time_index] = integration_time[row]
vis_list.append(BlockVisibility(uvw=bv_uvw,
time=bv_times,
frequency=cfrequency,
channel_bandwidth=cchannel_bandwidth,
vis=bv_vis,
weight=bv_weight,
integration_time = bv_integration_time,
imaging_weight=bv_imaging_weight,
configuration=configuration,
phasecentre=phasecentre,
polarisation_frame=polarisation_frame,
source=source, meta=meta))
tab.close()
return vis_list
def __initData_WGS84(self):
"""Private function to generate a random set of data for writing a Measurements
file. The data is returned as a dictionary with keys:
* freq - frequency array in Hz
* site - observatory object
* stands - array of stand numbers
* bl - list of baseline pairs in real stand numbers
* vis - array of visibility data in baseline x freq format
"""
if run_ms_tests == False:
return
# Frequency range
freq = numpy.arange(0, 512) * 20e6 / 512 + 40e6
channel_width = numpy.full_like(freq, 20e6 / 512.)
# Site and stands
obs = EarthLocation(lon="+116.6356824", lat="-26.70130064", height=377.0)
names = numpy.array(['A%02d' % i for i in range(36)])
mount = numpy.array(['equat' for i in range(36)])
diameter = numpy.array([12 for i in range(36)])
xyz = numpy.array([[-175.233429, +1673.460938, 0.0000],
[+261.119019, +796.922119, 0.0000],
[-29.2005200, +744.432068, 0.0000],
[-289.355286, +586.936035, 0.0000],
[-157.031570, +815.570068, 0.0000],
[-521.311646, +754.674927, 0.0000],
[-1061.114258, +840.541443, 0.0000],
[-921.829407, +997.627686, 0.0000],
[-818.293579, +1142.272095, 0.0000],
[-531.752808, +850.726257, 0.0000],
[+81.352448, +790.245117, 0.0000],
[+131.126358, +1208.831909, 0.0000],
[-1164.709351, +316.779236, 0.0000],
[-686.248901, +590.285278, 0.0000],
[-498.987305, +506.338226, 0.0000],
[-182.249146, +365.113464, 0.0000],
[+420.841858, +811.081543, 0.0000],
[+804.107910, +1273.328369, 0.0000],
[-462.810394, +236.353790, 0.0000],
[-449.772339, +15.039755, 0.0000],
[+13.791821, +110.971809, 0.0000],
[-425.687317, -181.908752, 0.0000],
[-333.404053, -503.603394, 0.0000],
[-1495.472412, +1416.063232, 0.0000],
[-1038.578857, +1128.367920, 0.0000],
[-207.151749, +956.312561, 0.0000],
[-389.051880, +482.405670, 0.0000],
[-434.000000, +510.000000, 0.0000],
[-398.000000, +462.000000, 0.0000],
[-425.000000, +475.000000, 0.0000],
[-400.000000, +3664.000000, 0.0000],
[+1796.000000, +1468.000000, 0.0000],
[+2600.000000, -1532.000000, 0.0000],
[-400.000000, -2336.000000, 0.0000],
[-3400.00000, -1532.000000, 0.0000],
[-2596.000000, +1468.000000, 0.0000]])
site_config = Configuration(name='ASKAP', data=None, location=obs,
names=names, xyz=xyz, mount=mount, frame=None,
receptor_frame=ReceptorFrame("linear"),
diameter=diameter)
antennas = []
for i in range(len(names)):
antennas.append(Antenna(i, Stand(names[i], xyz[i, 0], xyz[i, 1], xyz[i, 2])))
# Set baselines and data
blList = []
N = len(antennas)
antennas2 = antennas
for i in range(0, N - 1):
for j in range(i + 1, N):
blList.append((antennas[i], antennas2[j]))
visData = numpy.random.rand(len(blList), len(freq))
visData = visData.astype(numpy.complex64)
return {'freq': freq, 'channel_width': channel_width, 'site': site_config, 'antennas': antennas, 'bl': blList,
'vis': visData}