def LoadHDF5(HDF5Filename, header=False):
try:
d = scape.DataSet(HDF5Filename, baseline=opts.baseline)
except ValueError:
print "WARNING:THIS FILE", HDF5Filename.split(
'/'
)[-1], "IS CORRUPTED AND SCAPE WILL NOT PROCESS IT, YOU MAY NEED TO REAUGMENT IT,BUT ITS AN EXPENSIVE TASK..!!"
else:
print "SUCCESSFULLY LOADED: Wellcome to scape Library and scape is busy processing your request"
lo_freq = 4200.0 + d.freqs[len(d.freqs) / 2.0]
# try to check all the rfi channels across all the channels
rfi_chan_across_all = d.identify_rfi_channels()
d = d.select(freqkeep=range(100, 420))
# rfi channels across fringe finder channels ( i.e frequancy range around 100 to 420)
rfi_channels = d.identify_rfi_channels()
freqs = d.freqs
sky_frequency = d.freqs[rfi_channels]
ant = d.antenna.name
data_filename = os.path.splitext(
os.path.basename(HDF5Filename))[0] + '.h5'
# obs_date = os.path.splitext(os.path.basename(HDF5Filename))[0]
#date = time.ctime(float(obs_date))
for compscan in d.compscans:
azimuth = np.hstack(
[scan.pointing['az'] for scan in compscan.scans])
elevation = np.hstack(
[scan.pointing['el'] for scan in compscan.scans])
compscan_times = np.hstack(
[scan.timestamps for scan in compscan.scans])
compscan_start_time = np.hstack(
[scan.timestamps[0] for scan in compscan.scans])
compscan_end_time = np.hstack(
[scan.timestamps[-1] for scan in compscan.scans])
middle_time = np.median(compscan_times, axis=None)
obs_date = katpoint.Timestamp(middle_time)
middle_start_time = np.median(compscan_start_time)
middle_end_time = np.median(compscan_end_time)
end_time = katpoint.Timestamp(middle_end_time)
min_compscan_az = katpoint.rad2deg(azimuth.min())
max_compscan_az = katpoint.rad2deg(azimuth.max())
min_compscan_el = katpoint.rad2deg(elevation.min())
max_compscan_el = katpoint.rad2deg(elevation.max())
start_time = katpoint.Timestamp(middle_start_time)
requested_azel = compscan.target.azel(middle_time)
#ant_az = katpoint.rad2deg(np.array(requested_azel[0]))
#ant_el = katpoint.rad2deg(np.array(requested_azel[1]))
target = compscan.target.name
f = file(opts.outfilebase + '.csv', 'a')
for index in range(0, len(rfi_channels)):
rfi_chan = rfi_channels[index] + 100
rfi_freq = freqs[rfi_channels[index]]
f.write('%s, %s, %s, %s, %s,%f, %f,%f,%f, %f, %d, %f\n' % (data_filename,start_time, end_time, ant,target,min_compscan_az,max_compscan_az,\
min_compscan_el, max_compscan_el,lo_freq, rfi_chan, rfi_freq))
f.close()
def remove_duplicates(sensor):
"""Remove duplicate timestamp values from sensor data.
This sorts the 'timestamp' field of the sensor record array and removes any
duplicate values, updating the corresponding 'value' and 'status' fields as
well. If more than one timestamp have the same value, the value and status
of the last of these timestamps are selected. If the values differ for the
same timestamp, a warning is logged (and the last one is still picked).
Parameters
----------
sensor : :class:`SensorData` object, length N
Sensor dataset, which acts like a record array with fields 'timestamp',
'value' and 'status'
Returns
-------
unique_sensor : :class:`SensorData` object, length M
Sensor data with duplicate timestamps removed (M <= N)
"""
x = np.atleast_1d(sensor['timestamp'])
y = np.atleast_1d(sensor['value'])
z = np.atleast_1d(sensor['status'])
# Sort x via mergesort, as it is usually already sorted and stability is important
sort_ind = np.argsort(x, kind='mergesort')
x, y = x[sort_ind], y[sort_ind]
# Array contains True where an x value is unique or the last of a run of identical x values
last_of_run = np.asarray(list(np.diff(x) != 0) + [True])
# Discard the False values, as they represent duplicates - simultaneously keep last of each run of duplicates
unique_ind = last_of_run.nonzero()[0]
# Determine the index of the x value chosen to represent each original x value (used to pick y values too)
replacement = unique_ind[len(unique_ind) -
np.cumsum(last_of_run[::-1])[::-1]]
# All duplicates should have the same y and z values - complain otherwise, but continue
if not np.all(y[replacement] == y) or not np.all(z[replacement] == z):
logger.debug(
"Sensor '%s' has duplicate timestamps with different values or statuses"
% (sensor.name, ))
for ind in (y[replacement] != y).nonzero()[0]:
logger.debug(
"At %s, sensor '%s' has values of %s and %s - keeping last one"
% (katpoint.Timestamp(
x[ind]).local(), sensor.name, y[ind], y[replacement][ind]))
for ind in (z[replacement] != z).nonzero()[0]:
logger.debug(
"At %s, sensor '%s' has statuses of '%s' and '%s' - keeping last one"
% (katpoint.Timestamp(
x[ind]).local(), sensor.name, z[ind], z[replacement][ind]))
return SensorData(
np.rec.fromarrays([x[unique_ind], y[unique_ind], z[unique_ind]],
dtype=[('timestamp', x.dtype), ('value', y.dtype),
('status', z.dtype)]), sensor.name)
def test_numerical_timestamp(self):
"""Test numerical properties of timestamps."""
t = katpoint.Timestamp(self.valid_timestamps[0][0])
self.assertEqual(t, t + 0.0)
self.assertNotEqual(t, t + 1.0)
self.assertTrue(t > t - 1.0)
self.assertTrue(t < t + 1.0)
self.assertEqual(t, eval('katpoint.' + repr(t)))
self.assertEqual(float(t), self.valid_timestamps[0][0])
t = katpoint.Timestamp(self.valid_timestamps[1][0])
self.assertAlmostEqual(t.to_ephem_date(),
self.valid_timestamps[1][0],
places=9)
def test_numerical_timestamp(self):
"""Test numerical properties of timestamps."""
t = katpoint.Timestamp(self.valid_timestamps[0][0])
self.assertEqual(t, t + 0.0)
self.assertNotEqual(t, t + 1.0)
self.assertTrue(t > t - 1.0)
self.assertTrue(t < t + 1.0)
self.assertEqual(t, eval('katpoint.' + repr(t)))
self.assertEqual(float(t), self.valid_timestamps[0][0])
t = katpoint.Timestamp(self.valid_timestamps[1][0])
self.assertAlmostEqual(t.to_ephem_date(), self.valid_timestamps[1][0], places=9)
try:
self.assertEqual(hash(t), hash(t + 0.0), 'Timestamp hashes not equal')
except TypeError:
self.fail('Timestamp object not hashable')
def test_construct_timestamp(self):
"""Test construction of timestamps."""
for v, s in self.valid_timestamps:
t = katpoint.Timestamp(v)
self.assertEqual(str(t), s, "Timestamp string ('%s') differs from expected one ('%s')" % (str(t), s))
for v in self.invalid_timestamps:
self.assertRaises(ValueError, katpoint.Timestamp, v)
def set_delay(time_after_now, delay=None):
t = katpoint.Timestamp() + time_after_now
if delay is None:
delay = cable2 - cable1 + tgt.geometric_delay(ant2, t, ant1)[0]
print delay
roach.req.poco_delay('0x', int(t.secs) * 1000, '%.9f' % (delay * 1000))
roach.req.poco_delay('0y', int(t.secs) * 1000, '%.9f' % (delay * 1000))
def plot_flags(timestamps, freqs, flags):
"""timestamps is an array of unix timstamps
freqs is an array of frequencys in MHz
flags is a 2D array of boolean with [time,freq]
it plots the percentage of the data flagged
as a function of time vs frequency """
fig = plt.figure(figsize=(10, 5))
#
print((timestamps[:] - timestamps[0]).shape, (freqs).shape,
((flags).T).shape)
plt.pcolormesh(timestamps[:] - timestamps[0],
freqs, (flags).T,
rasterized=True)
plt.title('Flags in "quiet" part of the band')
plt.xlabel('Time (s), since %s' %
(katpoint.Timestamp(timestamps[0]).local(), ))
plt.ylabel('Frequency/[MHz]')
plt.xlim(0, timestamps[-1] - timestamps[0])
plt.ylim(freqs[0], freqs[-1])
plt.figtext(0.89,
0.11,
git_info(),
horizontalalignment='right',
fontsize=10)
return fig
def _create_timestamps(self, timestamp_defs, dump_defs):
"""
Creates observation starting and ending times,
as well as the dump period, indices and timestamps
and number of dumps.
Parameters
----------
timestamp_defs : dict
Dictionary { 'start_time' : float, 'dump_period' : float}
dump_defs : list of tuples
List of (event, number of dumps, target) tuples
"""
try:
start_time = timestamp_defs['start_time']
except KeyError:
epoch = datetime.utcfromtimestamp(0)
start_time = (datetime.now() - epoch).total_seconds()
dump_period = timestamp_defs.get('dump_period', 4.0)
self._ndumps = ndumps = sum(nd for _, nd, _ in dump_defs)
# Observation times
self.start_time = katpoint.Timestamp(start_time)
self.end_time = self.start_time + dump_period * ndumps
self.dump_period = dump_period
self.dumps = np.arange(ndumps)
self._timestamps = np.arange(self.start_time.secs + (dump_period / 2.),
self.end_time.secs,
step=dump_period,
dtype=np.float64)
def get_lst_range(date):
date_timestamp = time.mktime(
date.timetuple()) # this will be local time
time_range = katpoint.Timestamp(date_timestamp).secs + numpy.arange(
0, 24.0 * 3600.0, 60)
lst_range = numpy.degrees(
target.antenna.local_sidereal_time(time_range)) / 15.0
return time_range, lst_range
def test_get_scan_area_extents_for_rising_target(self):
"""Test of function get_scan_area_extents with rising target."""
test_date = katpoint.Timestamp('2010/12/05 20:00:00').to_ephem_date()
el, az_min, az_max, t_start, t_end = self.get_scan_area_extents(test_date)
self.assertAlmostEqual(el, 0.39849024) # In radians
self.assertAlmostEqual(az_min, 2.06043911) # In radians
self.assertAlmostEqual(az_max, 2.25462604) # In radians
self.assertEqual(int(t_start.secs), 1291579227)
self.assertEqual(int(t_end.secs), 1291585208)
def test_get_scan_area_extents_for_setting_target(self):
"""Test of function get_scan_area_extents with setting target."""
test_date = katpoint.Timestamp('2010/12/05 02:00:00').to_ephem_date()
el, az_min, az_max, t_start, t_end = self.get_scan_area_extents(test_date)
self.assertAlmostEqual(el, 0.82988305) # In radians
self.assertAlmostEqual(az_min, 3.67187738) # In radians
self.assertAlmostEqual(az_max, 4.015025139) # In radians
self.assertEqual(int(t_start.secs), 1291514447)
self.assertEqual(int(t_end.secs), 1291522147)
def compscan_key(compscan):
"""List of strings that identifies compound scan."""
# Name of data set that contains compound scan
path = compscan.scans[0].path
filename_end = path.find('.h5')
dataset_name = os.path.basename(path[:filename_end]) if filename_end > 0 else os.path.basename(path)
# Time when compound scan is exactly half-way through its operation (i.e. 50% complete)
middle_time = np.median(np.hstack([scan.timestamps for scan in compscan.scans]), axis=None)
return compscan.dataset.antenna.name, dataset_name, compscan.target.name, str(katpoint.Timestamp(middle_time))
def setUp(self):
self.target1 = katpoint.construct_azel_target('45:00:00.0',
'75:00:00.0')
self.target2 = katpoint.Target('Sun, special')
self.ant1 = katpoint.Antenna('A1, -31.0, 18.0, 0.0, 12.0, 0.0 0.0 0.0')
self.ant2 = katpoint.Antenna(
'A2, -31.0, 18.0, 0.0, 12.0, 10.0 -10.0 0.0')
self.ant3 = katpoint.Antenna(
'A3, -31.0, 18.0, 0.0, 12.0, 5.0 10.0 3.0')
self.ts = katpoint.Timestamp('2013-08-14 08:25')
self.delays = katpoint.DelayCorrection([self.ant2, self.ant3],
self.ant1, 1.285e9)
def get_power_array(data, name, pol):
url = "http://portal.mkat.karoo.kat.ac.za/katstore/samples" # this is for live data
#url='http://kat-flap-cpt.mkat.control.kat.ac.za/katstore/samples' # flap in CT for older values
start = katpoint.Timestamp(np.str(data.start_time))
end = katpoint.Timestamp(np.str(data.end_time))
#print start, end
params = {
'sensor': '%s_dig_l_band_adc_%spol_rf_power_in' %
(name, pol), # name of the sensor.
'start': start.secs, # start time in seconds - float should work.
'end': end.secs, # end time in seconds
'time_type': 's'
} # Specify that we work in seconds.
res = requests.get(url, params) # Make the call.
res.json(
) # Data comes back as JSON with the correct mime type set so Requests can convert for you.
#data_dict = {r[0]: float(r[3]) for r in res.json()} # E.g. Making dict from the response. You'll prob do something different here.
data_temp_list = [
float(r[3]) for r in res.json()
] # E.g. Making list from the response. You'll prob do something different here.
return np.array(data_temp_list)
def test_operators(self):
"""Test operators defined for timestamps."""
T = katpoint.Timestamp(self.valid_timestamps[0][0])
S = T.secs
# Logical operators, float treated as absolute time
self.assertTrue(T == S)
self.assertTrue(T < S+1)
self.assertTrue(T > S-1)
# Arithmetic operators, float treated as interval
self.assertTrue(isinstance(T - S, katpoint.Timestamp))
self.assertTrue(isinstance(S - T, float))
self.assertTrue(isinstance(T - T, float))
def test_time_conversion_symmetry(self):
"""Test katpoint and astrokat time conversion methods match and are symmetrical"""
test_files = [
"test_obs/below-horizon-sim.yaml",
"test_obs/image-cals-sim.yaml",
"test_obs/image-sim.yaml",
"test_obs/image-single-sim.yaml",
"test_obs/targets-sim.yaml",
"test_obs/two-calib-sim.yaml",
]
for test_file in test_files:
file_path = yaml_path(test_file)
yaml_start_time = extract_start_time(file_path)
yaml_start_time_str = str(yaml_start_time)
astrokat_sec_since_epoch = utility.datetime2timestamp(
yaml_start_time)
katpoint_sec_since_epoch = katpoint.Timestamp(
yaml_start_time_str).secs
self.assertAlmostEqual(
astrokat_sec_since_epoch,
katpoint_sec_since_epoch,
places=6,
msg="timestamp conversion mismatch {}".format(test_file))
astrokat_datetime = utility.timestamp2datetime(
astrokat_sec_since_epoch)
katpoint_timestamp = katpoint.Timestamp(katpoint_sec_since_epoch)
self.assertEqual(
str(astrokat_datetime),
yaml_start_time_str,
msg="astrokat str time conversion mismatch for {}".format(
test_file))
self.assertEqual(
str(katpoint_timestamp),
yaml_start_time_str,
msg="katpoint str time conversion mismatch for {}".format(
test_file))
def phase_plot(phasedata, baseline_names, num_chans, scan_timestamps, num_bls):
"""phasedata is an array consiting of phase information for all correlation products,
num_chans is the number of channels"""
x1, x2 = scan_timestamps[0][0], scan_timestamps[-1][-1]
scan_freqinds = [np.arange(num_bls * num_chans)] * len(scan_timestamps)
y1, y2 = scan_freqinds[0][0] - 0.5, scan_freqinds[0][-1] + 0.5
#num_bls = len(baseline_names)
if num_bls > 0:
fig2 = plt.figure(0, figsize=(12, num_bls), tight_layout=True)
else:
fig2 = plt.figure(0, figsize=(12, 1), tight_layout=True)
#Put plots side-by-side if there are <10 scans.
p = 0
for p, active_pol in enumerate(['H', 'V']):
if len(scan_timestamps) < 10:
plt.subplot(1, 2, p + 1)
else:
plt.subplot(2, 1, p + 1)
plt.imshow(phasedata[active_pol]
[~np.all(np.isnan(phasedata[active_pol]), axis=1)],
aspect='auto',
origin='lower',
extent=[x1, x2, y1, y2],
interpolation='nearest',
rasterized=True)
plt.xlabel('Time (s), since %s' %
(katpoint.Timestamp(data.start_time).local(), ))
if p == 1 and len(scan_timestamps) < 10:
plt.yticks(
np.arange(num_chans // 2, num_bls * num_chans, num_chans),
np.repeat('', num_bls))
else:
plt.yticks(
np.arange(num_chans // 2, num_bls * num_chans, num_chans),
baseline_names)
for yval in range(0, num_bls * num_chans, num_chans):
plt.axhline(yval, color='k', lw=2)
plt.title('Raw visibility phase per baseline, Pol %s' % active_pol)
return fig2
# Get the average dump time for this scan (equal to scan length if the dump period is longer than a scan)
dump_time_width = min(time_av, scan_len * h5.dump_period)
scan_size_mb = 0.0
# Get UTC timestamps
utc_seconds = h5.timestamps[:]
# Update field lists if this is a new target
if target.name not in field_names:
# Since this will be an 'radec' target, we don't need antenna or timestamp to get the (astrometric) ra, dec
ra, dec = target.radec()
field_names.append(target.name)
field_centers.append((ra, dec))
field_times.append(
katpoint.Timestamp(utc_seconds[0]).to_mjd() * 60 * 60 * 24)
if options.verbose:
print "Added new field %d: '%s' %s %s" % (len(field_names) - 1,
target.name, ra, dec)
field_id = field_names.index(target.name)
# Determine the observation tag for this scan
obs_tag = []
for tag in target.tags:
if tag in tag_to_intent:
obs_tag.append(tag_to_intent[tag])
obs_tag = ','.join(obs_tag)
# add tag to obs_modes list
if obs_tag and obs_tag not in obs_modes:
obs_modes.append(obs_tag)
# get state_id from obs_modes list if it is in the list, else 0 'UNKNOWN'
def onselect_next(eclick,erelease):
global output_data, output_chan, output_ts
xmin = min(eclick.xdata, erelease.xdata)
xmax = max(eclick.xdata, erelease.xdata)
ymin = min(eclick.ydata, erelease.ydata)
ymax = max(eclick.ydata, erelease.ydata)
ind = (FF >= xmin) & (FF <= xmax) & (PP >= ymin) & (PP <= ymax)
selected_freq = FF[ind]
selected_amp = 10.0*np.log10(PP[ind])
selected_ts = TT[ind]
selected_az = AA[ind]
selected_el = EE[ind]
print "SUCCESSFUL, CLICK AND DRAG TO SELECT THE NEXT RFI CHANNELS OR NEXT TO LOAD NEW DATASET"
#sorting with increasng X_new
indices = np.lexsort(keys = (selected_ts, selected_freq))
for index in indices:
output_data.append([out_filename, selected_freq[index],selected_ts[index], katpoint.Timestamp(selected_ts[index]).local(), selected_az[index], selected_el[index], selected_amp[index]])
for point in output_data:
output_chan.append(point[1])
output_ts.append(point[2])
output_az.append(point[4])
output_el.append(point[5])
% (', '.join(nd_cycles), ', '.join(bf_ants)))
#tell dspsr to expect cal data
if float(cycle_length) > 0:
period = float(cycle_length)
freq = 1.0 / period
print "kat.ptuse_1.req.ptuse_cal_freq (" + data_product_id + ", " + beam_id + ", " + str(
freq) + ")"
reply = kat.ptuse_1.req.ptuse_cal_freq(data_product_id,
beam_id, freq)
print "kat.ptuse_1.req.ptuse_cal_freq returned " + str(reply)
# Temporary haxx to make sure that AP accepts the upcoming track request
time.sleep(2)
if opts.cal == 'flux':
timenow = katpoint.Timestamp()
sources = katpoint.Catalogue(add_specials=False)
user_logger.info('Performing flux calibration')
ra, dec = target.apparent_radec(timestamp=timenow)
targetName = target.name.replace(" ", "")
print targetName
target.name = targetName + '_O'
sources.add(target)
if opts.cal == 'fluxN':
timenow = katpoint.Timestamp()
sources = katpoint.Catalogue(add_specials=False)
user_logger.info('Performing flux calibration')
ra, dec = target.apparent_radec(timestamp=timenow)
'--no-delays',
action="store_true",
default=True,
help='Do not use delay tracking, and zero delays (default="%default")')
# Set default value for any option (both standard and experiment-specific options)
parser.set_defaults(description='Tipping Curve')
# Parse the command line
opts, args = parser.parse_args()
on_time = 15.0
with verify_and_connect(opts) as kat:
# Ensure that azimuth is in valid physical range of -185 to 275 degrees
if opts.az is None:
user_logger.info("No Azimuth selected , selecting clear Azimith")
timestamp = [
katpoint.Timestamp(time.time() + 1) for i in range(
int((np.arange(10.0, 90.1, opts.spacing).shape[0] *
(on_time + 20.0 + 1.0))))
]
#load the standard KAT sources ... similar to the SkyPlot of the katgui
observation_sources = kat.sources
source_az = []
for source in observation_sources.targets:
az, el = np.degrees(
source.azel(timestamp=timestamp)) # was rad2deg
az[az > 180] = az[az > 180] - 360
source_az += list(set(az[el > 5]))
source_az.sort()
gap = np.diff(source_az).argmax() + 1
opts.az = (source_az[gap] + source_az[gap + 1]) / 2.0
user_logger.info("Selecting Satillite clear Azimuth=%f" % (opts.az, ))
def ms_writer_process(work_queue, result_queue, options, antennas, cp_info,
ms_name, raw_vis_data, raw_weight_data, raw_flag_data):
"""
Function to be run in a separate process for writing to a Measurement Set.
The MS is assumed to have already been created with the appropriate
columns.
Incoming work is provided by submitting instances of :class:`QueueItem`
to `work_queue`. The `slot` indexes the first dimension of the shared
memory arrays. One may also submit an :class:`EndOfScan`, which will flush
to disk and return a :class:`ScanResult` through the `result_queue` (these
are not actually required to match katdal scans).
To terminate the process, submit ``None`` to `work_queue`.
If an exception occurs, it will be placed into `result_queue`, after which
work_queue items will be fetched and discarded until ``None`` is received.
When finished (either successfully or after an error), ``None`` is put in
`result_queue`.
Parameters
----------
work_queue : :class:`multiprocessing.Queue`
Incoming work. Note that this function gives no explicit indication
when it is done with a piece of work, so the queue capacity needs to
be bounded to prevent data races.
result_queue : :class:`multiprocessing.Queue`
Information about progress (see :class:`ScanResult`)
options : :class:`argparse.Namespace`
Command-line options to mvftoms
antennas : list of :class:`katpoint.Antenna`
Antennas (used to compute UVW coordinates)
cp_info : namedtuple
Correlation product info (see mvftoms.py)
ms_name : str
Name of the Measurement Set to write
raw_vis_data, raw_weight_data, raw_flag_data : :class:`RawArray`
Circular buffers for the data, with shape
(slots, time, baseline, channel, pol).
"""
none_seen = False
try:
vis_arrays = raw_vis_data.asarray()
weight_arrays = raw_weight_data.asarray()
flag_arrays = raw_flag_data.asarray()
scan_size = 0
tdiff = vis_arrays.shape[1]
nbl = vis_arrays.shape[2]
main_table = ms_extra.open_main(ms_name, verbose=options.verbose)
with contextlib.closing(main_table):
array_centre = katpoint.Antenna('',
*antennas[0].ref_position_wgs84)
baseline_vectors = np.array([
array_centre.baseline_toward(antenna) for antenna in antennas
])
while True:
item = work_queue.get()
if item is None:
none_seen = True
break
elif isinstance(item, EndOfScan):
main_table.flush(
) # Mostly to get realistic throughput stats
result_queue.put(ScanResult(scan_size))
scan_size = 0
else:
# Extract the slot, and flatten time and baseline into a single axis
new_shape = (-1, vis_arrays.shape[-2],
vis_arrays.shape[-1])
vis_data = vis_arrays[item.slot].reshape(new_shape)
weight_data = weight_arrays[item.slot].reshape(new_shape)
flag_data = flag_arrays[item.slot].reshape(new_shape)
# Iterate through baselines, computing UVW coordinates
# for a chunk of timesteps
uvw_basis = item.target.uvw_basis(item.time_utc,
array_centre)
# Axes in uvw_ant are antenna, axis (u/v/w), and time
uvw_ant = np.tensordot(baseline_vectors, uvw_basis,
([1], [1]))
# Permute to time, antenna, axis
uvw_ant = np.transpose(uvw_ant, (2, 0, 1))
# Compute baseline UVW coordinates from per-antenna coordinates.
# The sign convention matches `CASA`_, rather than the
# Measurement Set `definition`_.
# .. _CASA: https://casa.nrao.edu/Memos/CoordConvention.pdf
# .. _definition: https://casa.nrao.edu/Memos/229.html#SECTION00064000000000000000
uvw_coordinates = (
np.take(uvw_ant, cp_info.ant1_index, axis=1) -
np.take(uvw_ant, cp_info.ant2_index, axis=1))
# Flatten time and baseline axes together
uvw_coordinates = uvw_coordinates.reshape(-1, 3)
# Convert averaged UTC timestamps to MJD seconds.
# Blow time up to (ntime*nbl,)
out_mjd = np.asarray([
katpoint.Timestamp(t).to_mjd() * 24 * 60 * 60
for t in item.time_utc
])
out_mjd = np.broadcast_to(out_mjd[:, np.newaxis],
(tdiff, nbl)).ravel()
# Repeat antenna indices to (ntime*nbl,)
a1 = np.broadcast_to(cp_info.ant1_index[np.newaxis, :],
(tdiff, nbl)).ravel()
a2 = np.broadcast_to(cp_info.ant2_index[np.newaxis, :],
(tdiff, nbl)).ravel()
# Blow field ID up to (ntime*nbl,)
big_field_id = np.full((tdiff * nbl, ),
item.field_id,
dtype=np.int32)
big_state_id = np.full((tdiff * nbl, ),
item.state_id,
dtype=np.int32)
big_scan_itr = np.full((tdiff * nbl, ),
item.scan_itr,
dtype=np.int32)
# Setup model_data and corrected_data if required
model_data = None
corrected_data = None
if options.model_data:
# unity intensity zero phase model data set, same shape as vis_data
model_data = np.ones(vis_data.shape,
dtype=np.complex64)
# corrected data set copied from vis_data
corrected_data = vis_data
# Populate dictionary for write to MS
main_dict = ms_extra.populate_main_dict(
uvw_coordinates, vis_data, flag_data, out_mjd, a1, a2,
item.dump_time_width, big_field_id, big_state_id,
big_scan_itr, model_data, corrected_data)
# Write data to MS.
ms_extra.write_rows(main_table,
main_dict,
verbose=options.verbose)
# Calculate bytes written from the summed arrays in the dict
scan_size += sum(a.nbytes for a in main_dict.itervalues()
if isinstance(a, np.ndarray))
except Exception as error:
result_queue.put(error)
while not none_seen:
item = work_queue.get()
if item is None:
none_seen = True
finally:
result_queue.put(None)
def __init__(self,
filename,
ref_ant='',
time_offset=0.0,
mode='r',
quicklook=False,
keepdims=False,
**kwargs):
DataSet.__init__(self, filename, ref_ant, time_offset)
# Load file
self.file, self.version = H5DataV2._open(filename, mode)
f = self.file
# Load main HDF5 groups
data_group, sensors_group, config_group = f['Data'], f[
'MetaData/Sensors'], f['MetaData/Configuration']
markup_group = f['Markup']
# Get observation script parameters, with defaults
for k, v in config_group['Observation'].attrs.iteritems():
# For KAT-7 (v2.1) data, strip the 'script_' prefix from most parameters
k = k if self.version > '2.1' or k in (
'script_name', 'script_arguments') else k[7:]
self.obs_params[str(k)] = v
self.observer = self.obs_params.get('observer', '')
self.description = self.obs_params.get('description', '')
self.experiment_id = self.obs_params.get('experiment_id', '')
# Get script log from History group
self.obs_script_log = f['History/script_log'].value['log'].tolist()
# ------ Extract timestamps ------
self.dump_period = get_single_value(config_group['Correlator'],
'int_time')
# Obtain visibility data and timestamps
self._vis = data_group['correlator_data']
self._timestamps = data_group['timestamps']
num_dumps = len(self._timestamps)
if num_dumps != self._vis.shape[0]:
raise BrokenFile('Number of timestamps received from k7_capture '
'(%d) differs from number of dumps in data (%d)' %
(num_dumps, self._vis.shape[0]))
# Discard the last sample if the timestamp is a duplicate (caused by stop packet in k7_capture)
num_dumps = (num_dumps - 1) if num_dumps > 1 and (
self._timestamps[-1] == self._timestamps[-2]) else num_dumps
# Do quick test for uniform spacing of timestamps (necessary but not sufficient)
expected_dumps = (self._timestamps[num_dumps - 1] -
self._timestamps[0]) / self.dump_period + 1
# The expected_dumps should always be an integer (like num_dumps), unless the timestamps and/or dump period
# are messed up in the file, so the threshold of this test is a bit arbitrary (e.g. could use > 0.5)
irregular = abs(expected_dumps - num_dumps) >= 0.01
if irregular:
# Warn the user, as this is anomalous
logger.warning((
"Irregular timestamps detected in file '%s': "
"expected %.3f dumps based on dump period and start/end times, got %d instead"
) % (filename, expected_dumps, num_dumps))
if quicklook:
logger.warning(
"Quicklook option selected - partitioning data based on synthesised timestamps instead"
)
if not irregular or quicklook:
# Estimate timestamps by assuming they are uniformly spaced (much quicker than loading them from file).
# This is useful for the purpose of segmenting data set, where accurate timestamps are not that crucial.
# The real timestamps are still loaded when the user explicitly asks for them.
data_timestamps = self._timestamps[
0] + self.dump_period * np.arange(num_dumps)
else:
# Load the real timestamps instead (could take several seconds on a large data set)
data_timestamps = self._timestamps[:num_dumps]
# Move timestamps from start of each dump to the middle of the dump
data_timestamps += 0.5 * self.dump_period + self.time_offset
if data_timestamps[0] < 1e9:
logger.warning(
"File '%s' has invalid first correlator timestamp (%f)" % (
filename,
data_timestamps[0],
))
self._time_keep = np.ones(num_dumps, dtype=np.bool)
self.start_time = katpoint.Timestamp(data_timestamps[0] -
0.5 * self.dump_period)
self.end_time = katpoint.Timestamp(data_timestamps[-1] +
0.5 * self.dump_period)
self._keepdims = keepdims
# ------ Extract flags ------
# Check if flag group is present, else use dummy flag data
self._flags = markup_group['flags'] if 'flags' in markup_group else \
dummy_dataset('dummy_flags', shape=self._vis.shape[:-1], dtype=np.uint8, value=0)
# Obtain flag descriptions from file or recreate default flag description table
self._flags_description = markup_group['flags_description'] if 'flags_description' in markup_group else \
np.array(zip(FLAG_NAMES, FLAG_DESCRIPTIONS))
self._flags_select = np.array([0], dtype=np.uint8)
self._flags_keep = 'all'
# ------ Extract weights ------
# Check if weight group present, else use dummy weight data
self._weights = markup_group['weights'] if 'weights' in markup_group else \
dummy_dataset('dummy_weights', shape=self._vis.shape[:-1], dtype=np.float32, value=1.0)
# Obtain weight descriptions from file or recreate default weight description table
self._weights_description = markup_group['weights_description'] if 'weights_description' in markup_group else \
np.array(zip(WEIGHT_NAMES, WEIGHT_DESCRIPTIONS))
self._weights_select = []
self._weights_keep = 'all'
# ------ Extract sensors ------
# Populate sensor cache with all HDF5 datasets below sensor group that fit the description of a sensor
cache = {}
def register_sensor(name, obj):
"""A sensor is defined as a non-empty dataset with expected dtype."""
if isinstance(obj, h5py.Dataset) and obj.shape != () and \
obj.dtype.names == ('timestamp', 'value', 'status'):
# Rename pedestal sensors from the old regime to become sensors of the corresponding antenna
name = (
'Antennas/ant' +
name[13:]) if name.startswith('Pedestals/ped') else name
cache[name] = RecordSensorData(obj, name)
sensors_group.visititems(register_sensor)
# Use estimated data timestamps for now, to speed up data segmentation
self.sensor = SensorCache(cache,
data_timestamps,
self.dump_period,
keep=self._time_keep,
props=SENSOR_PROPS,
virtual=VIRTUAL_SENSORS,
aliases=SENSOR_ALIASES)
# ------ Extract subarrays ------
# By default, only pick antennas that were in use by the script
script_ants = config_group['Observation'].attrs['script_ants'].split(
',')
self.ref_ant = script_ants[0] if not ref_ant else ref_ant
# Original list of correlation products as pairs of input labels
corrprods = get_single_value(config_group['Correlator'],
'bls_ordering')
if len(corrprods) != self._vis.shape[2]:
# Apply k7_capture baseline mask after the fact, in the hope that it fixes correlation product mislabelling
corrprods = np.array([
cp for cp in corrprods
if cp[0][:-1] in script_ants and cp[1][:-1] in script_ants
])
# If there is still a mismatch between labels and data shape, file is considered broken (maybe bad labels?)
if len(corrprods) != self._vis.shape[2]:
raise BrokenFile(
'Number of baseline labels (containing expected antenna names) '
'received from correlator (%d) differs from number of baselines in data (%d)'
% (len(corrprods), self._vis.shape[2]))
else:
logger.warning(
'Reapplied k7_capture baseline mask to fix unexpected number of baseline labels'
)
# All antennas in configuration as katpoint Antenna objects
ants = [
katpoint.Antenna(
config_group['Antennas'][name].attrs['description'])
for name in config_group['Antennas']
]
self.subarrays = [Subarray(ants, corrprods)]
self.sensor['Observation/subarray'] = CategoricalData(
self.subarrays, [0, len(data_timestamps)])
self.sensor['Observation/subarray_index'] = CategoricalData(
[0], [0, len(data_timestamps)])
# Store antenna objects in sensor cache too, for use in virtual sensor calculations
for ant in ants:
self.sensor['Antennas/%s/antenna' %
(ant.name, )] = CategoricalData(
[ant], [0, len(data_timestamps)])
# ------ Extract spectral windows / frequencies ------
# Ideally we would like to use calculated center-frequency-hz sensor produced by k7_capture (better for nband)
if self.version >= '2.1':
centre_freq = self.sensor.get('RFE/center-frequency-hz')
else:
# Fall back to basic RFE7 LO frequency, as this supported multiple spectral windows before k7_capture did
# This assumes WBC mode, though (NBC modes only fully supported since HDF5 v2.1)
centre_freq = self.sensor.get('RFE/rfe7.lo1.frequency')
centre_freq.unique_values = [
freq - 4200e6 for freq in centre_freq.unique_values
]
num_chans = get_single_value(config_group['Correlator'], 'n_chans')
if num_chans != self._vis.shape[1]:
raise BrokenFile(
'Number of channels received from correlator '
'(%d) differs from number of channels in data (%d)' %
(num_chans, self._vis.shape[1]))
bandwidth = get_single_value(config_group['Correlator'], 'bandwidth')
channel_width = bandwidth / num_chans
try:
mode = self.sensor.get('DBE/dbe.mode').unique_values[0]
except (KeyError, IndexError):
# Guess the mode for version 2.0 files that haven't been re-augmented
mode = 'wbc' if num_chans <= 1024 else 'wbc8k' if bandwidth > 200e6 else 'nbc'
self.spectral_windows = [
SpectralWindow(spw_centre, channel_width, num_chans, mode)
for spw_centre in centre_freq.unique_values
]
self.sensor['Observation/spw'] = CategoricalData(
[self.spectral_windows[idx] for idx in centre_freq.indices],
centre_freq.events)
self.sensor['Observation/spw_index'] = CategoricalData(
centre_freq.indices, centre_freq.events)
# ------ Extract scans / compound scans / targets ------
# Use the activity sensor of reference antenna to partition the data set into scans (and to set their states)
scan = self.sensor.get('Antennas/%s/activity' % (self.ref_ant, ))
# If the antenna starts slewing on the second dump, incorporate the first dump into the slew too.
# This scenario typically occurs when the first target is only set after the first dump is received.
# The workaround avoids putting the first dump in a scan by itself, typically with an irrelevant target.
if len(scan) > 1 and scan.events[1] == 1 and scan[1] == 'slew':
scan.events, scan.indices = scan.events[1:], scan.indices[1:]
scan.events[0] = 0
# Use labels to partition the data set into compound scans
label = sensor_to_categorical(markup_group['labels']['timestamp'],
markup_group['labels']['label'],
data_timestamps, self.dump_period,
**SENSOR_PROPS['Observation/label'])
# Discard empty labels (typically found in raster scans, where first scan has proper label and rest are empty)
# However, if all labels are empty, keep them, otherwise whole data set will be one pathological compscan...
if len(label.unique_values) > 1:
label.remove('')
# Create duplicate scan events where labels are set during a scan (i.e. not at start of scan)
# ASSUMPTION: Number of scans >= number of labels (i.e. each label should introduce a new scan)
scan.add_unmatched(label.events)
self.sensor['Observation/scan_state'] = scan
self.sensor['Observation/scan_index'] = CategoricalData(
range(len(scan)), scan.events)
# Move proper label events onto the nearest scan start
# ASSUMPTION: Number of labels <= number of scans (i.e. only a single label allowed per scan)
label.align(scan.events)
# If one or more scans at start of data set have no corresponding label, add a default label for them
if label.events[0] > 0:
label.add(0, '')
self.sensor['Observation/label'] = label
self.sensor['Observation/compscan_index'] = CategoricalData(
range(len(label)), label.events)
# Use the target sensor of reference antenna to set the target for each scan
target = self.sensor.get('Antennas/%s/target' % (self.ref_ant, ))
# Move target events onto the nearest scan start
# ASSUMPTION: Number of targets <= number of scans (i.e. only a single target allowed per scan)
target.align(scan.events)
self.sensor['Observation/target'] = target
self.sensor['Observation/target_index'] = CategoricalData(
target.indices, target.events)
# Set up catalogue containing all targets in file, with reference antenna as default antenna
self.catalogue.add(target.unique_values)
self.catalogue.antenna = self.sensor['Antennas/%s/antenna' %
(self.ref_ant, )][0]
# Ensure that each target flux model spans all frequencies in data set if possible
self._fix_flux_freq_range()
# Avoid storing reference to self in transform closure below, as this hinders garbage collection
dump_period, time_offset = self.dump_period, self.time_offset
# Restore original (slow) timestamps so that subsequent sensors (e.g. pointing) will have accurate values
extract_time = LazyTransform(
'extract_time',
lambda t, keep: t + 0.5 * dump_period + time_offset)
self.sensor.timestamps = LazyIndexer(self._timestamps,
keep=slice(num_dumps),
transforms=[extract_time])
# Apply default selection and initialise all members that depend on selection in the process
self.select(spw=0, subarray=0, ants=script_ants)
def reduce_compscan_inf(h5,
channel_mask=None,
chunks=16,
return_raw=False,
use_weights=False,
compscan_index=None,
debug=False):
"""Break the band up into chunks"""
chunk_size = chunks
rfi_static_flags = np.tile(False, h5.shape[0])
if len(channel_mask) > 0:
pickle_file = open(channel_mask, "rb")
rfi_static_flags = pickle.load(pickle_file)
pickle_file.close()
gains_p = {}
stdv = {}
calibrated = False # placeholder for calibration
h5.select(compscans=compscan_index)
a = []
if len(h5.target_indices) > 1:
print("Warning multiple targets in the compscan")
for scan in h5.scans():
a.append(h5.target_indices[0])
target = h5.catalogue.targets[np.median(a).astype(
np.int)] # Majority Track
compscan_index = h5.compscan_indices[0]
#h5.select(targets=target,compscans=h5.compscan_indices[0]) # Majority Track in compscan
if not return_raw: # Calculate average target flux over entire band
flux_spectrum = h5.catalogue.targets[
h5.target_indices[0]].flux_density(h5.freqs) # include flags
average_flux = np.mean(
[flux for flux in flux_spectrum if not np.isnan(flux)])
temperature = np.mean(h5.temperature)
pressure = np.mean(h5.pressure)
humidity = np.mean(h5.humidity)
wind_speed = np.mean(h5.wind_speed)
wind_direction = np.degrees(
np.angle(np.mean(np.exp(
1j * np.radians(h5.wind_direction))))) # Vector Mean
sun = katpoint.Target('Sun, special')
# Calculate pointing offset
# Obtain middle timestamp of compound scan, where all pointing calculations are done
middle_time = np.median(h5.timestamps[:], axis=None)
# Start with requested (az, el) coordinates, as they apply at the middle time for a moving target
requested_azel = target.azel(middle_time)
# Correct for refraction, which becomes the requested value at input of pointing model
rc = katpoint.RefractionCorrection()
requested_azel = [
requested_azel[0],
rc.apply(requested_azel[1], temperature, pressure, humidity)
]
requested_azel = katpoint.rad2deg(np.array(requested_azel))
gaussian_centre = np.zeros((chunk_size * 2, 2, len(h5.ants)))
gaussian_centre_std = np.zeros((chunk_size * 2, 2, len(h5.ants)))
gaussian_width = np.zeros((chunk_size * 2, 2, len(h5.ants)))
gaussian_width_std = np.zeros((chunk_size * 2, 2, len(h5.ants)))
gaussian_height = np.zeros((chunk_size * 2, len(h5.ants)))
gaussian_height_std = np.zeros((chunk_size * 2, len(h5.ants)))
if debug: #debug_text
debug_text = []
line = []
line.append("#AntennaPol")
line.append("Target")
line.append("Freq(MHz)") #MHz
line.append("Centre Az")
line.append("Centre El")
line.append("Centre Az Std")
line.append("Centre El Std")
line.append("Centre Az Width")
line.append("Centre El Width")
line.append("Centre Az Width Std")
line.append("Centre El Width Std")
line.append("Height")
line.append("Height Std")
debug_text.append(','.join(line))
pols = ["H", "V"] # Put in logic for Intensity
for i, pol in enumerate(pols):
gains_p[pol] = []
pos = []
stdv[pol] = []
h5.select(pol=pol,
corrprods='cross',
ants=h5.antlist,
targets=target,
compscans=compscan_index)
h5.bls_lookup = calprocs.get_bls_lookup(h5.antlist, h5.corr_products)
for scan in h5.scans():
if scan[1] != 'track': continue
valid_index = activity(h5, state='track')
data = h5.vis[valid_index]
if data.shape[0] > 0: # need at least one data point
#g0 = np.ones(len(h5.ants),np.complex)
if use_weights:
weights = h5.weights[valid_index].mean(axis=0)
else:
weights = np.ones(data.shape[1:]).astype(np.float)
gains_p[pol].append(
calprocs.g_fit(data[:].mean(axis=0),
weights,
h5.bls_lookup,
refant=0))
stdv[pol].append(
np.ones(
(data.shape[0], data.shape[1],
len(h5.ants))).sum(axis=0)) #number of data points
# Get coords in (x(time,ants),y(time,ants) coords)
pos.append([
h5.target_x[valid_index, :].mean(axis=0),
h5.target_y[valid_index, :].mean(axis=0)
])
for ant in range(len(h5.ants)):
for chunk in range(chunks):
if np.array(pos).shape[
0] > 4: # Make sure there is enough data for a fit
freq = slice(chunk * (h5.shape[1] // chunks),
(chunk + 1) * (h5.shape[1] // chunks))
rfi = ~rfi_static_flags[freq]
fitobj = fit.GaussianFit(
np.array(pos)[:, :, ant].mean(axis=0), [1., 1.], 1)
x = np.column_stack(
(np.array(pos)[:, 0, ant], np.array(pos)[:, 1, ant]))
y = np.abs(
np.array(gains_p[pol])[:, freq, :][:, rfi,
ant]).mean(axis=1)
y_err = 1. / np.sqrt(
np.array(stdv[pol])[:, freq, :][:, rfi,
ant].sum(axis=1))
gaussian = fitobj.fit(x.T, y, y_err)
#Fitted beam center is in (x, y) coordinates, in projection centred on target
snr = np.abs(np.r_[gaussian.std / gaussian.std_std])
valid_fit = np.all(
np.isfinite(np.r_[gaussian.mean, gaussian.std_mean,
gaussian.std, gaussian.std_std,
gaussian.height, gaussian.std_height,
snr]))
theta = np.sqrt((gaussian.mean**2).sum(
)) # this is to see if the co-ord is out of range
#The valid fit is needed because I have no way of working out if the gain solution was ok.
if not valid_fit or np.any(
theta > np.pi
): # the checks to see if the fit is ok
gaussian_centre[chunk + i * chunk_size, :,
ant] = np.nan
gaussian_centre_std[chunk + i * chunk_size, :,
ant] = np.nan
gaussian_width[chunk + i * chunk_size, :, ant] = np.nan
gaussian_width_std[chunk + i * chunk_size, :,
ant] = np.nan
gaussian_height[chunk + i * chunk_size, ant] = np.nan
gaussian_height_std[chunk + i * chunk_size,
ant] = np.nan
else:
# Convert this offset back to spherical (az, el) coordinates
beam_center_azel = target.plane_to_sphere(
np.radians(gaussian.mean[0]),
np.radians(gaussian.mean[1]), middle_time)
# Now correct the measured (az, el) for refraction and then apply the old pointing model
# to get a "raw" measured (az, el) at the output of the pointing model
beam_center_azel = [
beam_center_azel[0],
rc.apply(beam_center_azel[1], temperature,
pressure, humidity)
]
beam_center_azel = h5.ants[ant].pointing_model.apply(
*beam_center_azel)
beam_center_azel = np.degrees(
np.array(beam_center_azel))
gaussian_centre[chunk + i * chunk_size, :,
ant] = beam_center_azel
gaussian_centre_std[chunk + i * chunk_size, :,
ant] = gaussian.std_mean
gaussian_width[chunk + i * chunk_size, :,
ant] = gaussian.std
gaussian_width_std[chunk + i * chunk_size, :,
ant] = gaussian.std_std
gaussian_height[chunk + i * chunk_size,
ant] = gaussian.height
gaussian_height_std[chunk + i * chunk_size,
ant] = gaussian.std_height
if return_raw:
return np.r_[gaussian_centre, gaussian_centre_std, gaussian_width,
gaussian_width_std, gaussian_height, gaussian_height_std]
else:
ant_pointing = {}
pols = ["HH", "VV", 'I']
pol_ind = {}
pol_ind['HH'] = np.arange(0.0 * chunk_size,
1.0 * chunk_size,
dtype=int)
pol_ind['VV'] = np.arange(1.0 * chunk_size,
2.0 * chunk_size,
dtype=int)
pol_ind['I'] = np.arange(0.0 * chunk_size, 2.0 * chunk_size, dtype=int)
for ant in range(len(h5.ants)):
h_pol = ~np.isnan(
gaussian_centre[pol_ind['HH'], :, ant]) & ~np.isnan(
1. / gaussian_centre_std[pol_ind['HH'], :, ant])
v_pol = ~np.isnan(
gaussian_centre[pol_ind['VV'], :, ant]) & ~np.isnan(
1. / gaussian_centre_std[pol_ind['VV'], :, ant])
valid_solutions = np.count_nonzero(
h_pol & v_pol
) # Note this is twice the number of solutions because of the Az & El parts
print("%i valid solutions out of %s for %s on %s at %s " %
(valid_solutions // 2, chunks, h5.ants[ant].name,
target.name, str(katpoint.Timestamp(middle_time))))
if debug: #debug_text
for pol_i, pol in enumerate(["H", "V"]):
for chunk in range(chunks * pol_i, chunks * (pol_i + 1)):
line = []
freq = h5.channel_freqs[slice(
chunk * (h5.shape[1] // chunks),
(chunk + 1) * (h5.shape[1] // chunks))].mean()
line.append(h5.ants[ant].name + pol)
line.append(target.name)
line.append(str(freq / 1e6)) #MHz
line.append(str(gaussian_centre[chunk, 0, ant]))
line.append(str(gaussian_centre[chunk, 1, ant]))
line.append(str(gaussian_centre_std[chunk, 0, ant]))
line.append(str(gaussian_centre_std[chunk, 1, ant]))
line.append(str(gaussian_width[chunk, 0, ant]))
line.append(str(gaussian_width[chunk, 1, ant]))
line.append(str(gaussian_width_std[chunk, 0, ant]))
line.append(str(gaussian_width_std[chunk, 1, ant]))
line.append(str(gaussian_height[chunk, ant]))
line.append(str(gaussian_height_std[chunk, ant]))
debug_text.append(','.join(line))
if valid_solutions // 2 > 0: # a bit overboard
name = h5.ants[ant].name
ant_pointing[name] = {}
ant_pointing[name]["antenna"] = h5.ants[ant].name
ant_pointing[name]["valid_solutions"] = valid_solutions
ant_pointing[name]["dataset"] = h5.name.split('/')[-1].split(
'.')[0]
ant_pointing[name]["target"] = target.name
ant_pointing[name]["timestamp_ut"] = str(
katpoint.Timestamp(middle_time))
ant_pointing[name][
"data_unit"] = 'Jy' if calibrated else 'counts'
ant_pointing[name]["frequency"] = h5.freqs.mean()
ant_pointing[name]["flux"] = average_flux
ant_pointing[name]["temperature"] = temperature
ant_pointing[name]["pressure"] = pressure
ant_pointing[name]["humidity"] = humidity
ant_pointing[name]["wind_speed"] = wind_speed
ant_pointing[name]["wind_direction"] = wind_direction
# work out the sun's angle
sun_azel = katpoint.rad2deg(
np.array(sun.azel(middle_time, antenna=h5.ants[ant])))
ant_pointing[name]["sun_az"] = sun_azel.tolist()[0]
ant_pointing[name]["sun_el"] = sun_azel.tolist()[1]
ant_pointing[name]["timestamp"] = middle_time.astype(int)
#Work out the Target position and the requested position
# Start with requested (az, el) coordinates, as they apply at the middle time for a moving target
requested_azel = target.azel(middle_time, antenna=h5.ants[ant])
# Correct for refraction, which becomes the requested value at input of pointing model
rc = katpoint.RefractionCorrection()
requested_azel = [
requested_azel[0],
rc.apply(requested_azel[1], temperature, pressure,
humidity)
]
requested_azel = katpoint.rad2deg(np.array(requested_azel))
target_azel = katpoint.rad2deg(
np.array(target.azel(middle_time, antenna=h5.ants[ant])))
ant_pointing[name]["azimuth"] = target_azel.tolist()[0]
ant_pointing[name]["elevation"] = target_azel.tolist()[1]
azel_beam = w_average(
gaussian_centre[pol_ind["I"], :, ant],
axis=0,
weights=1. / gaussian_centre_std[pol_ind["I"], :, ant]**2)
# Make sure the offset is a small angle around 0 degrees
offset_azel = katpoint.wrap_angle(azel_beam - requested_azel,
360.)
ant_pointing[name]["delta_azimuth"] = offset_azel.tolist()[0]
ant_pointing[name]["delta_elevation"] = offset_azel.tolist()[1]
ant_pointing[name]["delta_elevation_std"] = 0.0 #calc
ant_pointing[name]["delta_azimuth_std"] = 0.0 #calc
for pol in pol_ind:
ant_pointing[name]["beam_height_%s" % (pol)] = w_average(
gaussian_height[pol_ind[pol], ant],
axis=0,
weights=1. / gaussian_height_std[pol_ind[pol], ant]**2)
ant_pointing[name]["beam_height_%s_std" % (pol)] = np.sqrt(
np.nansum(1. /
gaussian_height_std[pol_ind[pol], ant]**2))
ant_pointing[name]["beam_width_%s" % (pol)] = w_average(
gaussian_width[pol_ind[pol], :, ant],
axis=0,
weights=1. /
gaussian_width_std[pol_ind[pol], :, ant]**2).mean()
ant_pointing[name]["beam_width_%s_std" % (pol)] = np.sqrt(
np.nansum(1. /
gaussian_width_std[pol_ind[pol], :, ant]**2))
ant_pointing[name]["baseline_height_%s" % (pol)] = 0.0
ant_pointing[name]["baseline_height_%s_std" % (pol)] = 0.0
ant_pointing[name][
"refined_%s" %
(pol)] = 5.0 # I don't know what this means
ant_pointing[name]["azimuth_%s" % (pol)] = w_average(
gaussian_centre[pol_ind[pol], 0, ant],
axis=0,
weights=1. /
gaussian_centre_std[pol_ind[pol], 0, ant]**2)
ant_pointing[name]["elevation_%s" % (pol)] = w_average(
gaussian_centre[pol_ind[pol], 1, ant],
axis=0,
weights=1. /
gaussian_centre_std[pol_ind[pol], 1, ant]**2)
ant_pointing[name]["azimuth_%s_std" % (pol)] = np.sqrt(
np.nansum(
1. / gaussian_centre_std[pol_ind[pol], 0, ant]**2))
ant_pointing[name]["elevation_%s_std" % (pol)] = np.sqrt(
np.nansum(
1. / gaussian_centre_std[pol_ind[pol], 1, ant]**2))
else:
print("No (%i) solutions for %s on %s at %s " %
(valid_solutions, h5.ants[ant].name, target.name,
str(katpoint.Timestamp(middle_time))))
if debug: #debug_text
debug_text.append('')
base = "%s_%s" % (h5.name.split('/')[-1].split('.')[0],
"interferometric_pointing_DEBUG")
g = file('%s:Scan%i:%s' % (base, compscan_index, target.name), 'w')
g.write("\n".join(debug_text))
g.close()
return ant_pointing
def plot_timeseries(num_bls, baseline_names, scan_az, scan_el, scan_timestamps,
scan_vis, scan_freqinds, scan_autos):
"""num_bls is the number of baselines in the array, baseline_names is a list of all the correlation product
in active_pols, scan_az is a list of all the scan in azemuth, scan_el is the list of all the scans in elevation
,scan_timestamps is a list of timestamps for each each scan,scan_autos is a list amplitude for each antenna.
"""
#plot timeseries,spectrograph and spectrum for each antenna
#for p,active_pol in enumerate(['H','V']):
print 'active_pol', active_pol
for i in range(num_ants):
print 'Antenna name', data.corr_products[i][0], i
ant_name = data.corr_products[i][0]
amp = np.vstack([scan[:, :, i] for scan in scan_autos])
az = np.hstack([a[i] for a in scan_az])
el = np.hstack([e[i] for e in scan_el])
fig, [[ax1, ax2], [ax3, ax4]] = plt.subplots(2, 2, figsize=(12, 8))
fig.suptitle('Ant: %s' % (ant_name), fontsize=16)
fig.subplots_adjust(top=0.95)
ax1.errorbar(np.hstack(scan_timestamps),
np.nanmean(amp, axis=1),
np.nanstd(amp, axis=1),
color='grey')
ax1.fill_between(np.hstack(scan_timestamps),
np.nanmin(amp, axis=1),
np.nanmax(amp, axis=1),
color='lightgrey')
ax1.plot(np.hstack(scan_timestamps),
np.nanmean(amp, axis=1),
color='b')
ax1.set_title('Time series')
ax1.set_xlabel('Time (s), since %s' %
(katpoint.Timestamp(data.start_time).local(), ))
ax1.set_ylabel('Correlator counts')
ax1.set_ylim(0, 4000)
ax1.set_xlim(scan_timestamps[0][0], scan_timestamps[-1][-1])
for t in scan_timestamps:
ax1.axvline(t[0],
linestyle='dotted',
color='mistyrose',
linewidth=1)
ma_amp = np.ma.masked_where(np.isnan(amp.T), amp.T)
ax2.pcolormesh(np.hstack(scan_timestamps), data.channel_freqs / 1e6,
10 * np.log10(ma_amp))
ax2.set_title('Spectrogram')
ax2.set_xlabel('Time (s), since %s' %
(katpoint.Timestamp(data.start_time).local(), ))
ax2.set_xlim(scan_timestamps[0][0], scan_timestamps[-1][-1])
ax2.set_ylabel('Frequency/[MHz]')
ax2.set_ylim(data.channel_freqs[0] / 1e6, data.channel_freqs[-1] / 1e6)
ax3.errorbar(data.channel_freqs / 1e6,
np.nanmean(amp, axis=0),
np.nanstd(amp, axis=0),
color='grey')
ax3.fill_between(data.channel_freqs / 1e6,
np.nanmin(amp, axis=0),
np.nanmax(amp, axis=0),
color='lightgrey')
ax3.plot(data.channel_freqs / 1e6, np.nanmean(amp, axis=0), color='b')
ax3.set_title('Spectrum')
ax3.set_xlabel('Frequency/[MHz]')
ax3.set_xlim(data.channel_freqs[0] / 1e6, data.channel_freqs[-1] / 1e6)
ax3.set_ylabel('Correlator counts')
ax3.set_ylim(0, 4000)
ax4.plot(az, el, "*")
ax4.set_title('Pointing')
ax4.set_xlabel('Azimuth/[Degrees]')
ax4.set_ylabel('Elevation/[Degrees]')
return fig
def main():
tag_to_intent = {
'gaincal': 'CALIBRATE_PHASE,CALIBRATE_AMPLI',
'bpcal': 'CALIBRATE_BANDPASS,CALIBRATE_FLUX',
'target': 'TARGET'
}
usage = "%prog [options] <dataset> [<dataset2>]*"
description = "Convert MVF dataset(s) to CASA MeasurementSet. The datasets may " \
"be local filenames or archive URLs (including access tokens). " \
"If there are multiple datasets they will be concatenated via " \
"katdal before conversion."
parser = optparse.OptionParser(usage=usage, description=description)
parser.add_option("-o",
"--output-ms",
default=None,
help="Name of output MeasurementSet")
parser.add_option(
"-c",
"--circular",
action="store_true",
default=False,
help="Produce quad circular polarisation. (RR, RL, LR, LL) "
"*** Currently just relabels the linear pols ****")
parser.add_option(
"-r",
"--ref-ant",
help="Override the reference antenna used to pick targets "
"and scans (default is the 'array' antenna in MVFv4 "
"and the first antenna in older formats)")
parser.add_option("-t",
"--tar",
action="store_true",
default=False,
help="Tar-ball the MS")
parser.add_option(
"-f",
"--full_pol",
action="store_true",
default=False,
help="Produce a full polarisation MS in CASA canonical order "
"(HH, HV, VH, VV). Default is to produce HH,VV only.")
parser.add_option("-v",
"--verbose",
action="store_true",
default=False,
help="More verbose progress information")
parser.add_option("-w",
"--stop-w",
action="store_true",
default=False,
help="Use W term to stop fringes for each baseline")
parser.add_option("-p",
"--pols-to-use",
default=None,
help="Select polarisation products to include in MS as "
"comma-separated list (from: HH, HV, VH, VV). "
"Default is all available from (HH, VV).")
parser.add_option(
"-u",
"--uvfits",
action="store_true",
default=False,
help="Print command to convert MS to miriad uvfits in casapy")
parser.add_option("-a",
"--no-auto",
action="store_true",
default=False,
help="MeasurementSet will exclude autocorrelation data")
parser.add_option(
"-s",
"--keep-spaces",
action="store_true",
default=False,
help="Keep spaces in source names, default removes spaces")
parser.add_option(
"-C",
"--channel-range",
help="Range of frequency channels to keep (zero-based inclusive "
"'first_chan,last_chan', default is all channels)")
parser.add_option("-e",
"--elevation-range",
help="Flag elevations outside the range "
"'lowest_elevation,highest_elevation'")
parser.add_option(
"-m",
"--model-data",
action="store_true",
default=False,
help="Add MODEL_DATA and CORRECTED_DATA columns to the MS. "
"MODEL_DATA initialised to unity amplitude zero phase, "
"CORRECTED_DATA initialised to DATA.")
flag_names = ', '.join(name for name in FLAG_NAMES
if not name.startswith('reserved'))
parser.add_option("--flags",
default="all",
help="List of online flags to apply "
"(from " + flag_names + ") "
"default is all flags, '' will apply no flags)")
parser.add_option("--dumptime",
type=float,
default=0.0,
help="Output time averaging interval in seconds, "
"default is no averaging")
parser.add_option("--chanbin",
type=int,
default=0,
help="Bin width for channel averaging in channels, "
"default is no averaging")
parser.add_option(
"--flagav",
action="store_true",
default=False,
help="If a single element in an averaging bin is flagged, "
"flag the averaged bin")
parser.add_option("--caltables",
action="store_true",
default=False,
help="Create calibration tables from gain solutions in "
"the dataset (if present)")
parser.add_option("--quack",
type=int,
default=1,
metavar='N',
help="Discard the first N dumps "
"(which are frequently incomplete)")
parser.add_option("--applycal",
default="",
help="List of calibration solutions to apply to data as "
"a string of comma-separated names, e.g. 'l1' or "
"'K,B,G'. Use 'default' for L1 + L2 and 'all' for "
"all available products.")
(options, args) = parser.parse_args()
# Loading is I/O-bound, so give more threads than CPUs
dask.config.set(
pool=multiprocessing.pool.ThreadPool(4 * multiprocessing.cpu_count()))
if len(args) < 1:
parser.print_help()
raise RuntimeError(
"Please provide one or more MVF dataset names as arguments")
if options.elevation_range and len(options.elevation_range.split(',')) < 2:
raise RuntimeError(
"You have selected elevation flagging. Please provide elevation "
"limits in the form 'lowest_elevation,highest_elevation'.")
if len(args) > 1:
print("Concatenating multiple datasets into single MS.")
def antenna_indices(na, no_auto_corr):
"""Get default antenna1 and antenna2 arrays."""
return np.triu_indices(na, 1 if no_auto_corr else 0)
def corrprod_index(dataset):
"""The correlator product index (with -1 representing missing indices)."""
corrprod_to_index = {
tuple(cp): n
for n, cp in enumerate(dataset.corr_products)
}
# ==========================================
# Generate per-baseline antenna pairs and
# correlator product indices
# ==========================================
def _cp_index(a1, a2, pol):
"""Create correlator product index from antenna pair and pol."""
a1 = a1.name + pol[0].lower()
a2 = a2.name + pol[1].lower()
return corrprod_to_index.get((a1, a2), -1)
# Generate baseline antenna pairs
ant1_index, ant2_index = antenna_indices(len(dataset.ants),
options.no_auto)
# Order as similarly to the input as possible, which gives better performance
# in permute_baselines.
bl_indices = list(zip(ant1_index, ant2_index))
bl_indices.sort(key=lambda ants: _cp_index(dataset.ants[ants[
0]], dataset.ants[ants[1]], pols_to_use[0]))
# Undo the zip
ant1_index[:] = [bl[0] for bl in bl_indices]
ant2_index[:] = [bl[1] for bl in bl_indices]
ant1 = [dataset.ants[a1] for a1 in ant1_index]
ant2 = [dataset.ants[a2] for a2 in ant2_index]
# Create actual correlator product index
cp_index = [
_cp_index(a1, a2, p) for a1, a2 in zip(ant1, ant2)
for p in pols_to_use
]
cp_index = np.array(cp_index, dtype=np.int32)
CPInfo = namedtuple(
"CPInfo", ["ant1_index", "ant2_index", "ant1", "ant2", "cp_index"])
return CPInfo(ant1_index, ant2_index, ant1, ant2, cp_index)
# Open dataset
open_args = args[0] if len(args) == 1 else args
# katdal can handle a list of datasets, which get virtually concatenated internally
dataset = katdal.open(open_args,
ref_ant=options.ref_ant,
applycal=options.applycal)
if dataset.applycal_products:
print('The following calibration products will be applied:',
', '.join(dataset.applycal_products))
else:
print('No calibration products will be applied')
# Get list of unique polarisation products in the dataset
pols_in_dataset = np.unique([(cp[0][-1] + cp[1][-1]).upper()
for cp in dataset.corr_products])
# Which polarisation do we want to write into the MS
# select all possible pols if full-pol selected, otherwise the selected polarisations via pols_to_use
# otherwise finally select any of HH,VV present (the default).
pols_to_use = ['HH', 'HV', 'VH', 'VV'] if (options.full_pol or options.circular) else \
list(np.unique(options.pols_to_use.split(','))) if options.pols_to_use else \
[pol for pol in ['HH', 'VV'] if pol in pols_in_dataset]
# Check we have the chosen polarisations
if np.any([pol not in pols_in_dataset for pol in pols_to_use]):
raise RuntimeError(
f"Selected polarisation(s): {', '.join(pols_to_use)} not available. "
f"Available polarisation(s): {','.join(pols_in_dataset)}")
# Set full_pol if this is selected via options.pols_to_use
if set(pols_to_use) == {'HH', 'HV', 'VH', 'VV'} and not options.circular:
options.full_pol = True
# Extract one MS per spectral window in the dataset(s)
for win in range(len(dataset.spectral_windows)):
dataset.select(reset='T')
centre_freq = dataset.spectral_windows[win].centre_freq
print(
f'Extract MS for spw {win}: centre frequency {int(centre_freq)} Hz'
)
# If no output MS directory name supplied, infer it from dataset(s)
if options.output_ms is None:
if len(dataset.spectral_windows) > 1:
# Use frequency label to disambiguate multiple spectral windows
ms_name = default_ms_name(args, centre_freq)
else:
ms_name = default_ms_name(args)
else:
ms_name = options.output_ms
basename = os.path.splitext(ms_name)[0]
# Discard first N dumps which are frequently incomplete
dataset.select(spw=win,
scans='track',
flags=options.flags,
dumps=slice(options.quack, None))
# The first step is to copy the blank template MS to our desired output
# (making sure it's not already there)
if os.path.exists(ms_name):
raise RuntimeError(
f"MS '{ms_name}' already exists - please remove it "
"before running this script")
print("Will create MS output in " + ms_name)
# Instructions to flag by elevation if requested
if options.elevation_range is not None:
emin, emax = options.elevation_range.split(',')
print(
"\nThe MS can be flagged by elevation in casapy v3.4.0 or higher, with the command:"
)
print(
f" tflagdata(vis='{ms_name}', mode='elevation', lowerlimit={emin}, "
f"upperlimit={emax}, action='apply')\n")
# Instructions to create uvfits file if requested
if options.uvfits:
uv_name = basename + ".uvfits"
print(
"\nThe MS can be converted into a uvfits file in casapy, with the command:"
)
print(
f" exportuvfits(vis='{ms_name}', fitsfile='{uv_name}', datacolumn='data')\n"
)
if options.full_pol:
print(
"\n#### Producing a full polarisation MS (HH,HV,VH,VV) ####\n")
else:
print(
f"\n#### Producing MS with {','.join(pols_to_use)} polarisation(s) ####\n"
)
# if fringe stopping is requested, check that it has not already been done in hardware
if options.stop_w:
print(
"W term in UVW coordinates will be used to stop the fringes.")
try:
autodelay = [
int(ad) for ad in dataset.sensor['DBE/auto-delay']
]
if all(autodelay):
print("Fringe-stopping already performed in hardware... "
"do you really want to stop the fringes here?")
except KeyError:
pass
# Select frequency channel range
if options.channel_range is not None:
channel_range = [
int(chan_str) for chan_str in options.channel_range.split(',')
]
first_chan, last_chan = channel_range[0], channel_range[1]
if (first_chan < 0) or (last_chan >= dataset.shape[1]):
raise RuntimeError(
"Requested channel range outside data set boundaries. "
f"Set channels in the range [0,{dataset.shape[1] - 1}]")
if first_chan > last_chan:
raise RuntimeError(
f"First channel ({first_chan}) bigger than last channel "
f"({last_chan}) - did you mean it the other way around?")
chan_range = slice(first_chan, last_chan + 1)
print(f"\nChannel range {first_chan} through {last_chan}.")
dataset.select(channels=chan_range)
# Are we averaging?
average_data = False
# Determine the number of channels
nchan = len(dataset.channels)
# Work out channel average and frequency increment
if options.chanbin > 1:
average_data = True
# Check how many channels we are dropping
chan_remainder = nchan % options.chanbin
avg_nchan = int(nchan / min(nchan, options.chanbin))
print(
f"Averaging {options.chanbin} channels, output ms will have {avg_nchan} channels."
)
if chan_remainder > 0:
print(
f"The last {chan_remainder} channels in the data will be dropped "
f"during averaging ({options.chanbin} does not divide {nchan})."
)
chan_av = options.chanbin
nchan = avg_nchan
else:
# No averaging in channel
chan_av = 1
# Get the frequency increment per averaged channel
channel_freq_width = dataset.channel_width * chan_av
# Work out dump average and dump increment
# Is the desired time bin greater than the dump period?
if options.dumptime > dataset.dump_period:
average_data = True
dump_av = int(np.round(options.dumptime / dataset.dump_period))
time_av = dump_av * dataset.dump_period
print(
f"Averaging {dataset.dump_period} second dumps to {time_av} seconds."
)
else:
# No averaging in time
dump_av = 1
time_av = dataset.dump_period
# Print a message if extending flags to averaging bins.
if average_data and options.flagav and options.flags != '':
print("Extending flags to averaging bins.")
# Optionally keep only cross-correlation products
if options.no_auto:
dataset.select(corrprods='cross')
print("\nCross-correlations only.")
print(
f"\nUsing {dataset.ref_ant} as the reference antenna. All targets and scans "
"will be based on this antenna.\n")
# MS expects timestamps in MJD seconds
start_time = dataset.start_time.to_mjd() * 24 * 60 * 60
end_time = dataset.end_time.to_mjd() * 24 * 60 * 60
# MVF version 1 and 2 datasets are KAT-7; the rest are MeerKAT
telescope_name = 'KAT-7' if dataset.version[0] in '12' else 'MeerKAT'
# increment scans sequentially in the ms
scan_itr = 1
print("\nIterating through scans in dataset(s)...\n")
cp_info = corrprod_index(dataset)
nbl = cp_info.ant1_index.size
npol = len(pols_to_use)
field_names, field_centers, field_times = [], [], []
obs_modes = ['UNKNOWN']
total_size = 0
# Create the MeasurementSet
table_desc, dminfo = ms_extra.kat_ms_desc_and_dminfo(
nbl=nbl, nchan=nchan, ncorr=npol, model_data=options.model_data)
ms_extra.create_ms(ms_name, table_desc, dminfo)
ms_dict = {}
ms_dict['ANTENNA'] = ms_extra.populate_antenna_dict(
[ant.name for ant in dataset.ants],
[ant.position_ecef
for ant in dataset.ants], [ant.diameter for ant in dataset.ants])
ms_dict['FEED'] = ms_extra.populate_feed_dict(len(dataset.ants),
num_receptors_per_feed=2)
ms_dict['DATA_DESCRIPTION'] = ms_extra.populate_data_description_dict()
ms_dict['POLARIZATION'] = ms_extra.populate_polarization_dict(
ms_pols=pols_to_use, circular=options.circular)
ms_dict['OBSERVATION'] = ms_extra.populate_observation_dict(
start_time, end_time, telescope_name, dataset.observer,
dataset.experiment_id)
# before resetting ms_dict, copy subset to caltable dictionary
if options.caltables:
caltable_dict = {}
caltable_dict['ANTENNA'] = ms_dict['ANTENNA']
caltable_dict['OBSERVATION'] = ms_dict['OBSERVATION']
print("Writing static meta data...")
ms_extra.write_dict(ms_dict, ms_name, verbose=options.verbose)
# Pre-allocate memory buffers
tsize = dump_av
in_chunk_shape = (tsize, ) + dataset.shape[1:]
scan_vis_data = np.empty(in_chunk_shape, dataset.vis.dtype)
scan_weight_data = np.empty(in_chunk_shape, dataset.weights.dtype)
scan_flag_data = np.empty(in_chunk_shape, dataset.flags.dtype)
ms_chunk_shape = (SLOTS, tsize // dump_av, nbl, nchan, npol)
raw_vis_data = ms_async.RawArray(ms_chunk_shape, scan_vis_data.dtype)
raw_weight_data = ms_async.RawArray(ms_chunk_shape,
scan_weight_data.dtype)
raw_flag_data = ms_async.RawArray(ms_chunk_shape, scan_flag_data.dtype)
ms_vis_data = raw_vis_data.asarray()
ms_weight_data = raw_weight_data.asarray()
ms_flag_data = raw_flag_data.asarray()
# Need to limit the queue to prevent overwriting slots before they've
# been processed. The -2 allows for the one we're writing and the one
# the writer process is reading.
work_queue = multiprocessing.Queue(maxsize=SLOTS - 2)
result_queue = multiprocessing.Queue()
writer_process = multiprocessing.Process(
target=ms_async.ms_writer_process,
args=(work_queue, result_queue, options, dataset.ants, cp_info,
ms_name, raw_vis_data, raw_weight_data, raw_flag_data))
writer_process.start()
try:
slot = 0
for scan_ind, scan_state, target in dataset.scans():
s = time.time()
scan_len = dataset.shape[0]
prefix = f'scan {scan_ind:3d} ({scan_len:4d} samples)'
if scan_state != 'track':
if options.verbose:
print(f"{prefix} skipped '{scan_state}' - not a track")
continue
if scan_len < 2:
if options.verbose:
print(f'{prefix} skipped - too short')
continue
if target.body_type != 'radec':
if options.verbose:
print(
f"{prefix} skipped - target '{target.name}' not RADEC"
)
continue
print(
f"{prefix} loaded. Target: '{target.name}'. Writing to disk..."
)
# Get the average dump time for this scan (equal to scan length
# if the dump period is longer than a scan)
dump_time_width = min(time_av, scan_len * dataset.dump_period)
# Get UTC timestamps
utc_seconds = dataset.timestamps[:]
# Update field lists if this is a new target
if target.name not in field_names:
# Since this will be an 'radec' target, we don't need antenna
# or timestamp to get the (astrometric) ra, dec
ra, dec = target.radec()
field_names.append(target.name)
field_centers.append((ra, dec))
field_times.append(
katpoint.Timestamp(utc_seconds[0]).to_mjd() * 60 * 60 *
24)
if options.verbose:
print(
f"Added new field {len(field_names) - 1}: '{target.name}' {ra} {dec}"
)
field_id = field_names.index(target.name)
# Determine the observation tag for this scan
obs_tag = ','.join(tag_to_intent[tag] for tag in target.tags
if tag in tag_to_intent)
# add tag to obs_modes list
if obs_tag and obs_tag not in obs_modes:
obs_modes.append(obs_tag)
# get state_id from obs_modes list if it is in the list, else 0 'UNKNOWN'
state_id = obs_modes.index(
obs_tag) if obs_tag in obs_modes else 0
# Iterate over time in some multiple of dump average
ntime = utc_seconds.size
ntime_av = 0
for ltime in range(0, ntime - tsize + 1, tsize):
utime = ltime + tsize
tdiff = utime - ltime
out_freqs = dataset.channel_freqs
# load all visibility, weight and flag data
# for this scan's timestamps.
# Ordered (ntime, nchan, nbl*npol)
load(dataset, np.s_[ltime:utime, :, :], scan_vis_data,
scan_weight_data, scan_flag_data)
# This are updated as we go to point to the current storage
vis_data = scan_vis_data
weight_data = scan_weight_data
flag_data = scan_flag_data
out_utc = utc_seconds[ltime:utime]
# Overwrite the input visibilities with averaged visibilities,
# flags, weights, timestamps, channel freqs
if average_data:
vis_data, weight_data, flag_data, out_utc, out_freqs = \
averager.average_visibilities(vis_data, weight_data, flag_data,
out_utc, out_freqs, timeav=dump_av,
chanav=chan_av, flagav=options.flagav)
# Infer new time dimension from averaged data
tdiff = vis_data.shape[0]
# Select correlator products and permute axes
cp_index = cp_info.cp_index.reshape((nbl, npol))
vis_data, weight_data, flag_data = permute_baselines(
vis_data, weight_data, flag_data, cp_index,
ms_vis_data[slot], ms_weight_data[slot],
ms_flag_data[slot])
# Increment the number of averaged dumps
ntime_av += tdiff
# Check if writer process has crashed and abort if so
try:
result = result_queue.get_nowait()
raise result
except queue.Empty:
pass
work_queue.put(
ms_async.QueueItem(slot=slot,
target=target,
time_utc=out_utc,
dump_time_width=dump_time_width,
field_id=field_id,
state_id=state_id,
scan_itr=scan_itr))
slot += 1
if slot == SLOTS:
slot = 0
work_queue.put(ms_async.EndOfScan())
result = result_queue.get()
if isinstance(result, Exception):
raise result
scan_size = result.scan_size
s1 = time.time() - s
if average_data and utc_seconds.shape != ntime_av:
print(
f'Averaged {np.shape(utc_seconds)[0]} x {dataset.dump_period} second dumps '
f'to {ntime_av} x {dump_time_width} second dumps')
scan_size_mb = float(scan_size) / (1024**2)
print(f'Wrote scan data ({scan_size_mb:.3f} MiB) '
f'in {s1:.3f} s ({scan_size_mb / s1:.3f} MiBps)\n')
scan_itr += 1
total_size += scan_size
finally:
work_queue.put(None)
writer_exc = None
# Drain the result_queue so that we unblock the writer process
while True:
result = result_queue.get()
if isinstance(result, Exception):
writer_exc = result
elif result is None:
break
writer_process.join()
# This raise is deferred to outside the finally block, so that we don't
# raise an exception while unwinding another one.
if isinstance(writer_exc, Exception):
raise writer_exc
if total_size == 0:
raise RuntimeError("No usable data found in MVF dataset "
"(pick another reference antenna, maybe?)")
# Remove spaces from source names, unless otherwise specified
field_names = [f.replace(' ', '') for f in field_names] \
if not options.keep_spaces else field_names
ms_dict = {}
ms_dict['SPECTRAL_WINDOW'] = ms_extra.populate_spectral_window_dict(
out_freqs, channel_freq_width * np.ones(len(out_freqs)))
ms_dict['FIELD'] = ms_extra.populate_field_dict(
field_centers, field_times, field_names)
ms_dict['STATE'] = ms_extra.populate_state_dict(obs_modes)
ms_dict['SOURCE'] = ms_extra.populate_source_dict(
field_centers, field_times, field_names)
print("\nWriting dynamic fields to disk....\n")
# Finally we write the MS as per our created dicts
ms_extra.write_dict(ms_dict, ms_name, verbose=options.verbose)
if options.tar:
tar = tarfile.open(f'{ms_name}.tar', 'w')
tar.add(ms_name, arcname=os.path.basename(ms_name))
tar.close()
# --------------------------------------
# Now write calibration product tables if required
# Open first HDF5 file in the list to extract TelescopeState parameters from
# (can't extract telstate params from contatenated katdal file as it
# uses the hdf5 file directly)
first_dataset = katdal.open(args[0], ref_ant=options.ref_ant)
main_table = ms_extra.open_table(ms_name, verbose=options.verbose)
if options.caltables:
# copy extra subtable dictionary values necessary for caltable
caltable_dict['SPECTRAL_WINDOW'] = ms_dict['SPECTRAL_WINDOW']
caltable_dict['FIELD'] = ms_dict['FIELD']
solution_types = ['G', 'B', 'K']
ms_soltype_lookup = {
'G': 'G Jones',
'B': 'B Jones',
'K': 'K Jones'
}
print("\nWriting calibration solution tables to disk....")
if 'TelescopeState' not in first_dataset.file.keys():
print(
" No TelescopeState in first dataset. Can't create solution tables.\n"
)
else:
# first get solution antenna ordering
# newer files have the cal antlist as a sensor
if 'cal_antlist' in first_dataset.file['TelescopeState'].keys(
):
a0 = first_dataset.file['TelescopeState/cal_antlist'][()]
antlist = telstate_decode(a0[0][1])
# older files have the cal antlist as an attribute
elif 'cal_antlist' in first_dataset.file[
'TelescopeState'].attrs.keys():
antlist = np.safe_eval(
first_dataset.file['TelescopeState'].
attrs['cal_antlist'])
else:
print(" No calibration antenna ordering in first dataset. "
"Can't create solution tables.\n")
continue
antlist_indices = list(range(len(antlist)))
# for each solution type in the file, create a table
for sol in solution_types:
caltable_name = f'{basename}.{sol}'
sol_name = f'cal_product_{sol}'
if sol_name in first_dataset.file['TelescopeState'].keys():
print(
f' - creating {sol} solution table: {caltable_name}\n'
)
# get solution values from the file
solutions = first_dataset.file['TelescopeState'][
sol_name][()]
soltimes, solvals = [], []
for t, s in solutions:
soltimes.append(t)
solvals.append(telstate_decode(s))
solvals = np.array(solvals)
# convert averaged UTC timestamps to MJD seconds.
sol_mjd = np.array([
katpoint.Timestamp(time_utc).to_mjd() * 24 * 60 *
60 for time_utc in soltimes
])
# determine solution characteristics
if len(solvals.shape) == 4:
ntimes, nchans, npols, nants = solvals.shape
else:
ntimes, npols, nants = solvals.shape
nchans = 1
solvals = solvals.reshape(ntimes, nchans, npols,
nants)
# create calibration solution measurement set
caltable_desc = ms_extra.caltable_desc_float \
if sol == 'K' else ms_extra.caltable_desc_complex
caltable = ms_extra.open_table(caltable_name,
tabledesc=caltable_desc)
# add other keywords for main table
if sol == 'K':
caltable.putkeyword('ParType', 'Float')
else:
caltable.putkeyword('ParType', 'Complex')
caltable.putkeyword('MSName', ms_name)
caltable.putkeyword('VisCal', ms_soltype_lookup[sol])
caltable.putkeyword('PolBasis', 'unknown')
# add necessary units
caltable.putcolkeywords(
'TIME', {
'MEASINFO': {
'Ref': 'UTC',
'type': 'epoch'
},
'QuantumUnits': ['s']
})
caltable.putcolkeywords('INTERVAL',
{'QuantumUnits': ['s']})
# specify that this is a calibration table
caltable.putinfo({
'readme': '',
'subType': ms_soltype_lookup[sol],
'type': 'Calibration'
})
# get the solution data to write to the main table
solutions_to_write = solvals.transpose(
0, 3, 1, 2).reshape(ntimes * nants, nchans, npols)
# MS's store delays in nanoseconds
if sol == 'K':
solutions_to_write = 1e9 * solutions_to_write
times_to_write = np.repeat(sol_mjd, nants)
antennas_to_write = np.tile(antlist_indices, ntimes)
# just mock up the scans -- this doesnt actually correspond to scans in the data
scans_to_write = np.repeat(list(range(len(sol_mjd))),
nants)
# write the main table
main_cal_dict = ms_extra.populate_caltable_main_dict(
times_to_write, solutions_to_write,
antennas_to_write, scans_to_write)
ms_extra.write_rows(caltable,
main_cal_dict,
verbose=options.verbose)
# create and write subtables
subtables = [
'OBSERVATION', 'ANTENNA', 'FIELD',
'SPECTRAL_WINDOW', 'HISTORY'
]
subtable_key = [(os.path.join(caltable.name(), st))
for st in subtables]
# Add subtable keywords and create subtables
# ------------------------------------------------------------------------------
# # this gives an error in casapy:
# *** Error *** MSObservation(const Table &) - table is not a valid MSObservation
# for subtable, subtable_location in zip(subtables, subtable_key)
# ms_extra.open_table(subtable_location, tabledesc=ms_extra.ms_desc[subtable])
# caltable.putkeyword(subtable, 'Table: {0}'.format(subtable_location))
# # write the static info for the table
# ms_extra.write_dict(caltable_dict, caltable.name(), verbose=options.verbose)
# ------------------------------------------------------------------------------
# instead try just copying the main table subtables
# this works to plot the data casapy, but the solutions still can't be
# applied in casapy...
for subtable, subtable_location in zip(
subtables, subtable_key):
main_subtable = ms_extra.open_table(
os.path.join(main_table.name(), subtable))
main_subtable.copy(subtable_location, deep=True)
caltable.putkeyword(subtable,
f'Table: {subtable_location}')
if subtable == 'ANTENNA':
caltable.putkeyword('NAME', antlist)
caltable.putkeyword('STATION', antlist)
if sol != 'B':
spw_table = ms_extra.open_table(
os.path.join(caltable.name(),
'SPECTRAL_WINDOW'))
spw_table.removerows(spw_table.rownumbers())
cen_index = len(out_freqs) // 2
# the delay values in the cal pipeline are calculated relative to frequency 0
ref_freq = 0.0 if sol == 'K' else None
spw_dict = {
'SPECTRAL_WINDOW':
ms_extra.populate_spectral_window_dict(
np.atleast_1d(out_freqs[cen_index]),
np.atleast_1d(channel_freq_width),
ref_freq=ref_freq)
}
ms_extra.write_dict(spw_dict,
caltable.name(),
verbose=options.verbose)
# done with this caltable
caltable.flush()
caltable.close()
main_table.close()
def select_environment(data, antenna, condition="normal"):
""" Flag data for environmental conditions. Options are:
normal: Wind < 9.8m/s, -5C < Temperature < 40C, DeltaTemp < 3deg in 20 minutes
optimal: Wind < 2.9m/s, -5C < Temperature < 35C, DeltaTemp < 2deg in 10 minutes
ideal: Wind < 1m/s, 19C < Temp < 21C, DeltaTemp < 1deg in 30 minutes
"""
# Convert timestamps to UTCseconds using katpoint
timestamps = np.array(
[katpoint.Timestamp(timestamp) for timestamp in data["timestamp_ut"]],
dtype='float32')
# Fit a smooth function (cubic spline) in time to the temperature and wind data
raw_wind = data["wind_speed"]
raw_temp = data["temperature"]
fit_wind = interpolate.InterpolatedUnivariateSpline(timestamps,
raw_wind,
k=3)
fit_temp = interpolate.InterpolatedUnivariateSpline(timestamps,
raw_temp,
k=3)
#fit_temp_grad = fit_temp.derivative()
# Day/Night
# Night is defined as when the Sun is at -5deg.
# Set up Sun target
sun = katpoint.Target('Sun, special', antenna=antenna)
sun_elevation = katpoint.rad2deg(sun.azel(timestamps)[1])
# Apply limits on environmental conditions
good = [True] * data.shape[0]
# Set up limits on environmental conditions
if condition == 'ideal':
windlim = 1.
temp_low = 19.
temp_high = 21.
deltatemp = 1. / (30. * 60.)
sun_elev_lim = -5.
elif condition == 'optimum':
windlim = 2.9
temp_low = -5.
temp_high = 35.
deltatemp = 2. / (10. * 60.)
sun_elev_lim = -5.
elif condition == 'normal':
windlim = 9.8
temp_low = -5.
temp_high = 40.
deltatemp = 3. / (20. * 60.)
sun_elev_lim = 100. #Daytime
else:
return good
good = good & (fit_wind(timestamps) < windlim)
good = good & ((fit_temp(timestamps) > temp_low) &
(fit_temp(timestamps) < temp_high))
#Get the temperature gradient
temp_grad = [
fit_temp.derivatives(timestamp)[1] for timestamp in timestamps
]
good = good & (np.abs(temp_grad) < deltatemp)
#Day or night?
good = good & (sun_elevation < sun_elev_lim)
return good
def __init__(self,
filename,
ref_ant='',
time_offset=0.0,
mode='r',
**kwargs):
DataSet.__init__(self, filename, ref_ant, time_offset)
# Load file
self.file, self.version = H5DataV1._open(filename, mode)
f = self.file
# Load main HDF5 groups
ants_group, corr_group, data_group = f['Antennas'], f['Correlator'], f[
'Scans']
# Get observation script parameters, with defaults
self.observer = self.obs_params['observer'] = f.attrs.get(
'observer', '')
self.description = self.obs_params['description'] = f.attrs.get(
'description', '')
self.experiment_id = self.obs_params['experiment_id'] = f.attrs.get(
'experiment_id', '')
# Collect all groups below data group that fit the description of a scan group
scan_groups = []
def register_scan_group(name, obj):
"""A scan group is defined as a group named 'Scan*' with non-empty timestamps and data."""
if isinstance(obj, h5py.Group) and name.split('/')[-1].startswith('Scan') and \
'data' in obj and 'timestamps' in obj and len(obj['timestamps']) > 0:
scan_groups.append(obj)
data_group.visititems(register_scan_group)
# Sort scan groups in chronological order via 'decorate-sort-undecorate' (DSU) idiom
decorated_scan_groups = [(s['timestamps'][0], s) for s in scan_groups]
decorated_scan_groups.sort()
self._scan_groups = [s[-1] for s in decorated_scan_groups]
# ------ Extract timestamps ------
self.dump_period = 1.0 / corr_group.attrs['dump_rate_hz']
self._segments = np.cumsum(
[0] + [len(s['timestamps']) for s in self._scan_groups])
num_dumps = self._segments[-1]
self._time_keep = np.ones(num_dumps, dtype=np.bool)
data_timestamps = self.timestamps
if data_timestamps[0] < 1e9:
logger.warning(
"File '%s' has invalid first correlator timestamp (%f)" % (
filename,
data_timestamps[0],
))
# Estimate timestamps by assuming they are uniformly spaced (much quicker than loading them from file).
# This is useful for the purpose of segmenting data set, where accurate timestamps are not that crucial.
# The real timestamps are still loaded when the user explicitly asks for them.
# Do quick test for uniform spacing of timestamps (necessary but not sufficient).
if abs((data_timestamps[-1] - data_timestamps[0]) / self.dump_period +
1 - num_dumps) < 0.01:
# Estimate the timestamps as being uniformly spaced
data_timestamps = data_timestamps[
0] + self.dump_period * np.arange(num_dumps)
else:
# Load the real timestamps instead and warn the user, as this is anomalous
data_timestamps = data_timestamps[:]
expected_dumps = (data_timestamps[-1] -
data_timestamps[0]) / self.dump_period + 1
logger.warning((
"Irregular timestamps detected in file '%s':"
"expected %.3f dumps based on dump period and start/end times, got %d instead"
) % (filename, expected_dumps, num_dumps))
self.start_time = katpoint.Timestamp(data_timestamps[0] -
0.5 * self.dump_period)
self.end_time = katpoint.Timestamp(data_timestamps[-1] +
0.5 * self.dump_period)
# ------ Extract sensors ------
# Populate sensor cache with all HDF5 datasets below antennas group that fit the description of a sensor
cache = {}
def register_sensor(name, obj):
if isinstance(obj, h5py.Dataset) and obj.shape != (
) and obj.dtype.names == ('timestamp', 'value', 'status'):
# Assume sensor dataset name is AntennaN/Sensors/dataset and rename it to Antennas/{ant}/dataset
ant_name = obj.parent.parent.attrs['description'].split(',')[0]
standardised_name = 'Antennas/%s/%s' % (ant_name,
name.split('/')[-1])
cache[standardised_name] = SensorData(obj, standardised_name)
ants_group.visititems(register_sensor)
# Use estimated data timestamps for now, to speed up data segmentation
# This will linearly interpolate pointing coordinates to correlator data timestamps (on access)
# As long as azimuth is in natural antenna coordinates, no special angle interpolation required
self.sensor = SensorCache(cache,
data_timestamps,
self.dump_period,
keep=self._time_keep,
props=SENSOR_PROPS,
virtual=VIRTUAL_SENSORS,
aliases=SENSOR_ALIASES)
# ------ Extract subarrays ------
ants = [
katpoint.Antenna(ants_group[group].attrs['description'])
for group in ants_group
]
self.ref_ant = ants[0].name if not ref_ant else ref_ant
# Map from (old-style) DBE input label (e.g. '0x') to the new antenna-based input label (e.g. 'ant1h')
input_label = dict([(ants_group[group]['H'].attrs['dbe_input'],
ant.name + 'h')
for ant, group in zip(ants, ants_group.keys())
if 'H' in ants_group[group]])
input_label.update(
dict([(ants_group[group]['V'].attrs['dbe_input'], ant.name + 'v')
for ant, group in zip(ants, ants_group.keys())
if 'V' in ants_group[group]]))
# Split DBE input product string into its separate inputs
split_product = re.compile(r'(\d+[xy])(\d+[xy])')
# Iterate over map from correlation product index to DBE input product string and convert
# the latter to pairs of input labels (this assumes that the corrprod indices are sorted)
corrprods = []
for corrind, product in corr_group['input_map']:
match = split_product.match(product)
if match is None:
raise BrokenFile(
"Unknown DBE input product '%s' in input map (expected e.g. '0x1y')"
% (product, ))
corrprods.append(
tuple([input_label[inp] for inp in match.groups()]))
data_cp_len = len(self._scan_groups[0]['data'].dtype)
if len(corrprods) != data_cp_len:
raise BrokenFile(
'Number of baseline labels received from correlator '
'(%d) differs from number of baselines in data (%d)' %
(len(corrprods), data_cp_len))
self.subarrays = [Subarray(ants, corrprods)]
self.sensor['Observation/subarray'] = CategoricalData(
self.subarrays, [0, len(data_timestamps)])
self.sensor['Observation/subarray_index'] = CategoricalData(
[0], [0, len(data_timestamps)])
# Store antenna objects in sensor cache too, for use in virtual sensor calculations
for ant in ants:
self.sensor['Antennas/%s/antenna' %
(ant.name, )] = CategoricalData(
[ant], [0, len(data_timestamps)])
# ------ Extract spectral windows / frequencies ------
centre_freq = corr_group.attrs['center_frequency_hz']
num_chans = corr_group.attrs['num_freq_channels']
data_num_chans = self._scan_groups[0]['data'].shape[1]
if num_chans != data_num_chans:
raise BrokenFile(
'Number of channels received from correlator '
'(%d) differs from number of channels in data (%d)' %
(num_chans, data_num_chans))
channel_width = corr_group.attrs['channel_bandwidth_hz']
self.spectral_windows = [
SpectralWindow(centre_freq, channel_width, num_chans, 'poco')
]
self.sensor['Observation/spw'] = CategoricalData(
self.spectral_windows, [0, len(data_timestamps)])
self.sensor['Observation/spw_index'] = CategoricalData(
[0], [0, len(data_timestamps)])
# ------ Extract scans / compound scans / targets ------
# Fringe Finder augment does not store antenna activity sensors - use scan + compscan labels as a guess
scan_labels = [s.attrs.get('label', '') for s in self._scan_groups]
compscan_labels = [
s.parent.attrs.get('label', '') for s in self._scan_groups
]
scan_states = [
_labels_to_state(s, cs)
for s, cs in zip(scan_labels, compscan_labels)
]
# The scans are already partitioned into groups - use corresponding segments as start events
self.sensor['Observation/scan_state'] = CategoricalData(
scan_states, self._segments)
self.sensor['Observation/scan_index'] = CategoricalData(
range(len(scan_states)), self._segments)
# Group scans together based on compscan group name and have one label per compound scan
compscan = CategoricalData([s.parent.name for s in self._scan_groups],
self._segments)
compscan.remove_repeats()
label = CategoricalData(compscan_labels, self._segments)
label.align(compscan.events)
self.sensor['Observation/label'] = label
self.sensor['Observation/compscan_index'] = CategoricalData(
range(len(label)), label.events)
# Extract targets from compscan groups, replacing empty or bad descriptions with dummy target
target = CategoricalData([
_robust_target(s.parent.attrs.get('target', ''))
for s in self._scan_groups
], self._segments)
target.align(compscan.events)
self.sensor['Observation/target'] = target
self.sensor['Observation/target_index'] = CategoricalData(
target.indices, target.events)
# Set up catalogue containing all targets in file, with reference antenna as default antenna
self.catalogue.add(target.unique_values)
self.catalogue.antenna = self.sensor['Antennas/%s/antenna' %
(self.ref_ant, )][0]
# Ensure that each target flux model spans all frequencies in data set if possible
self._fix_flux_freq_range()
# Restore original (slow) timestamps so that subsequent sensors (e.g. pointing) will have accurate values
self.sensor.timestamps = self.timestamps
# Apply default selection and initialise all members that depend on selection in the process
self.select(spw=0, subarray=0)
def __init__(self, source, ref_ant='', time_offset=0.0, **kwargs):
DataSet.__init__(self, source.name, ref_ant, time_offset)
attrs = source.metadata.attrs
# ------ Extract timestamps ------
self.source = source
self.file = {}
self.version = '4.0'
self.dump_period = attrs['int_time']
num_dumps = len(source.timestamps)
source.timestamps += self.time_offset
if source.timestamps[0] < 1e9:
logger.warning(
"Data set has invalid first correlator timestamp "
"(%f)", source.timestamps[0])
half_dump = 0.5 * self.dump_period
self.start_time = katpoint.Timestamp(source.timestamps[0] - half_dump)
self.end_time = katpoint.Timestamp(source.timestamps[-1] + half_dump)
self._time_keep = np.full(num_dumps, True, dtype=np.bool_)
all_dumps = [0, num_dumps]
# Assemble sensor cache
self.sensor = SensorCache(source.metadata.sensors, source.timestamps,
self.dump_period, self._time_keep,
SENSOR_PROPS, VIRTUAL_SENSORS,
SENSOR_ALIASES)
# ------ Extract flags ------
# Internal flag mask overridden whenever _flags_keep is set via select()
self._flags_select = np.array([255], dtype=np.uint8)
self._flags_keep = 'all'
# ------ Extract observation parameters and script log ------
self.obs_params = attrs['obs_params']
# Get observation script parameters, with defaults
self.observer = self.obs_params.get('observer', '')
self.description = self.obs_params.get('description', '')
self.experiment_id = self.obs_params.get('experiment_id', '')
# Extract script log data verbatim (it is not a standard sensor anyway)
try:
self.obs_script_log = self.sensor.get(
'obs_script_log', extract=False)['value'].tolist()
except KeyError:
self.obs_script_log = []
# ------ Extract subarrays ------
# List of correlation products as pairs of input labels
corrprods = attrs['bls_ordering']
# Crash if there is mismatch between labels and data shape (bad labels?)
if source.data and (len(corrprods) != source.data.shape[2]):
raise BrokenFile('Number of baseline labels (containing expected '
'antenna names) received from correlator (%d) '
'differs from number of baselines in data (%d)' %
(len(corrprods), source.data.shape[2]))
# Find all antennas in subarray with valid katpoint Antenna objects
ants = []
for resource in attrs['sub_pool_resources'].split(','):
try:
ant_description = attrs[resource + '_observer']
ants.append(katpoint.Antenna(ant_description))
except (KeyError, ValueError):
continue
# Keep the basic list sorted as far as possible
ants = sorted(ants)
cam_ants = set(ant.name for ant in ants)
# Find names of all antennas with associated correlator data
sdp_ants = set([cp[0][:-1] for cp in corrprods] +
[cp[1][:-1] for cp in corrprods])
# By default, only pick antennas that were in use by the script
obs_ants = self.obs_params.get('ants')
# Otherwise fall back to the list of antennas common to CAM and SDP / CBF
obs_ants = obs_ants.split(',') if obs_ants else list(cam_ants
& sdp_ants)
self.ref_ant = obs_ants[0] if not ref_ant else ref_ant
self.subarrays = subs = [Subarray(ants, corrprods)]
self.sensor['Observation/subarray'] = CategoricalData(subs, all_dumps)
self.sensor['Observation/subarray_index'] = CategoricalData([0],
all_dumps)
# Store antenna objects in sensor cache too, for use in virtual sensors
for ant in ants:
sensor_name = 'Antennas/%s/antenna' % (ant.name, )
self.sensor[sensor_name] = CategoricalData([ant], all_dumps)
# ------ Extract spectral windows / frequencies ------
# Get the receiver band identity ('l', 's', 'u', 'x')
band = attrs['sub_band']
# Populate antenna -> receiver mapping and figure out noise diode
for ant in cam_ants:
# Try sanitised version of RX serial number first
rx_serial = attrs.get('%s_rsc_rx%s_serial_number' % (ant, band), 0)
self.receivers[ant] = '%s.%d' % (band, rx_serial)
nd_sensor = '%s_dig_%s_band_noise_diode' % (ant, band)
if nd_sensor in self.sensor:
# A sensor alias would be ideal for this but it only deals with suffixes ATM
new_nd_sensor = 'Antennas/%s/nd_coupler' % (ant, )
self.sensor[new_nd_sensor] = self.sensor.get(nd_sensor,
extract=False)
num_chans = attrs['n_chans']
bandwidth = attrs['bandwidth']
centre_freq = attrs['center_freq']
channel_width = bandwidth / num_chans
# Continue with different channel count, but invalidate centre freq
# (keep channel width though)
if source.data and (num_chans != source.data.shape[1]):
logger.warning(
'Number of channels reported in metadata (%d) differs'
' from actual number of channels in data (%d) - '
'trusting the latter', num_chans, source.data.shape[1])
num_chans = source.data.shape[1]
centre_freq = 0.0
product = attrs.get('sub_product', '')
sideband = 1
band_map = dict(l='L', s='S', u='UHF', x='X')
spw_params = (centre_freq, channel_width, num_chans, product, sideband,
band_map[band])
# We only expect a single spectral window within a single v4 data set
self.spectral_windows = spws = [SpectralWindow(*spw_params)]
self.sensor['Observation/spw'] = CategoricalData(spws, all_dumps)
self.sensor['Observation/spw_index'] = CategoricalData([0], all_dumps)
# ------ Extract scans / compound scans / targets ------
# Use the activity sensor of reference antenna to partition the data
# set into scans (and to set their states)
scan = self.sensor.get(self.ref_ant + '_activity')
# If the antenna starts slewing on the second dump, incorporate the
# first dump into the slew too. This scenario typically occurs when the
# first target is only set after the first dump is received.
# The workaround avoids putting the first dump in a scan by itself,
# typically with an irrelevant target.
if len(scan) > 1 and scan.events[1] == 1 and scan[1] == 'slew':
scan.events, scan.indices = scan.events[1:], scan.indices[1:]
scan.events[0] = 0
# Use labels to partition the data set into compound scans
try:
label = self.sensor.get('obs_label')
except KeyError:
label = CategoricalData([''], all_dumps)
# Discard empty labels (typically found in raster scans, where first
# scan has proper label and rest are empty) However, if all labels are
# empty, keep them, otherwise whole data set will be one pathological
# compscan...
if len(label.unique_values) > 1:
label.remove('')
# Create duplicate scan events where labels are set during a scan
# (i.e. not at start of scan)
# ASSUMPTION: Number of scans >= number of labels
# (i.e. each label should introduce a new scan)
scan.add_unmatched(label.events)
self.sensor['Observation/scan_state'] = scan
self.sensor['Observation/scan_index'] = CategoricalData(
range(len(scan)), scan.events)
# Move proper label events onto the nearest scan start
# ASSUMPTION: Number of labels <= number of scans
# (i.e. only a single label allowed per scan)
label.align(scan.events)
# If one or more scans at start of data set have no corresponding label,
# add a default label for them
if label.events[0] > 0:
label.add(0, '')
self.sensor['Observation/label'] = label
self.sensor['Observation/compscan_index'] = CategoricalData(
range(len(label)), label.events)
# Use the target sensor of reference antenna to set target for each scan
target = self.sensor.get(self.ref_ant + '_target')
# Remove initial blank target (typically because antenna starts stopped)
if len(target) > 1 and target[0] == 'Nothing, special':
target.events, target.indices = target.events[1:], target.indices[
1:]
target.events[0] = 0
# Move target events onto the nearest scan start
# ASSUMPTION: Number of targets <= number of scans
# (i.e. only a single target allowed per scan)
target.align(scan.events)
self.sensor['Observation/target'] = target
self.sensor['Observation/target_index'] = CategoricalData(
target.indices, target.events)
# Set up catalogue containing all targets in file, with reference
# antenna as default antenna
self.catalogue.add(target.unique_values)
ref_sensor = 'Antennas/%s/antenna' % (self.ref_ant, )
self.catalogue.antenna = self.sensor.get(ref_sensor)[0]
# Ensure that each target flux model spans all frequencies
# in data set if possible
self._fix_flux_freq_range()
# Apply default selection and initialise all members that depend
# on selection in the process
self.select(spw=0, subarray=0, ants=obs_ants)
def _before(date):
return source.timestamps[0] < katpoint.Timestamp(date).secs