def calculate_corrections(G_gains, B_gains, delays, cal_channel_freqs,
random_phase, flatten_bandpass,
target_average_correction):
"""Turn cal pipeline products into corrections to be passed to F-engine."""
average_gain = {}
gain_corrections = {}
# First find relative corrections per input with arbitrary global average
for inp in G_gains:
# Combine all calibration products for input into single array of gains
K_gains = np.exp(-2j * np.pi * delays[inp] * cal_channel_freqs)
gains = K_gains * B_gains[inp] * G_gains[inp]
if np.isnan(gains).all():
average_gain[inp] = gain_corrections[inp] = 0.0
continue
abs_gains = np.abs(gains)
# Track the average gain to fix overall power level (and as diagnostic)
average_gain[inp] = np.nanmedian(abs_gains)
corrections = 1.0 / gains
if not flatten_bandpass:
# Let corrections have constant magnitude equal to 1 / (avg gain),
# which ensures that power levels are still equalised between inputs
corrections *= abs_gains / average_gain[inp]
if random_phase:
corrections *= np.exp(2j * np.pi * np.random.rand(len(corrections)))
gain_corrections[inp] = np.nan_to_num(corrections)
# All invalid gains (NaNs) have now been turned into zeros
valid_average_gains = [g for g in average_gain.values() if g > 0]
if not valid_average_gains:
raise ValueError("All gains invalid and beamformer output will be zero!")
global_average_gain = np.median(valid_average_gains)
# Iterate over inputs again and fix average values of corrections
for inp in sorted(G_gains):
relative_gain = average_gain[inp] / global_average_gain
if relative_gain == 0.0:
user_logger.warning("%s has no valid gains and will be zeroed", inp)
continue
# This ensures that input at the global average gets target correction
gain_corrections[inp] *= target_average_correction * global_average_gain
safe_relative_gain = np.clip(relative_gain, 0.5, 2.0)
if relative_gain == safe_relative_gain:
user_logger.info("%s: average gain relative to global average = %5.2f",
inp, relative_gain)
else:
user_logger.warning("%s: average gain relative to global average "
"= %5.2f out of range, clipped to %.1f",
inp, relative_gain, safe_relative_gain)
gain_corrections[inp] *= relative_gain / safe_relative_gain
return gain_corrections
)
with start_session(kat, **vars(opts)) as session:
# If centre frequency is specified, set it accordingly
user_logger.info('Current centre frequency: %s MHz' %
(session.get_centre_freq(), ))
if not kat.dry_run and opts.centre_freq and not session.get_centre_freq(
) == opts.centre_freq:
session.set_centre_freq(opts.centre_freq)
time.sleep(1.0)
user_logger.info('Updated centre frequency: %s MHz' %
(session.get_centre_freq(), ))
if np.abs(session.get_centre_freq() -
opts.centre_freq) > 2e-5: # we have 10 HZ resolution
user_logger.warning(
'Failed to updated centre frequency to %s MHz, it is currently set to %s MHz waning is due to the difference between actual & spcified frequency is larger that 10 Hz'
% (
opts.centre_freq,
session.get_centre_freq(),
))
session.standard_setup(**vars(opts))
session.capture_start()
if True: # Dry run will now exec this branch, in the past this branch was exculded from the dry checking
# check that the selected dbe is set to the correct mode
dbe_mode = kat.dbe7.sensor.dbe_mode.get_value()
dbe7_mode_dict = {
'bc16n400M1k': 160,
'c16n400M1k': 160,
'c16n400M8k': 160,
'wbc': 160,
'wbc8k': 160,
'c16n7M4k': 31,
if len(args) == 0:
raise ValueError("Please specify the target")
with verify_and_connect(opts) as kat:
# check for PTUSE proxies in the subarray
ptuses = [
kat.children[name] for name in sorted(kat.children)
if name.startswith('ptuse')
]
if len(ptuses) == 0:
raise ValueError("This script requires a PTUSE proxy")
elif len(ptuses) > 1:
# for now keep script simple and only use first proxy
ptuse = ptuses[0]
user_logger.warning('Found PTUSE proxies: %s - only using: %s', ptuses,
ptuse.name)
else:
ptuse = ptuses[0]
user_logger.info('Using PTUSE proxy: %s', ptuse.name)
bf_ants = opts.ants.split(',') if opts.ants else [
ant.name for ant in kat.ants
]
cbf = SessionCBF(kat)
# Special hack for Lab CBF - set to zero on site
# TODO: stop hacking, or make this a command line option
freq_delta_hack = 0
if freq_delta_hack:
user_logger.warning(
'Hack: Adjusting beam centre frequency by %s MHz for lab',
freq_delta_hack)
def get_offset_gains(session, offsets, offset_end_times, track_duration):
"""Extract gains per pointing offset, per receptor and per frequency chunk.
Parameters
----------
session : :class:`katcorelib.observe.CaptureSession` object
The active capture session
offsets : sequence of *N* pairs of float (i.e. shape (*N*, 2))
Requested (x, y) pointing offsets relative to target, in degrees
offset_end_times : sequence of *N* floats
Unix timestamp at the end of each pointing track
track_duration : float
Duration of each pointing track, in seconds
Returns
-------
data_points : dict mapping receptor index to (x, y, freq, gain, weight) seq
Complex gains per receptor, as multiple records per offset and frequency
"""
cal_channel_freqs = session.get_cal_channel_freqs()
chunk_freqs = cal_channel_freqs.reshape(NUM_CHUNKS, -1).mean(axis=1)
pols = session.telstate['cal_pol_ordering']
data_points = {}
# Iterate over offset pointings
for offset, offset_end in zip(offsets, offset_end_times):
offset_start = offset_end - track_duration
# Obtain interferometric gains per pointing from the cal pipeline
try:
bp_gains = session.get_cal_solutions('B',
start_time=offset_start,
end_time=offset_end)
gains = session.get_cal_solutions('G',
start_time=offset_start,
end_time=offset_end)
except CalSolutionsUnavailable as err:
user_logger.warning('Skipping offset %s: %s', offset, err)
continue
# Iterate over receptors
for a, ant in enumerate(session.observers):
pol_gain = np.zeros(NUM_CHUNKS)
pol_weight = np.zeros(NUM_CHUNKS)
# Iterate over polarisations (effectively over inputs)
for pol in pols:
inp = ant.name + pol
bp_gain = bp_gains.get(inp)
gain = gains.get(inp)
if bp_gain is None or gain is None:
continue
masked_gain = np.ma.masked_invalid(bp_gain * gain)
abs_gain_chunked = np.abs(masked_gain).reshape(NUM_CHUNKS, -1)
abs_gain_mean = abs_gain_chunked.mean(axis=1)
abs_gain_std = abs_gain_chunked.std(axis=1)
abs_gain_var = abs_gain_std.filled(np.inf)**2
# Replace any zero variance with the smallest non-zero variance
# across chunks, but if all are zero it is fishy and ignored.
zero_var = abs_gain_var == 0.
if all(zero_var):
abs_gain_var = np.ones_like(abs_gain_var) * np.inf
else:
abs_gain_var[zero_var] = abs_gain_var[~zero_var].min()
# Number of valid samples going into statistics
abs_gain_N = (~abs_gain_chunked.mask).sum(axis=1)
# Generate standard precision weights based on empirical stdev
abs_gain_weight = abs_gain_N / abs_gain_var
# Prepare some debugging output
stats_mean = ' '.join("%4.2f" % (m, )
for m in abs_gain_mean.filled(np.nan))
stats_std = ' '.join("%4.2f" % (s, )
for s in abs_gain_std.filled(np.nan))
stats_N = ' '.join("%4d" % (n, ) for n in abs_gain_N)
bp_mean = np.nanmean(np.abs(bp_gain))
user_logger.debug("%s %s %4.2f mean | %s", tuple(offset), inp,
np.abs(gain), stats_mean)
user_logger.debug("%s %s %4.2f std | %s", tuple(offset), inp,
bp_mean, stats_std)
user_logger.debug("%s %s N | %s", tuple(offset), inp,
stats_N)
# Blend new gains into existing via weighted averaging.
# XXX We currently combine HH and VV gains at the start to get
# Stokes I gain but in future it might be better to fit
# separate beams to HH and VV.
pol_gain, pol_weight = np.ma.average(
np.c_[pol_gain, abs_gain_mean],
axis=1,
weights=np.c_[pol_weight, abs_gain_weight],
returned=True)
if pol_weight.sum() > 0:
# Turn masked values into NaNs pre-emptively to avoid warning
# when recarray in beam fitting routine forces this later on.
pol_gain = pol_gain.filled(np.nan)
data = data_points.get(a, [])
for freq, gain, weight in zip(chunk_freqs, pol_gain,
pol_weight):
data.append((offset[0], offset[1], freq, gain, weight))
data_points[a] = data
if not data_points:
raise CalSolutionsUnavailable("No complete gain solutions found in "
"telstate for any offset")
return data_points
session.label('un_corrected')
user_logger.info("Initiating %g-second track on target '%s'",
opts.track_duration, target.name)
session.track(target, duration=opts.track_duration, announce=False)
# Attempt to jiggle cal pipeline to drop its gains
session.stop_antennas()
user_logger.info("Waiting for gains to materialise in cal pipeline")
# Wait for the last bfcal product from the pipeline
gains = session.get_cal_solutions('G', timeout=opts.track_duration)
bp_gains = session.get_cal_solutions('B')
delays = session.get_cal_solutions('K')
cal_channel_freqs = session.get_cal_channel_freqs()
bp_gains = clean_bandpass(bp_gains, cal_channel_freqs, max_gap_Hz=64e6)
if opts.random_phase:
user_logger.warning("Setting F-engine gains with random phases "
"(you asked for it)")
else:
user_logger.info("Setting F-engine gains to phase up antennas")
if not kat.dry_run:
corrections = calculate_corrections(gains, bp_gains, delays,
cal_channel_freqs, opts.random_phase,
opts.flatten_bandpass, fengine_gain)
session.set_fengine_gains(corrections)
if opts.verify_duration > 0:
user_logger.info("Revisiting target %r for %g seconds to verify phase-up",
target.name, opts.verify_duration)
session.label('corrected')
session.track(target, duration=opts.verify_duration, announce=False)
if not opts.random_phase:
# Set last-phaseup script sensor on the subarray.
keep_going = True
while keep_going:
keep_going = opts.max_duration is not None
targets_before_loop = len(targets_observed)
# Iterate through source list, picking the next one that is up
bgain_list = np.linspace(b_start, b_end, b_step)
for bgain in bgain_list:
for target in targets.iterfilter(el_limit_deg=opts.horizon):
duration = opts.track_duration
if opts.max_duration is not None:
time_left = opts.max_duration - (time.time() -
start_time)
# Cut the track short if time runs out
if time_left <= 0.:
user_logger.warning(
"Maximum duration of %g seconds "
"has elapsed - stopping script",
opts.max_duration)
keep_going = False
break
duration = min(duration, time_left)
# Set the b-gain
session.label('track_bgain, %g' % bgain)
for stream in cbf.beamformers:
stream.req.quant_gains(bgain)
user_logger.info(
"B-engine %s quantisation gain set to %g", stream,
bgain)
if session.track(target, duration=duration):
targets_observed.append(target.description)
if keep_going and len(targets_observed) == targets_before_loop:
user_logger.warning(
actual_centre_freq = float(reply.messages[0].arguments[3])
user_logger.info(
"Beamformer %r has bandwidth %g Hz and centre freq %g Hz",
stream, actual_bandwidth, actual_centre_freq)
else:
raise ValueError("Could not set beamformer %r passband - (%s)" %
(stream, ' '.join(reply.messages[0].arguments)))
user_logger.info('Setting beamformer weights for stream %r:', stream)
for inp in stream.inputs:
weight = 1.0 / np.sqrt(
len(bf_ants)) if inp[:-1] in bf_ants else 0.0
reply = stream.req.weights(inp, weight)
if reply.succeeded:
user_logger.info(' input %r got weight %f', inp, weight)
else:
user_logger.warning(' input %r weight could not be set', inp)
# We are only interested in first target
target_name = args[:1][0]
print target_name
user_logger.info('Looking up main beamformer target...')
target = collect_targets(kat, args[:1]).targets[0]
# Ensure that the target is up
target_elevation = np.degrees(target.azel()[1])
if target_elevation < opts.horizon:
raise ValueError("The target %r is below the horizon" %
(target.description, ))
# Verify backend_args
# Check options and build KAT configuration, connecting to proxies and devices
with verify_and_connect(opts) as kat:
ants = kat.ants
obs_ants = [ant.name for ant in ants]
observation_sources = collect_targets(kat,args)
# Find out what inputs are curremtly active
reply = kat.data.req.dbe_label_input()
inputs = [m.arguments[0] for m in reply.messages[3:]]
user_logger.info("Resetting f-engine gains to 160 to allow phasing up")
for inp in inputs:
kat.data.req.dbe_k7_gain(inp,160)
# Quit early if there are no sources to observe
if len(observation_sources.filter(el_limit_deg=opts.horizon)) == 0:
user_logger.warning("No targets are currently visible - please re-run the script later")
else:
# Start capture session, which creates HDF5 file
with start_session(kat, **vars(opts)) as session:
session.standard_setup(**vars(opts))
session.capture_start()
start_time = time.time()
targets_observed = []
# Keep going until the time is up
keep_going = True
while keep_going:
for target in observation_sources.iterfilter(el_limit_deg=opts.horizon):
if target.flux_model is None:
user_logger.warning("Target has no flux model - stopping script")
keep_going=False
raise ValueError(
"Could not set beamformer %r passband - (%s)" %
(stream.name, ' '.join(reply.messages[0].arguments)))
user_logger.info('Setting beamformer weights for stream %r:',
stream.name)
weights = []
for inp in stream.inputs:
weight = 1.0 if inp[:-1] in bf_ants else 0.0
weights.append(weight)
user_logger.info(' input %r will get weight %f', inp, weight)
reply = stream.req.weights(*weights)
if reply.succeeded:
user_logger.info('Set input weights successfully')
else:
user_logger.warning('Failed to set input weights!')
# We are only interested in first target
user_logger.info('Looking up main beamformer target...')
tgt_with_spaces = [tgt for tgt in args[:1] if len(tgt.strip().split()) > 1]
if len(tgt_with_spaces) > 0:
user_logger.error(
"Please replace '%s' with '%s'", '& '.join(tgt_with_spaces),
'& '.join(
[''.join(tgt.strip().split()) for tgt in tgt_with_spaces]))
raise ValueError(
'Found spaces in target names, which will cause an error in digifits'
)
target = collect_targets(kat, args[:1]).targets[0]
"please re-run the script later")
session.standard_setup(**vars(opts))
if opts.fft_shift is not None:
session.cbf.fengine.req.fft_shift(opts.fft_shift)
session.cbf.correlator.req.capture_start()
for target in [observation_sources.sort('el').targets[-1]]:
target.add_tags('bfcal single_accumulation')
print target.flux_model
print target.name
if not opts.default_gain:
channels = 32768 if session.product.endswith('32k') else 4096
opts.default_gain = DEFAULT_GAIN[channels]
user_logger.info("Target to be observed: %s", target.description)
if target.flux_model is None:
user_logger.warning("Target has no flux model (katsdpcal will need it in future)")
user_logger.info("Resetting F-engine gains to %g to allow phasing up",
opts.default_gain)
for inp in session.cbf.fengine.inputs:
session.cbf.fengine.req.gain(inp, opts.default_gain)
session.label('un_corrected')
user_logger.info("Initiating %g-second track on target '%s'",
opts.track_duration, target.name)
session.track(target, duration=opts.track_duration, announce=False)
# Attempt to jiggle cal pipeline to drop its gains
session.ants.req.target('')
user_logger.info("Waiting for gains to materialise in cal pipeline")
delays = bp_gains = gains = {}
cal_channel_freqs = None
if not kat.dry_run:
# Wait for the last bfcal product from the pipeline