def calculate_corrections(G_gains, B_gains, delays, cal_channel_freqs, random_phase, flatten_bandpass, target_average_correction): """Turn cal pipeline products into corrections to be passed to F-engine.""" average_gain = {} gain_corrections = {} # First find relative corrections per input with arbitrary global average for inp in G_gains: # Combine all calibration products for input into single array of gains K_gains = np.exp(-2j * np.pi * delays[inp] * cal_channel_freqs) gains = K_gains * B_gains[inp] * G_gains[inp] if np.isnan(gains).all(): average_gain[inp] = gain_corrections[inp] = 0.0 continue abs_gains = np.abs(gains) # Track the average gain to fix overall power level (and as diagnostic) average_gain[inp] = np.nanmedian(abs_gains) corrections = 1.0 / gains if not flatten_bandpass: # Let corrections have constant magnitude equal to 1 / (avg gain), # which ensures that power levels are still equalised between inputs corrections *= abs_gains / average_gain[inp] if random_phase: corrections *= np.exp(2j * np.pi * np.random.rand(len(corrections))) gain_corrections[inp] = np.nan_to_num(corrections) # All invalid gains (NaNs) have now been turned into zeros valid_average_gains = [g for g in average_gain.values() if g > 0] if not valid_average_gains: raise ValueError("All gains invalid and beamformer output will be zero!") global_average_gain = np.median(valid_average_gains) # Iterate over inputs again and fix average values of corrections for inp in sorted(G_gains): relative_gain = average_gain[inp] / global_average_gain if relative_gain == 0.0: user_logger.warning("%s has no valid gains and will be zeroed", inp) continue # This ensures that input at the global average gets target correction gain_corrections[inp] *= target_average_correction * global_average_gain safe_relative_gain = np.clip(relative_gain, 0.5, 2.0) if relative_gain == safe_relative_gain: user_logger.info("%s: average gain relative to global average = %5.2f", inp, relative_gain) else: user_logger.warning("%s: average gain relative to global average " "= %5.2f out of range, clipped to %.1f", inp, relative_gain, safe_relative_gain) gain_corrections[inp] *= relative_gain / safe_relative_gain return gain_corrections
) with start_session(kat, **vars(opts)) as session: # If centre frequency is specified, set it accordingly user_logger.info('Current centre frequency: %s MHz' % (session.get_centre_freq(), )) if not kat.dry_run and opts.centre_freq and not session.get_centre_freq( ) == opts.centre_freq: session.set_centre_freq(opts.centre_freq) time.sleep(1.0) user_logger.info('Updated centre frequency: %s MHz' % (session.get_centre_freq(), )) if np.abs(session.get_centre_freq() - opts.centre_freq) > 2e-5: # we have 10 HZ resolution user_logger.warning( 'Failed to updated centre frequency to %s MHz, it is currently set to %s MHz waning is due to the difference between actual & spcified frequency is larger that 10 Hz' % ( opts.centre_freq, session.get_centre_freq(), )) session.standard_setup(**vars(opts)) session.capture_start() if True: # Dry run will now exec this branch, in the past this branch was exculded from the dry checking # check that the selected dbe is set to the correct mode dbe_mode = kat.dbe7.sensor.dbe_mode.get_value() dbe7_mode_dict = { 'bc16n400M1k': 160, 'c16n400M1k': 160, 'c16n400M8k': 160, 'wbc': 160, 'wbc8k': 160, 'c16n7M4k': 31,
if len(args) == 0: raise ValueError("Please specify the target") with verify_and_connect(opts) as kat: # check for PTUSE proxies in the subarray ptuses = [ kat.children[name] for name in sorted(kat.children) if name.startswith('ptuse') ] if len(ptuses) == 0: raise ValueError("This script requires a PTUSE proxy") elif len(ptuses) > 1: # for now keep script simple and only use first proxy ptuse = ptuses[0] user_logger.warning('Found PTUSE proxies: %s - only using: %s', ptuses, ptuse.name) else: ptuse = ptuses[0] user_logger.info('Using PTUSE proxy: %s', ptuse.name) bf_ants = opts.ants.split(',') if opts.ants else [ ant.name for ant in kat.ants ] cbf = SessionCBF(kat) # Special hack for Lab CBF - set to zero on site # TODO: stop hacking, or make this a command line option freq_delta_hack = 0 if freq_delta_hack: user_logger.warning( 'Hack: Adjusting beam centre frequency by %s MHz for lab', freq_delta_hack)
def get_offset_gains(session, offsets, offset_end_times, track_duration): """Extract gains per pointing offset, per receptor and per frequency chunk. Parameters ---------- session : :class:`katcorelib.observe.CaptureSession` object The active capture session offsets : sequence of *N* pairs of float (i.e. shape (*N*, 2)) Requested (x, y) pointing offsets relative to target, in degrees offset_end_times : sequence of *N* floats Unix timestamp at the end of each pointing track track_duration : float Duration of each pointing track, in seconds Returns ------- data_points : dict mapping receptor index to (x, y, freq, gain, weight) seq Complex gains per receptor, as multiple records per offset and frequency """ cal_channel_freqs = session.get_cal_channel_freqs() chunk_freqs = cal_channel_freqs.reshape(NUM_CHUNKS, -1).mean(axis=1) pols = session.telstate['cal_pol_ordering'] data_points = {} # Iterate over offset pointings for offset, offset_end in zip(offsets, offset_end_times): offset_start = offset_end - track_duration # Obtain interferometric gains per pointing from the cal pipeline try: bp_gains = session.get_cal_solutions('B', start_time=offset_start, end_time=offset_end) gains = session.get_cal_solutions('G', start_time=offset_start, end_time=offset_end) except CalSolutionsUnavailable as err: user_logger.warning('Skipping offset %s: %s', offset, err) continue # Iterate over receptors for a, ant in enumerate(session.observers): pol_gain = np.zeros(NUM_CHUNKS) pol_weight = np.zeros(NUM_CHUNKS) # Iterate over polarisations (effectively over inputs) for pol in pols: inp = ant.name + pol bp_gain = bp_gains.get(inp) gain = gains.get(inp) if bp_gain is None or gain is None: continue masked_gain = np.ma.masked_invalid(bp_gain * gain) abs_gain_chunked = np.abs(masked_gain).reshape(NUM_CHUNKS, -1) abs_gain_mean = abs_gain_chunked.mean(axis=1) abs_gain_std = abs_gain_chunked.std(axis=1) abs_gain_var = abs_gain_std.filled(np.inf)**2 # Replace any zero variance with the smallest non-zero variance # across chunks, but if all are zero it is fishy and ignored. zero_var = abs_gain_var == 0. if all(zero_var): abs_gain_var = np.ones_like(abs_gain_var) * np.inf else: abs_gain_var[zero_var] = abs_gain_var[~zero_var].min() # Number of valid samples going into statistics abs_gain_N = (~abs_gain_chunked.mask).sum(axis=1) # Generate standard precision weights based on empirical stdev abs_gain_weight = abs_gain_N / abs_gain_var # Prepare some debugging output stats_mean = ' '.join("%4.2f" % (m, ) for m in abs_gain_mean.filled(np.nan)) stats_std = ' '.join("%4.2f" % (s, ) for s in abs_gain_std.filled(np.nan)) stats_N = ' '.join("%4d" % (n, ) for n in abs_gain_N) bp_mean = np.nanmean(np.abs(bp_gain)) user_logger.debug("%s %s %4.2f mean | %s", tuple(offset), inp, np.abs(gain), stats_mean) user_logger.debug("%s %s %4.2f std | %s", tuple(offset), inp, bp_mean, stats_std) user_logger.debug("%s %s N | %s", tuple(offset), inp, stats_N) # Blend new gains into existing via weighted averaging. # XXX We currently combine HH and VV gains at the start to get # Stokes I gain but in future it might be better to fit # separate beams to HH and VV. pol_gain, pol_weight = np.ma.average( np.c_[pol_gain, abs_gain_mean], axis=1, weights=np.c_[pol_weight, abs_gain_weight], returned=True) if pol_weight.sum() > 0: # Turn masked values into NaNs pre-emptively to avoid warning # when recarray in beam fitting routine forces this later on. pol_gain = pol_gain.filled(np.nan) data = data_points.get(a, []) for freq, gain, weight in zip(chunk_freqs, pol_gain, pol_weight): data.append((offset[0], offset[1], freq, gain, weight)) data_points[a] = data if not data_points: raise CalSolutionsUnavailable("No complete gain solutions found in " "telstate for any offset") return data_points
session.label('un_corrected') user_logger.info("Initiating %g-second track on target '%s'", opts.track_duration, target.name) session.track(target, duration=opts.track_duration, announce=False) # Attempt to jiggle cal pipeline to drop its gains session.stop_antennas() user_logger.info("Waiting for gains to materialise in cal pipeline") # Wait for the last bfcal product from the pipeline gains = session.get_cal_solutions('G', timeout=opts.track_duration) bp_gains = session.get_cal_solutions('B') delays = session.get_cal_solutions('K') cal_channel_freqs = session.get_cal_channel_freqs() bp_gains = clean_bandpass(bp_gains, cal_channel_freqs, max_gap_Hz=64e6) if opts.random_phase: user_logger.warning("Setting F-engine gains with random phases " "(you asked for it)") else: user_logger.info("Setting F-engine gains to phase up antennas") if not kat.dry_run: corrections = calculate_corrections(gains, bp_gains, delays, cal_channel_freqs, opts.random_phase, opts.flatten_bandpass, fengine_gain) session.set_fengine_gains(corrections) if opts.verify_duration > 0: user_logger.info("Revisiting target %r for %g seconds to verify phase-up", target.name, opts.verify_duration) session.label('corrected') session.track(target, duration=opts.verify_duration, announce=False) if not opts.random_phase: # Set last-phaseup script sensor on the subarray.
keep_going = True while keep_going: keep_going = opts.max_duration is not None targets_before_loop = len(targets_observed) # Iterate through source list, picking the next one that is up bgain_list = np.linspace(b_start, b_end, b_step) for bgain in bgain_list: for target in targets.iterfilter(el_limit_deg=opts.horizon): duration = opts.track_duration if opts.max_duration is not None: time_left = opts.max_duration - (time.time() - start_time) # Cut the track short if time runs out if time_left <= 0.: user_logger.warning( "Maximum duration of %g seconds " "has elapsed - stopping script", opts.max_duration) keep_going = False break duration = min(duration, time_left) # Set the b-gain session.label('track_bgain, %g' % bgain) for stream in cbf.beamformers: stream.req.quant_gains(bgain) user_logger.info( "B-engine %s quantisation gain set to %g", stream, bgain) if session.track(target, duration=duration): targets_observed.append(target.description) if keep_going and len(targets_observed) == targets_before_loop: user_logger.warning(
actual_centre_freq = float(reply.messages[0].arguments[3]) user_logger.info( "Beamformer %r has bandwidth %g Hz and centre freq %g Hz", stream, actual_bandwidth, actual_centre_freq) else: raise ValueError("Could not set beamformer %r passband - (%s)" % (stream, ' '.join(reply.messages[0].arguments))) user_logger.info('Setting beamformer weights for stream %r:', stream) for inp in stream.inputs: weight = 1.0 / np.sqrt( len(bf_ants)) if inp[:-1] in bf_ants else 0.0 reply = stream.req.weights(inp, weight) if reply.succeeded: user_logger.info(' input %r got weight %f', inp, weight) else: user_logger.warning(' input %r weight could not be set', inp) # We are only interested in first target target_name = args[:1][0] print target_name user_logger.info('Looking up main beamformer target...') target = collect_targets(kat, args[:1]).targets[0] # Ensure that the target is up target_elevation = np.degrees(target.azel()[1]) if target_elevation < opts.horizon: raise ValueError("The target %r is below the horizon" % (target.description, )) # Verify backend_args
# Check options and build KAT configuration, connecting to proxies and devices with verify_and_connect(opts) as kat: ants = kat.ants obs_ants = [ant.name for ant in ants] observation_sources = collect_targets(kat,args) # Find out what inputs are curremtly active reply = kat.data.req.dbe_label_input() inputs = [m.arguments[0] for m in reply.messages[3:]] user_logger.info("Resetting f-engine gains to 160 to allow phasing up") for inp in inputs: kat.data.req.dbe_k7_gain(inp,160) # Quit early if there are no sources to observe if len(observation_sources.filter(el_limit_deg=opts.horizon)) == 0: user_logger.warning("No targets are currently visible - please re-run the script later") else: # Start capture session, which creates HDF5 file with start_session(kat, **vars(opts)) as session: session.standard_setup(**vars(opts)) session.capture_start() start_time = time.time() targets_observed = [] # Keep going until the time is up keep_going = True while keep_going: for target in observation_sources.iterfilter(el_limit_deg=opts.horizon): if target.flux_model is None: user_logger.warning("Target has no flux model - stopping script") keep_going=False
raise ValueError( "Could not set beamformer %r passband - (%s)" % (stream.name, ' '.join(reply.messages[0].arguments))) user_logger.info('Setting beamformer weights for stream %r:', stream.name) weights = [] for inp in stream.inputs: weight = 1.0 if inp[:-1] in bf_ants else 0.0 weights.append(weight) user_logger.info(' input %r will get weight %f', inp, weight) reply = stream.req.weights(*weights) if reply.succeeded: user_logger.info('Set input weights successfully') else: user_logger.warning('Failed to set input weights!') # We are only interested in first target user_logger.info('Looking up main beamformer target...') tgt_with_spaces = [tgt for tgt in args[:1] if len(tgt.strip().split()) > 1] if len(tgt_with_spaces) > 0: user_logger.error( "Please replace '%s' with '%s'", '& '.join(tgt_with_spaces), '& '.join( [''.join(tgt.strip().split()) for tgt in tgt_with_spaces])) raise ValueError( 'Found spaces in target names, which will cause an error in digifits' ) target = collect_targets(kat, args[:1]).targets[0]
"please re-run the script later") session.standard_setup(**vars(opts)) if opts.fft_shift is not None: session.cbf.fengine.req.fft_shift(opts.fft_shift) session.cbf.correlator.req.capture_start() for target in [observation_sources.sort('el').targets[-1]]: target.add_tags('bfcal single_accumulation') print target.flux_model print target.name if not opts.default_gain: channels = 32768 if session.product.endswith('32k') else 4096 opts.default_gain = DEFAULT_GAIN[channels] user_logger.info("Target to be observed: %s", target.description) if target.flux_model is None: user_logger.warning("Target has no flux model (katsdpcal will need it in future)") user_logger.info("Resetting F-engine gains to %g to allow phasing up", opts.default_gain) for inp in session.cbf.fengine.inputs: session.cbf.fengine.req.gain(inp, opts.default_gain) session.label('un_corrected') user_logger.info("Initiating %g-second track on target '%s'", opts.track_duration, target.name) session.track(target, duration=opts.track_duration, announce=False) # Attempt to jiggle cal pipeline to drop its gains session.ants.req.target('') user_logger.info("Waiting for gains to materialise in cal pipeline") delays = bp_gains = gains = {} cal_channel_freqs = None if not kat.dry_run: # Wait for the last bfcal product from the pipeline