def test_pointing_model_load_save(self): """Test construction / load / save of pointing model.""" params = katpoint.deg2rad(np.random.randn(self.num_params + 1)) pm = katpoint.PointingModel(params[:-1]) print repr(pm), pm pm2 = katpoint.PointingModel(params[:-2]) self.assertEqual(pm2.values()[-1], 0.0, 'Unspecified pointing model params not zeroed') pm3 = katpoint.PointingModel(params) self.assertEqual(pm3.values()[-1], params[-2], 'Superfluous pointing model params not handled correctly') pm4 = katpoint.PointingModel(pm.description) self.assertEqual(pm4.description, pm.description, 'Saving pointing model to string and loading it again failed') self.assertEqual(pm4, pm, 'Pointing models should be equal') self.assertNotEqual(pm2, pm, 'Pointing models should be inequal') np.testing.assert_almost_equal(pm4.values(), pm.values(), decimal=6)
def pointing_model(antenna, data): new_model = katpoint.PointingModel() num_params = len(new_model) default_enabled = np.array([1, 3, 4, 5, 6, 7, 8]) - 1 enabled_params = np.tile(False, num_params) enabled_params[default_enabled] = True enabled_params = enabled_params.tolist() # For display purposes, throw out unused parameters P2 and P10 display_params = list(range(num_params)) display_params.pop(9) display_params.pop(1) # Fit new pointing model az, el = data['azimuth'], data['elevation'] measured_delta_az, measured_delta_el = data['delta_azimuth'], data[ 'delta_elevation'] # Uncertainties are optional min_std = deg2rad((np.sqrt(2) * 60. * 1e-12) / 60. / np.sqrt(2)) std_delta_az = np.clip(data['delta_azimuth_std'], min_std, np.inf) \ if 'delta_azimuth_std' in data.dtype.fields else np.tile(min_std, len(az)) std_delta_el = np.clip(data['delta_elevation_std'], min_std, np.inf) \ if 'delta_elevation_std' in data.dtype.fields else np.tile(min_std, len(el)) params, sigma_params = new_model.fit(az, el, measured_delta_az, measured_delta_el, std_delta_az, std_delta_el, enabled_params) antenna.pointing_model = new_model return antenna
def setUp(self): az_range = katpoint.deg2rad(np.arange(-185.0, 275.0, 5.0)) el_range = katpoint.deg2rad(np.arange(0.0, 86.0, 1.0)) mesh_az, mesh_el = np.meshgrid(az_range, el_range) self.az = mesh_az.ravel() self.el = mesh_el.ravel() # Generate random parameter values with this spread self.param_stdev = katpoint.deg2rad(20. / 60.) self.num_params = len(katpoint.PointingModel())
def test_pointing_closure(self): """Test closure between pointing correction and its reverse operation.""" # Generate random pointing model params = self.param_stdev * np.random.randn(self.num_params) pm = katpoint.PointingModel(params) # Test closure on (az, el) grid pointed_az, pointed_el = pm.apply(self.az, self.el) az, el = pm.reverse(pointed_az, pointed_el) assert_angles_almost_equal(az, self.az, decimal=6, err_msg='Azimuth closure error for params=%s' % (params,)) assert_angles_almost_equal(el, self.el, decimal=7, err_msg='Elevation closure error for params=%s' % (params,))
def test_pointing_fit(self): """Test fitting of pointing model.""" # Generate random pointing model and corresponding offsets on (az, el) grid params = self.param_stdev * np.random.randn(self.num_params) params[1] = params[9] = 0.0 pm = katpoint.PointingModel(params.copy()) delta_az, delta_el = pm.offset(self.az, self.el) enabled_params = (np.arange(self.num_params) + 1).tolist() # Comment out these removes, thereby testing more code paths in PointingModel # enabled_params.remove(2) # enabled_params.remove(10) # pylint: disable-msg=W0612 fitted_params, sigma_params = pm.fit(self.az, self.el, delta_az, delta_el, enabled_params=[]) np.testing.assert_equal(fitted_params, np.zeros(self.num_params)) fitted_params, sigma_params = pm.fit(self.az, self.el, delta_az, delta_el, enabled_params=enabled_params) np.testing.assert_almost_equal(fitted_params, params, decimal=9)
def test_construct_antenna(self): """Test construction of antennas from strings and vice versa.""" valid_antennas = [ katpoint.Antenna(descr) for descr in self.valid_antennas ] valid_strings = [a.description for a in valid_antennas] for descr in valid_strings: ant = katpoint.Antenna(descr) print('%s %s' % (str(ant), repr(ant))) self.assertEqual( descr, ant.description, 'Antenna description differs from original string') self.assertEqual(ant.description, ant.format_katcp(), 'Antenna description differs from KATCP format') for descr in self.invalid_antennas: self.assertRaises(ValueError, katpoint.Antenna, descr) descr = valid_antennas[0].description self.assertEqual(descr, katpoint.Antenna(*descr.split(', ')).description) self.assertRaises(ValueError, katpoint.Antenna, descr, *descr.split(', ')[1:]) # Check that description string updates when object is updated a1 = katpoint.Antenna( 'FF1, -30:43:17.3, 21:24:38.5, 1038.0, 12.0, 18.4 -8.7 0.0') a2 = katpoint.Antenna( 'FF2, -30:43:17.3, 21:24:38.5, 1038.0, 13.0, 18.4 -8.7 0.0, 0.1, 1.22' ) self.assertNotEqual(a1, a2, 'Antennas should be inequal') a1.name = 'FF2' a1.diameter = 13.0 a1.pointing_model = katpoint.PointingModel('0.1') a1.beamwidth = 1.22 self.assertEqual(a1.description, a2.description, 'Antenna description string not updated') self.assertEqual(a1, a2.description, 'Antenna not equal to description string') self.assertEqual(a1, a2, 'Antennas not equal') self.assertEqual(a1, katpoint.Antenna(a2), 'Construction with antenna object failed') self.assertEqual(a1, pickle.loads(pickle.dumps(a1)), 'Pickling failed') try: self.assertEqual(hash(a1), hash(a2), 'Antenna hashes not equal') except TypeError: self.fail('Antenna object not hashable')
# Set up logging: logging everything (DEBUG & above) logging.basicConfig(level=logging.DEBUG, stream=sys.stdout, format="%(levelname)s: %(message)s") logger = logging.root logger.setLevel(logging.DEBUG) # Load old pointing model, if given old_model = None if opts.pmfilename: try: old_model = file(opts.pmfilename).readline().strip() logger.debug("Loaded %d-parameter pointing model from '%s'" % (len(old_model.split(',')), opts.pmfilename)) old_model = katpoint.PointingModel(old_model, strict=False) except IOError: logger.warning("Could not load old pointing model from '%s'" % (opts.pmfilename, )) # Load data file in one shot as an array of strings data = np.loadtxt(filename, dtype='string', comments='#', delimiter=', ') # Interpret first non-comment line as header fields = data[0].tolist() # By default, all fields are assumed to contain floats formats = np.tile(np.float, len(fields)) # The string_fields are assumed to contain strings - use data's string type, as it is of sufficient length formats[[fields.index(name) for name in string_fields if name in fields]] = data.dtype # Convert to heterogeneous record array data = np.rec.fromarrays(data[1:].transpose(), dtype=zip(fields, formats))
def analyse_point_source_scans(filename, opts): dataset_name = os.path.splitext(os.path.basename(filename))[0] # Default output file names are based on input file name if opts.outfilebase is None: opts.outfilebase = dataset_name + '_point_source_scans' # Set up logging: logging everything (DEBUG & above), both to console and file logger = logging.root logger.setLevel(logging.DEBUG) fh = logging.FileHandler(opts.outfilebase + '.log', 'w') fh.setLevel(logging.DEBUG) fh.setFormatter(logging.Formatter('%(levelname)s: %(message)s')) logger.addHandler(fh) # Produce canonical version of baseline string (remove duplicate antennas) baseline_ants = opts.baseline.split(',') if len(baseline_ants) == 2 and baseline_ants[0] == baseline_ants[1]: opts.baseline = baseline_ants[0] # Load old CSV file used to select compound scans from dataset keep_scans = keep_datasets = None if opts.keepfilename: ant_name = katpoint.Antenna( file(opts.keepfilename).readline().strip().partition('=')[2]).name try: data = np.loadtxt(opts.keepfilename, dtype='string', comments='#', delimiter=', ') except ValueError: raise ValueError( "CSV file '%s' contains rows with a different number of columns/commas" % opts.keepfilename) try: fields = data[0].tolist() id_fields = [ fields.index('dataset'), fields.index('target'), fields.index('timestamp_ut') ] except (IndexError, ValueError): raise ValueError("CSV file '%s' do not have the expected columns" % opts.keepfilename) keep_scans = set( [ant_name + ' ' + ' '.join(line) for line in data[1:, id_fields]]) keep_datasets = set(data[1:, id_fields[0]]) # Switch to batch mode if CSV file is given opts.batch = True logger.debug( "Loaded CSV file '%s' containing %d dataset(s) and %d compscan(s) for antenna '%s'" % (opts.keepfilename, len(keep_datasets), len(keep_scans), ant_name)) # Ensure we are using antenna found in CSV file (this assumes single dish setup for now) csv_baseline = ant_name if opts.baseline != 'sd' and opts.baseline != csv_baseline: logger.warn( "Requested baseline '%s' does not match baseline '%s' in CSV file '%s'" % (opts.baseline, csv_baseline, opts.keepfilename)) logger.warn("Using baseline '%s' found in CSV file '%s'" % (csv_baseline, opts.keepfilename)) opts.baseline = csv_baseline # Avoid loading the data set if it does not appear in specified CSV file if keep_datasets and dataset_name not in keep_datasets: raise RuntimeError("Skipping dataset '%s' (based on CSV file)" % (filename, )) # Load data set logger.info("Loading dataset '%s'" % (filename, )) dataset = scape.DataSet(filename, baseline=opts.baseline, nd_models=opts.nd_models, time_offset=opts.time_offset, katfile=not opts.old_loader) # Select frequency channels and setup defaults if not specified num_channels = len(dataset.channel_select) if opts.freq_chans is None: # Default is drop first and last 25% of the bandpass start_chan = num_channels // 4 end_chan = start_chan * 3 else: start_chan = int(opts.freq_chans.split(',')[0]) end_chan = int(opts.freq_chans.split(',')[1]) chan_range = range(start_chan, end_chan + 1) dataset = dataset.select(freqkeep=chan_range) # Check scan count if len(dataset.compscans) == 0 or len(dataset.scans) == 0: raise RuntimeError('No scans found in file, skipping data set') scan_dataset = dataset.select(labelkeep='scan', copy=False) if len(scan_dataset.compscans) == 0 or len(scan_dataset.scans) == 0: raise RuntimeError( 'No scans left after standard reduction, skipping data set (no scans labelled "scan", perhaps?)' ) # Override pointing model if it is specified (useful if it is not in data file, like on early KAT-7) if opts.pointing_model: pm = file(opts.pointing_model).readline().strip() logger.debug("Loaded %d-parameter pointing model from '%s'" % (len(pm.split(',')), opts.pointing_model)) dataset.antenna.pointing_model = katpoint.PointingModel(pm, strict=False) # Initialise the output data cache (None indicates the compscan has not been processed yet) reduced_data = [{} for n in range(len(scan_dataset.compscans))] ### BATCH MODE ### # This will cycle through all data sets and stop when done if opts.batch: # Go one past the end of compscan list to write the output data out to CSV file for current_compscan in range(len(scan_dataset.compscans) + 1): # Look up compscan key in list of compscans to keep (if provided, only applicable to batch mode anyway) if keep_scans and (current_compscan < len(scan_dataset.compscans)): cs_key = ' '.join( compscan_key(scan_dataset.compscans[current_compscan])) if cs_key not in keep_scans: logger.info( "==== Skipping compound scan '%s' (based on CSV file) ====" % (cs_key, )) continue output = reduce_and_plot(dataset, current_compscan, reduced_data, opts, logger=logger) return output ### INTERACTIVE MODE ### else: if not plt: raise ImportError( 'Interactive use of this script requires matplotlib - please install it or run in batch mode' ) # Set up figure with buttons plt.ion() fig = plt.figure(1) plt.clf() if opts.plot_spectrum: plt.subplot(311) plt.subplot(312) plt.subplot(313) else: plt.subplot(211) plt.subplot(212) plt.subplots_adjust(bottom=0.2, hspace=0.25) plt.figtext(0.05, 0.05, '', va='bottom', ha='left') plt.figtext(0.05, 0.945, '', va='bottom', ha='left') # Make button context manager that disables buttons during processing and re-enables it afterwards class DisableButtons(object): def __init__(self): """Start with empty button list.""" self.buttons = [] def append(self, button): """Add button to list.""" self.buttons.append(button) def __enter__(self): """Disable buttons on entry.""" if plt.fignum_exists(1): for button in self.buttons: button.eventson = False button.hovercolor = '0.85' button.label.set_color('gray') plt.draw() def __exit__(self, exc_type, exc_value, traceback): """Re-enable buttons on exit.""" if plt.fignum_exists(1): for button in self.buttons: button.eventson = True button.hovercolor = '0.95' button.label.set_color('k') plt.draw() all_buttons = DisableButtons() # Create buttons and their callbacks spectrogram_button = widgets.Button(plt.axes([0.37, 0.05, 0.1, 0.075]), 'Spectrogram') def spectrogram_callback(event): with all_buttons: plt.figure(2) plt.clf() out = reduced_data[fig.current_compscan] ax = scape.plot_xyz(out['unavg_dataset'], 'time', 'freq', 'amp', power_in_dB=True) ax.set_title(out['target'], size='medium') spectrogram_button.on_clicked(spectrogram_callback) all_buttons.append(spectrogram_button) keep_button = widgets.Button(plt.axes([0.48, 0.05, 0.1, 0.075]), 'Keep') def keep_callback(event): with all_buttons: reduced_data[fig.current_compscan]['keep'] = True fig.current_compscan += 1 reduce_and_plot(dataset, fig.current_compscan, reduced_data, opts, fig, logger=logger) keep_button.on_clicked(keep_callback) all_buttons.append(keep_button) discard_button = widgets.Button(plt.axes([0.59, 0.05, 0.1, 0.075]), 'Discard') def discard_callback(event): with all_buttons: reduced_data[fig.current_compscan]['keep'] = False fig.current_compscan += 1 reduce_and_plot(dataset, fig.current_compscan, reduced_data, opts, fig, logger=logger) discard_button.on_clicked(discard_callback) all_buttons.append(discard_button) back_button = widgets.Button(plt.axes([0.7, 0.05, 0.1, 0.075]), 'Back') def back_callback(event): with all_buttons: if fig.current_compscan > 0: fig.current_compscan -= 1 reduce_and_plot(dataset, fig.current_compscan, reduced_data, opts, fig, logger=logger) back_button.on_clicked(back_callback) all_buttons.append(back_button) done_button = widgets.Button(plt.axes([0.81, 0.05, 0.1, 0.075]), 'Done') def done_callback(event): with all_buttons: fig.current_compscan = len(reduced_data) reduce_and_plot(dataset, fig.current_compscan, reduced_data, opts, fig, logger=logger) done_button.on_clicked(done_callback) all_buttons.append(done_button) # Start off the processing on the first compound scan fig.current_compscan = 0 reduce_and_plot(dataset, fig.current_compscan, reduced_data, opts, fig, logger=logger) # Display plots - this should be called ONLY ONCE, at the VERY END of the script # The script stops here until you close the plots... plt.show()
text.append("List of targets used:") for tar in list(set(data['target'])): if i % linelength == linelength - 1: text.append(tmpstr) tmpstr = "" i = i + 1 tmpstr += '%s, ' % (tar) text.append(tmpstr) ##################################### # Load old pointing model, if given. ##################################### old_model = None if opts.pmfilename: try: old_model = katpoint.PointingModel(file(opts.pmfilename).readline()) logger.debug("Loaded %d-parameter pointing model from '%s'" % (len(old_model), opts.pmfilename)) except IOError: logger.warning("Could not load old pointing model from '%s'" % (opts.pmfilename, )) # If the antenna has no model specified, a default null model will be used antenna = katpoint.Antenna(file(filename).readline().strip().partition('=')[2]) if old_model is None: old_model = antenna.pointing_model targets = data['target'] #keep = data['keep'].astype(np.bool) if 'keep' in data.dtype.fields else np.tile(True, len(targets)) ########################################## # Initialise new pointing model and set
dataset = dataset.select(freqkeep=chan_range) # Check scan count if len(dataset.compscans) == 0 or len(dataset.scans) == 0: raise RuntimeError('No scans found in file, skipping data set') scan_dataset = dataset.select(labelkeep='scan', copy=False) if len(scan_dataset.compscans) == 0 or len(scan_dataset.scans) == 0: raise RuntimeError( 'No scans left after standard reduction, skipping data set (no scans labelled "scan", perhaps?)' ) # Override pointing model if it is specified (useful if it is not in data file, like on early KAT-7) if opts.pointing_model: pm = file(opts.pointing_model).readline().strip() logger.debug("Loaded %d-parameter pointing model from '%s'" % (len(pm.split(',')), opts.pointing_model)) dataset.antenna.pointing_model = katpoint.PointingModel(pm, strict=False) # Initialise the output data cache (None indicates the compscan has not been processed yet) reduced_data = [None] * len(scan_dataset.compscans) ### BATCH MODE ### # This will cycle through all data sets and stop when done if opts.batch: # Go one past the end of compscan list to write the output data out to CSV file for current_compscan in range(len(scan_dataset.compscans) + 1): # Look up compscan key in list of compscans to keep (if provided, only applicable to batch mode anyway) if keep_scans and (current_compscan < len(scan_dataset.compscans)): cs_key = ' '.join( compscan_key(scan_dataset.compscans[current_compscan])) if cs_key not in keep_scans:
def analyse_point_source_scans(filename, h5file, opts): # Default output file names are based on input file name dataset_name = os.path.splitext(os.path.basename(filename))[0] if opts.outfilebase is None: opts.outfilebase = dataset_name + '_point_source_scans' kwargs = {} #Force centre freqency if ku-band option is set if opts.ku_band: kwargs['centre_freq'] = 12.5005e9 # Produce canonical version of baseline string (remove duplicate antennas) baseline_ants = opts.baseline.split(',') if len(baseline_ants) == 2 and baseline_ants[0] == baseline_ants[1]: opts.baseline = baseline_ants[0] # Load data set if opts.baseline not in [ant.name for ant in h5file.ants]: raise RuntimeError('Cannot find antenna %s in dataset' % opts.baseline) # dataset = scape.DataSet(h5file, baseline=opts.baseline, nd_models=opts.nd_models, # time_offset=opts.time_offset, **kwargs) dataset = scape.DataSet(filename, baseline=opts.baseline, nd_models=opts.nd_models, time_offset=opts.time_offset, **kwargs) # Select frequency channels and setup defaults if not specified num_channels = len(dataset.channel_select) if opts.freq_chans is None: # Default is drop first and last 25% of the bandpass start_chan = num_channels // 4 end_chan = start_chan * 3 else: start_chan = int(opts.freq_chans.split(',')[0]) end_chan = int(opts.freq_chans.split(',')[1]) chan_select = list(range(start_chan, end_chan + 1)) # Check if a channel mask is specified and apply if opts.channel_mask: mask_file = open(opts.channel_mask, mode='rb') chan_select = ~(pickle.load(mask_file)) mask_file.close() if len(chan_select) != num_channels: raise ValueError( 'Number of channels in provided mask does not match number of channels in data' ) chan_select[:start_chan] = False chan_select[end_chan:] = False dataset = dataset.select(freqkeep=chan_select) # Check scan count if len(dataset.compscans) == 0 or len(dataset.scans) == 0: raise RuntimeError('No scans found in file, skipping data set') scan_dataset = dataset.select(labelkeep='scan', copy=False) if len(scan_dataset.compscans) == 0 or len(scan_dataset.scans) == 0: raise RuntimeError( 'No scans left after standard reduction, skipping data set (no scans labelled "scan", perhaps?)' ) # Override pointing model if it is specified (useful if it is not in data file, like on early KAT-7) if opts.pointing_model: if opts.pointing_model.split('/')[-2] == 'mkat': if opts.ku_band: band = 'ku' else: band = 'l' pt_file = os.path.join(opts.pointing_model, '%s.%s.pm.csv' % (opts.baseline, band)) else: pt_file = os.path.join(opts.pointing_model, '%s.pm.csv' % (opts.baseline)) if not os.path.isfile(pt_file): raise RuntimeError('Cannot find file %s' % (pt_file)) pm = file(pt_file).readline().strip() dataset.antenna.pointing_model = katpoint.PointingModel(pm) # Remove any noise diode models if the ku band option is set and flag for spikes if opts.ku_band: dataset.nd_h_model = None dataset.nd_v_model = None for i in range(len(dataset.scans)): dataset.scans[i].data = scape.stats.remove_spikes( dataset.scans[i].data, axis=1, spike_width=3, outlier_sigma=5.) # Initialise the output data cache (None indicates the compscan has not been processed yet) reduced_data = [{} for n in range(len(scan_dataset.compscans))] # Go one past the end of compscan list to write the output data out to CSV file for current_compscan in range(len(scan_dataset.compscans) + 1): # make things play nice opts.batch = True try: the_compscan = scan_dataset.compscans[current_compscan] except: the_compscan = None fig = plt.figure(1, figsize=(8, 8)) plt.clf() if opts.plot_spectrum: plt.subplot(311) plt.subplot(312) plt.subplot(313) else: plt.subplot(211) plt.subplot(212) plt.subplots_adjust(bottom=0.2, hspace=0.25) plt.figtext(0.05, 0.05, '', va='bottom', ha='left') plt.figtext(0.05, 0.945, '', va='bottom', ha='left') # Start off the processing on the first compound scan logger = logging.root fig.current_compscan = 0 reduce_and_plot(dataset, fig.current_compscan, reduced_data, opts, fig, logger=logger) # Initialise the output data cache (None indicates the compscan has not been processed yet) reduced_data = [{} for n in range(len(scan_dataset.compscans))] # Go one past the end of compscan list to write the output data out to CSV file for current_compscan in range(len(scan_dataset.compscans) + 1): # make things play nice opts.batch = True try: the_compscan = scan_dataset.compscans[current_compscan] except: the_compscan = None logger = logging.root output = local_reduce_and_plot(dataset, current_compscan, reduced_data, opts, logger=logger) offsetdata = output[1] from katpoint import deg2rad def angle_wrap(angle, period=2.0 * np.pi): """wrap angle into the interval -*period* / 2 ... *period* / 2.""" return (angle + 0.5 * period) % period - 0.5 * period az, el = angle_wrap(deg2rad(offsetdata['azimuth'])), deg2rad( offsetdata['elevation']) model_delta_az, model_delta_el = ant.pointing_model.offset(az, el) measured_delta_az = offsetdata[ 'delta_azimuth'] - model_delta_az # pointing model correction measured_delta_el = offsetdata[ 'delta_elevation'] - model_delta_el # pointing model correction """determine new residuals from current pointing model""" residual_az = measured_delta_az - model_delta_az residual_el = measured_delta_el - model_delta_el residual_xel = residual_az * np.cos(el) # Initialise new pointing model and set default enabled parameters keep = np.ones((len(offsetdata)), dtype=np.bool) min_rms = np.sqrt(2) * 60. * 1e-12 use_stats = True new_model = katpoint.PointingModel() num_params = len(new_model) default_enabled = np.array([1, 3, 4, 5, 6, 7]) - 1 enabled_params = np.tile(False, num_params) enabled_params[default_enabled] = True enabled_params = enabled_params.tolist() # Fit new pointing model az, el = angle_wrap(deg2rad(offsetdata['azimuth'])), deg2rad( offsetdata['elevation']) measured_delta_az, measured_delta_el = deg2rad( offsetdata['delta_azimuth']), deg2rad(offsetdata['delta_elevation']) # Uncertainties are optional min_std = deg2rad(min_rms / 60. / np.sqrt(2)) std_delta_az = np.clip(deg2rad(offsetdata['delta_azimuth_std']), min_std, np.inf) \ if 'delta_azimuth_std' in offsetdata.dtype.fields and use_stats else np.tile(min_std, len(az)) std_delta_el = np.clip(deg2rad(offsetdata['delta_elevation_std']), min_std, np.inf) \ if 'delta_elevation_std' in offsetdata.dtype.fields and use_stats else np.tile(min_std, len(el)) params, sigma_params = new_model.fit(az[keep], el[keep], measured_delta_az[keep], measured_delta_el[keep], std_delta_az[keep], std_delta_el[keep], enabled_params) """Determine new residuals from new fit""" newmodel_delta_az, newmodel_delta_el = new_model.offset(az, el) residual_az = measured_delta_az - newmodel_delta_az residual_el = measured_delta_el - newmodel_delta_el residual_xel = residual_az * np.cos(el) # Show actual scans h5file.select(scans='scan') fig1 = plt.figure(2, figsize=(8, 8)) plt.scatter(h5file.ra, h5file.dec, s=np.mean(np.abs(h5file.vis[:, 2200:2400, 1]), axis=1)) plt.title('Raster scan over target') plt.ylabel('Dec [deg]') plt.xlabel('Ra [deg]') # Try to fit beam for c in h5file.compscans(): if not dataset is None: dataset = dataset.select(flagkeep='~nd_on') dataset.average() dataset.fit_beams_and_baselines() # Generate output report with PdfPages(opts.outfilebase + '_' + opts.baseline + '.pdf') as pdf: out = reduced_data[0] offset_az, offset_el = "%.1f" % ( 60. * out['delta_azimuth'], ), "%.1f" % (60. * out['delta_elevation'], ) beam_width, beam_height = "%.1f" % ( 60. * out['beam_width_I'], ), "%.2f" % (out['beam_height_I'], ) baseline_height = "%.1f" % (out['baseline_height_I'], ) pagetext = "\nCheck Point Source Scan" pagetext += "\n\nDescription: %s\nName: %s\nExperiment ID: %s" % ( h5file.description, h5file.name, h5file.experiment_id) pagetext = pagetext + "\n" pagetext += "\n\nTest Setup:" pagetext += "\nRaster Scan across bright source" pagetext += "\n\nAntenna %(antenna)s" % out pagetext += "\n------------" pagetext += ("\nTarget = '%(target)s', azel=(%(azimuth).1f, %(elevation).1f) deg, " % out) +\ ("offset=(%s, %s) arcmin" % (offset_az, offset_el)) pagetext += ("\nBeam height = %s %s") % (beam_height, out['data_unit']) pagetext += ("\nBeamwidth = %s' (expected %.1f')") % ( beam_width, 60. * out['beam_expected_width_I']) pagetext += ("\nHH gain = %.3f Jy/%s") % ( out['flux'] / out['beam_height_HH'], out['data_unit']) pagetext += ("\nVV gain = %.3f Jy/%s") % ( out['flux'] / out['beam_height_VV'], out['data_unit']) pagetext += ("\nBaseline height = %s %s") % (baseline_height, out['data_unit']) pagetext = pagetext + "\n" pagetext += ("\nCurrent model AzEl=(%.3f, %.3f) deg" % (model_delta_az[0], model_delta_el[0])) pagetext += ("\nMeasured coordinates using rough fit") pagetext += ("\nMeasured AzEl=(%.3f, %.3f) deg" % (measured_delta_az[0], measured_delta_el[0])) pagetext = pagetext + "\n" pagetext += ("\nDetermine residuals from current pointing model") residual_az = measured_delta_az - model_delta_az residual_el = measured_delta_el - model_delta_el pagetext += ("\nResidual AzEl=(%.3f, %.3f) deg" % (residual_az[0], residual_el[0])) if dataset.compscans[0].beam is not None: if not dataset.compscans[0].beam.is_valid: pagetext += ("\nPossible bad fit!") if (residual_az[0] < 1.) and (residual_el[0] < 1.): pagetext += ("\nResiduals withing L-band beam") else: pagetext += ("\nMaximum Residual, %.2f, larger than L-band beam" % (numpy.max(residual_az[0], residual_el[0]))) pagetext = pagetext + "\n" pagetext += ("\nFitted parameters \n%s" % str(params[:5])) plt.figure(None, figsize=(16, 8)) plt.axes(frame_on=False) plt.xticks([]) plt.yticks([]) plt.title("AR1 Report %s" % opts.outfilebase, fontsize=14, fontweight="bold") plt.text(0, 0, pagetext, fontsize=12) pdf.savefig() plt.close() pdf.savefig(fig) pdf.savefig(fig1) d = pdf.infodict() import datetime d['Title'] = h5file.description d['Author'] = 'AR1' d['Subject'] = 'AR1 check point source scan' d['CreationDate'] = datetime.datetime(2015, 8, 13) d['ModDate'] = datetime.datetime.today()
keep[key] = target not in set(offsetdata) i = 0 tmpstr = "" linelength = 5 text.append("List of targets used:") for tar in list(set(data['target'])): if i % linelength == linelength - 1: text.append(tmpstr) tmpstr = "" i = i + 1 tmpstr += '%s, ' % (tar) text.append(tmpstr) # Initialise new pointing model and set default enabled parameters new_model = katpoint.PointingModel() num_params = len(new_model) default_enabled = np.array([1, 3, 4, 5, 6, 7]) - 1 enabled_params = np.tile(False, num_params) enabled_params[default_enabled] = True enabled_params = enabled_params.tolist() # Fit new pointing model az, el = angle_wrap(deg2rad(data['azimuth'])), deg2rad(data['elevation']) measured_delta_az, measured_delta_el = deg2rad(data['delta_azimuth']), deg2rad( data['delta_elevation']) # Uncertainties are optional min_std = deg2rad(min_rms / 60. / np.sqrt(2)) std_delta_az = np.clip(deg2rad(data['delta_azimuth_std']), min_std, np.inf) \ if 'delta_azimuth_std' in data.dtype.fields and opts.use_stats else np.tile(min_std, len(az)) std_delta_el = np.clip(deg2rad(data['delta_elevation_std']), min_std, np.inf) \ if 'delta_elevation_std' in data.dtype.fields and opts.use_stats else np.tile(min_std, len(el))
def fit_pointing_model(filename, opts): # declare the globals to update their values global az global el global measured_delta_az global measured_delta_el global std_delta_az global std_delta_el global keep # These fields contain strings, while the rest of the fields are assumed to contain floats string_fields = ['dataset', 'target', 'timestamp_ut', 'data_unit'] # Load old pointing model, if given old_model = None if opts.pmfilename: try: old_model = katpoint.PointingModel(file(opts.pmfilename).readline()) print("Loaded %d-parameter pointing model from '%s'" % (len(old_model), opts.pmfilename)) except IOError: raise RuntimeError("Could not load old pointing model from '%s'" % (opts.pmfilename,)) # Load data file in one shot as an array of strings data = np.loadtxt(filename, dtype='string', comments='#', delimiter=', ') # Interpret first non-comment line as header fields = data[0].tolist() # By default, all fields are assumed to contain floats formats = np.tile(np.float, len(fields)) # The string_fields are assumed to contain strings - use data's string type, as it is of sufficient length formats[[fields.index(name) for name in string_fields if name in fields]] = data.dtype # Convert to heterogeneous record array data = np.rec.fromarrays(data[1:].transpose(), dtype=list(zip(fields, formats))) # Load antenna description string from first line of file and construct antenna object from it antenna = katpoint.Antenna(file(filename).readline().strip().partition('=')[2]) # Use the pointing model contained in antenna object as the old model (if not overridden by file) # If the antenna has no model specified, a default null model will be used if old_model is None: old_model = antenna.pointing_model # Obtain desired fields and convert to radians az, el = wrap_angle(deg2rad(data['azimuth'])), deg2rad(data['elevation']) measured_delta_az, measured_delta_el = deg2rad(data['delta_azimuth']), deg2rad(data['delta_elevation']) # Uncertainties are optional min_std = deg2rad(opts.min_rms / 60. / np.sqrt(2)) std_delta_az = np.clip(deg2rad(data['delta_azimuth_std']), min_std, np.inf) \ if 'delta_azimuth_std' in data.dtype.fields and opts.use_stats else np.tile(min_std, len(az)) std_delta_el = np.clip(deg2rad(data['delta_elevation_std']), min_std, np.inf) \ if 'delta_elevation_std' in data.dtype.fields and opts.use_stats else np.tile(min_std, len(el)) targets = data['target'] keep = data['keep'].astype(np.bool) if 'keep' in data.dtype.fields else np.tile(True, len(targets)) # List of unique targets in data set and target index for each data point unique_targets = np.unique(targets).tolist() target_index = np.array([unique_targets.index(t) for t in targets]) # Initialise new pointing model and set default enabled parameters new_model = katpoint.PointingModel() num_params = len(new_model) default_enabled = np.nonzero(list(old_model.values()))[0] # If the old model is empty / null, select the most basic set of parameters for starters if len(default_enabled) == 0: default_enabled = np.array([1, 3, 4, 5, 6, 7]) - 1 enabled_params = np.tile(False, num_params) enabled_params[default_enabled] = True enabled_params = enabled_params.tolist() old = PointingResults(old_model) new = PointingResults(new_model) # Fit new pointing model and update results params, sigma_params = new_model.fit(az[keep], el[keep], measured_delta_az[keep], measured_delta_el[keep], std_delta_az[keep], std_delta_el[keep], enabled_params) new.update(new_model) # Axis limit to be applied to all residual plots resid_lim = 1.2 * old.abs_sky_error.max() # Save pointing model to file outfile = file(opts.outfilebase + '.csv', 'w') # The original pointing model description string was comma-separated outfile.write(new_model.description.replace(" ", ", ")) outfile.close() print("Saved %d-parameter pointing model to '%s'" % (len(new_model), opts.outfilebase + '.csv')) # Turn data recarray into list of dicts and add residuals to the mix extended_data = [] for n in range(len(data)): rec_dict = dict(list(zip(data.dtype.names, data[n]))) rec_dict['keep'] = int(keep[n]) rec_dict['old_residual_xel'] = rad2deg(old.residual_xel[n]) rec_dict['old_residual_el'] = rad2deg(old.residual_el[n]) rec_dict['new_residual_xel'] = rad2deg(new.residual_xel[n]) rec_dict['new_residual_el'] = rad2deg(new.residual_el[n]) extended_data.append(rec_dict) # Format the data similar to analyse_point_source_scans output CSV file, with four new columns at the end fields = '%(dataset)s, %(target)s, %(timestamp_ut)s, %(azimuth).7f, %(elevation).7f, ' \ '%(delta_azimuth).7f, %(delta_azimuth_std).7f, %(delta_elevation).7f, %(delta_elevation_std).7f, ' \ '%(data_unit)s, %(beam_height_I).7f, %(beam_height_I_std).7f, %(beam_width_I).7f, ' \ '%(beam_width_I_std).7f, %(baseline_height_I).7f, %(baseline_height_I_std).7f, %(refined_I).0f, ' \ '%(beam_height_HH).7f, %(beam_width_HH).7f, %(baseline_height_HH).7f, %(refined_HH).0f, ' \ '%(beam_height_VV).7f, %(beam_width_VV).7f, %(baseline_height_VV).7f, %(refined_VV).0f, ' \ '%(frequency).7f, %(flux).4f, %(temperature).2f, %(pressure).2f, %(humidity).2f, %(wind_speed).2f, ' \ '%(keep)d, %(old_residual_xel).7f, %(old_residual_el).7f, %(new_residual_xel).7f, %(new_residual_el).7f\n' field_names = [name.partition(')')[0] for name in fields[2:].split(', %(')] # Save residual data and flags to file outfile2 = file(opts.outfilebase + '_data.csv', 'w') outfile2.write('# antenna = %s\n' % antenna.description) outfile2.write(', '.join(field_names) + '\n') outfile2.writelines([fields % rec for rec in extended_data]) outfile2.close() # Pointing model report print('Generating report, this may take a few minutes...') nice_filename = os.path.splitext(os.path.basename(filename))[0]+ '_pointing_model' pp = PdfPages(nice_filename+'.pdf') pagetext = [] i = 0 tmpstr = "" linelength = 5 pagetext.append("List of targets used:") for tar in list(set(unique_targets)): if i % linelength == linelength-1 : pagetext.append(tmpstr) tmpstr = "" i = i + 1 tmpstr +='%s, '%(tar) pagetext.append(tmpstr) pagetext.append("Pointing metrics for fitted points. (N= %i Fitting Data Points) "%(np.sum(keep))) pagetext.append("New Pointing model:") pagetext.append(new_model.description.replace(" ", ", ")) pagetext.append("All sky RMS = %.3f' (robust %.3f') " % (new.sky_rms, new.robust_sky_rms)) fig = plt.figure(None,figsize = (10,16)) plt.figtext(0.1,0.1,'\n'.join(pagetext),fontsize=12) fig.savefig(pp,format='pdf') plt.close(fig) # List of colors used to represent different targets in scatter plots scatter_colors = ('b', 'r', 'g', 'k', 'c', 'm', 'y') target_colors = np.tile(scatter_colors, 1 + len(unique_targets) // len(scatter_colors))[:len(unique_targets)] # Quantity loosely related to the declination of the source north = (np.pi / 2. - el) / (np.pi / 2.) * np.cos(az) pseudo_dec = -np.ones(len(unique_targets)) for n, ind in enumerate(target_index): if north[n] > pseudo_dec[ind]: pseudo_dec[ind] = north[n] north_to_south = np.flipud(np.argsort(pseudo_dec)) target_colors = target_colors[north_to_south][target_index] for idx in np.unique(target_index): fig = plt.figure(1, figsize=(15, 10)) fig.clear() # Store highlighted target index on figure object fig.highlighted_target = idx # Axes to contain detail residual plots - initialise plots with old residuals ax = fig.add_axes([0.27, 0.74, 0.2, 0.2]) ax.axhline(0, color='k', zorder=0) plot_data_and_tooltip(ax, rad2deg(az), rad2deg(old.residual_xel) * 60.) ax.axis([-180., 180., -resid_lim, resid_lim]) ax.set_xticks([]) ax.yaxis.set_ticks_position('right') ax.yaxis.set_major_formatter(mpl.ticker.FuncFormatter(arcmin_formatter)) ax.set_ylabel('Cross-EL offset') ax.set_title('RESIDUALS') ax = fig.add_axes([0.27, 0.54, 0.2, 0.2]) ax.axhline(0, color='k', zorder=0) plot_data_and_tooltip(ax, rad2deg(az), rad2deg(old.residual_el) * 60.) ax.axis([-180., 180., -resid_lim, resid_lim]) ax.set_xlabel('Azimuth (deg)') ax.yaxis.set_ticks_position('right') ax.yaxis.set_major_formatter(mpl.ticker.FuncFormatter(arcmin_formatter)) ax.set_ylabel('EL offset') ax = fig.add_axes([0.27, 0.26, 0.2, 0.2]) ax.axhline(0, color='k', zorder=0) plot_data_and_tooltip(ax, rad2deg(el), rad2deg(old.residual_xel) * 60.) ax.axis([0., 90., -resid_lim, resid_lim]) ax.set_xticks([]) ax.yaxis.set_ticks_position('right') ax.yaxis.set_major_formatter(mpl.ticker.FuncFormatter(arcmin_formatter)) ax.set_ylabel('Cross-EL offset') ax = fig.add_axes([0.27, 0.06, 0.2, 0.2]) ax.axhline(0, color='k', zorder=0) plot_data_and_tooltip(ax, rad2deg(el), rad2deg(old.residual_el) * 60.) ax.axis([0., 90., -resid_lim, resid_lim]) ax.set_xlabel('Elevation (deg)') ax.yaxis.set_ticks_position('right') ax.yaxis.set_major_formatter(mpl.ticker.FuncFormatter(arcmin_formatter)) ax.set_ylabel('EL offset') # Axes to contain quiver plot - plot static measurement locations in ARC projection as a start ax = fig.add_axes([0.5, 0.43, 0.5, 0.5], projection='polar') plot_data_and_tooltip(ax, np.pi/2. - az, np.pi/2. - el) segms = quiver_segments(old.residual_az, old.residual_el, 0.) ax.quiv = mpl.collections.LineCollection(segms, color='0.3') ax.add_collection(ax.quiv) ax.set_xticks(deg2rad(np.arange(0., 360., 90.))) ax.set_xticklabels(['E', 'N', 'W', 'S']) ax.yaxis.set_major_formatter(mpl.ticker.FuncFormatter(angle_formatter)) ax.set_ylim(0., np.pi / 2.) ax.set_yticks(deg2rad(np.arange(0., 90., 10.))) ax.set_yticklabels([]) # Axes to contain before/after residual plot ax = fig.add_axes([0.5, 0.135, 0.25, 0.25], projection='polar') ax.yaxis.set_major_formatter(mpl.ticker.FuncFormatter(arcmin_formatter)) plot_data_and_tooltip(ax, np.arctan2(old.residual_el, old.residual_xel), old.abs_sky_error) ax.set_xticklabels([]) ax.set_title('OLD') fig.text(0.625, 0.09, "$\chi^2$ = %.1f" % (old.chi2,), ha='center', va='baseline') fig.text(0.625, 0.06, "all sky rms = %.3f' (robust %.3f')" % (old.sky_rms, old.robust_sky_rms), ha='center', va='baseline') old.metrics(target_index == fig.highlighted_target) fig.text(0.625, 0.03, "target sky rms = %.3f' (robust %.3f')" % (old.sky_rms, old.robust_sky_rms), ha='center', va='baseline', fontdict=dict(color=(0.25,0,0,1))) old.metrics(keep) ax = fig.add_axes([0.75, 0.135, 0.25, 0.25], projection='polar') ax.yaxis.set_major_formatter(mpl.ticker.FuncFormatter(arcmin_formatter)) plot_data_and_tooltip(ax, np.arctan2(new.residual_el, new.residual_xel), new.abs_sky_error) ax.set_xticklabels([]) ax.set_title('NEW') fig.text(0.875, 0.09, "$\chi^2$ = %.1f" % (new.chi2,), ha='center', va='baseline') fig.text(0.875, 0.06, "all sky rms = %.3f' (robust %.3f')" % (new.sky_rms, new.robust_sky_rms), ha='center', va='baseline') new.metrics(target_index == fig.highlighted_target) fig.text(0.875, 0.03, "target sky rms = %.3f' (robust %.3f')" % (new.sky_rms, new.robust_sky_rms), ha='center', va='baseline', fontdict=dict(color=(0.25,0,0,1))) new.metrics(keep) param_button_color = ['0.65', '0.0'] param_button_weight = ['normal', 'bold'] # For display purposes, throw out unused parameters P2 and P10 display_params = list(range(num_params)) display_params.pop(9) display_params.pop(1) def setup_param_button(p): """Set up individual parameter toggle button.""" param = display_params[p] param_button = mpl.widgets.Button(fig.add_axes([0.09, 0.94 - (0.85 + p * 0.9) / len(display_params), 0.03, 0.85 / len(display_params)]), 'P%d' % (param + 1,)) fig.text(0.19, 0.94 - (0.5 * 0.85 + p * 0.9) / len(display_params), '', ha='right', va='center') fig.text(0.24, 0.94 - (0.5 * 0.85 + p * 0.9) / len(display_params), '', ha='right', va='center') state = enabled_params[param] param_button.label.set_color(param_button_color[state]) param_button.label.set_weight(param_button_weight[state]) def toggle_param_callback(event): state = not enabled_params[param] enabled_params[param] = state param_button.label.set_color(param_button_color[state]) param_button.label.set_weight(param_button_weight[state]) save_button.color = (0.85, 0, 0) save_button.hovercolor = (0.95, 0, 0) update(fig) param_button.on_clicked(toggle_param_callback) return param_button # This is to stop the gc from deleting the data param_buttons = [setup_param_button(p) for p in range(len(display_params))] # Add old pointing model and labels list_o_names = 'Ant:%s , Datasets:'%(antenna.name) + ' ,'.join(np.unique(data['dataset']).tolist() ) fig.text(0.905, 0.98,list_o_names, horizontalalignment='right',fontsize=10) fig.text(0.053, 0.95, 'OLD', ha='center', va='bottom', size='large') fig.text(0.105, 0.95, 'MODEL', ha='center', va='bottom', size='large') fig.text(0.16, 0.95, 'NEW', ha='center', va='bottom', size='large') fig.text(0.225, 0.95, 'STD', ha='center', va='bottom', size='large') for p, param in enumerate(display_params): param_str = param_to_str(old_model, param) if list(old_model.values())[param] else '' fig.text(0.085, 0.94 - (0.5 * 0.85 + p * 0.9) / len(display_params), param_str, ha='right', va='center') # Create target selector buttons and related text (title + target string) fig.text(0.565, 0.95, 'TARGET', ha='center', va='bottom', size='large') fig.text(0.565, 0.89, unique_targets[fig.highlighted_target], ha='center', va='top', fontdict=dict(color=(0.25,0,0,1))) quiver_scale = 0.1 * 10 * np.pi / 6 / deg2rad(old.robust_sky_rms / 60.) fig.axes[4].quiv.set_segments(quiver_segments(new.residual_az, new.residual_el, quiver_scale)) # Target state: 0 = flagged, 1 = unflagged, 2 = highlighted target_state = keep * ((target_index == fig.highlighted_target) + 1) # Specify colours of flagged, unflagged and highlighted dots, respectively, as RGBA tuples dot_colors = np.choose(target_state, np.atleast_3d(np.vstack([(1,1,1,1), (0,0,1,1), (1,0,0,1)]))).T for ax in fig.axes[:7]: ax.dots.set_facecolors(dot_colors) fig.texts[-1].set_text(unique_targets[fig.highlighted_target]) for p, param in enumerate(display_params): fig.texts[2*p + 6].set_text(param_to_str(new_model, param) if enabled_params[param] else '') # HACK to convert sigmas to arcminutes, but not for P9 and P12 (which are scale factors) # This functionality should really reside inside the PointingModel class std_param = rad2deg(sigma_params[param]) * 60. if param not in [8, 11] else sigma_params[param] std_param_str = ("%.2f'" % std_param) if param not in [8, 11] else ("%.0e" % std_param) fig.texts[2*p + 7].set_text(std_param_str if enabled_params[param] and opts.use_stats else '') # Turn parameter string bold if it changed significantly from old value if np.abs(params[param] - list(old_model.values())[param]) > 3.0 * sigma_params[param]: fig.texts[2*p + 6].set_weight('bold') fig.texts[2*p + 7].set_weight('bold') else: fig.texts[2*p + 6].set_weight('normal') fig.texts[2*p + 7].set_weight('normal') daz_az, del_az, daz_el, del_el, quiver, before, after = fig.axes[:7] # Update residual plots daz_az.dots.set_offsets(np.c_[rad2deg(az), rad2deg(new.residual_xel) * 60.]) del_az.dots.set_offsets(np.c_[rad2deg(az), rad2deg(new.residual_el) * 60.]) daz_el.dots.set_offsets(np.c_[rad2deg(el), rad2deg(new.residual_xel) * 60.]) del_el.dots.set_offsets(np.c_[rad2deg(el), rad2deg(new.residual_el) * 60.]) after.dots.set_offsets(np.c_[np.arctan2(new.residual_el, new.residual_xel), new.abs_sky_error]) resid_lim = 1.2 * max(new.abs_sky_error.max(), old.abs_sky_error.max()) daz_az.set_ylim(-resid_lim, resid_lim) del_az.set_ylim(-resid_lim, resid_lim) daz_el.set_ylim(-resid_lim, resid_lim) del_el.set_ylim(-resid_lim, resid_lim) before.set_ylim(0, resid_lim) after.set_ylim(0, resid_lim) fig.savefig(pp,format='pdf') # plt.close(fig) pp.close()
az, el = angle_wrap(deg2rad(offsetdata['azimuth'])),deg2rad(offsetdata['elevation']) measured_delta_az, measured_delta_el = deg2rad(offsetdata['delta_azimuth']), deg2rad(offsetdata['delta_elevation']) time_stamps = np.zeros_like(az) for i in range(az.shape[0]) : time_stamps[i] = katpoint.Timestamp(offsetdata['timestamp_ut'][i]).secs # Fix Timestamps #print new_model.description dataset_str = '_'.join(np.unique(offsetdata['dataset']).tolist() ) nice_filename = '%s_%s_pointing_error'%(dataset_str ,ant.name) pp = PdfPages(nice_filename+'.pdf') new_model = katpoint.PointingModel() num_params = len(new_model) #default_enabled = np.array([1, 3, 4, 5, 6, 7]) - 1 # first 6 params default_enabled = np.array([1, 7]) - 1 # only az & el offset params enabled_params = np.tile(False, num_params) enabled_params[default_enabled] = True enabled_params = enabled_params.tolist() # Fit new pointing model # Uncertainties are optional min_std = deg2rad(min_rms / 60. / np.sqrt(2)) std_delta_az = np.clip(deg2rad(data['delta_azimuth_std']), min_std, np.inf) \ if 'delta_azimuth_std' in data.dtype.fields and opts.use_stats else np.tile(min_std, len(az)) std_delta_el = np.clip(deg2rad(data['delta_elevation_std']), min_std, np.inf) \ if 'delta_elevation_std' in data.dtype.fields and opts.use_stats else np.tile(min_std, len(el)) params, sigma_params = new_model.fit(az[keep], el[keep], measured_delta_az[keep], measured_delta_el[keep],