def load_cal(filename, baseline, nd_models, freq_channel=None,channel_bw=10.0,channel_mask='',n_chan = 4096,channel_range=None,band_input=None):
""" Load the dataset into memory """
print('Loading noise diode models')
try:
d = scape.DataSet(filename, baseline=baseline, nd_models=nd_models,band=band_input)
except IOError:
nd = scape.gaincal.NoiseDiodeModel(freq=[1,2000],temp=[20,20])
warnings.warn('Warning: Failed to load/find Noise Diode Models, setting models to 20K ')
print('Warning: Failed to load/find Noise Diode Models, setting models to 20K ')
d = scape.DataSet(filename, baseline=baseline, nd_h_model = nd, nd_v_model=nd ,band=band_input)
if not channel_range is None :
start_freq_channel = int(channel_range.split(',')[0])
end_freq_channel = int(channel_range.split(',')[1])
edge = np.tile(True, n_chan)
edge[slice(start_freq_channel, end_freq_channel)] = False
else :
edge = np.tile(False, n_chan)
#load static flags if pickle file is given
if len(channel_mask)>0:
pickle_file = open(channel_mask,mode='rb')
rfi_static_flags = pickle.load(pickle_file)
pickle_file.close()
else:
rfi_static_flags = np.tile(False, n_chan)
static_flags = np.logical_or(edge,rfi_static_flags)
#d = d.select(freqkeep=~static_flags)
freq_channel_flagged = []
for band in freq_channel:
tmp_band = []
for channel in band :
if not static_flags[channel] : # if not flagged
tmp_band.append(channel)
#if len(tmp_band) > 0 :
freq_channel_flagged.append(tmp_band)
#if not freq_channel is None :
# d = d.select(freqkeep=freq_channel)
#print "Flagging RFI"
#sd = remove_rfi(d,width=7,sigma=5) # rfi flaging Needed ?
print("Converting to Temperature")
print("Plotting the number of channels in each band of the list of lists freq_channel_flagged will be useful")
d = d.convert_power_to_temperature(freq_width=0.0)
if not d is None:
d = d.select(flagkeep='~nd_on')
d = d.select(labelkeep='track', copy=False)
d.average(channels_per_band=freq_channel_flagged)
return d
def read_and_select_file(file, bline=None, channels=None, rfi_mask=None, nd_models=None, **kwargs):
"""
Read in the input h5 file using scape and make a selection.
file: {string} filename of h5 file to open in katdal
Returns:
the visibility data to plot, the frequency array to plot, the flags to plot
"""
data = scape.DataSet(file, baseline=bline, nd_models=nd_models, katfile=True)
# Secect desired channel range and tracks
# Select frequency channels and setup defaults if not specified
num_channels = len(data.channel_select)
if channels is None:
# Default is drop first and last 20% of the bandpass
start_chan = num_channels // 4
end_chan = start_chan * 3
else:
start_chan = int(channels.split(',')[0])
end_chan = int(channels.split(',')[1])
chan_select = list(range(start_chan,end_chan+1))
if rfi_mask:
mask_file = open(rfi_mask)
chan_select = ~(pickle.load(mask_file))
mask_file.close()
if len(chan_select) != num_channels:
raise ValueError('Number of channels in provided mask does not match number of channels in data')
chan_select[:start_chan] = False
chan_select[end_chan:] = False
#return the selected data
return data.select(freqkeep=chan_select,labelkeep='track')
def read_and_select_file(file, bline=None, channels=None, **kwargs):
"""
Read in the input h5 file using scape and make a selection.
file: {string} filename of h5 file to open in katdal
Returns:
the visibility data to plot, the frequency array to plot, the flags to plot
"""
data = scape.DataSet(file, baseline=bline, katfile=True)
compscan_labels = []
# Get compscan names (workaround for broken labelling after selection in scape)
for compscan in data.compscans:
compscan_labels.append(compscan.label)
# Secect desired channel range and tracks
# Select frequency channels and setup defaults if not specified
num_channels = len(data.channel_select)
if channels is None:
# Default is drop first and last 20% of the bandpass
start_chan = num_channels // 4
end_chan = start_chan * 3
else:
start_chan = int(channels.split(',')[0])
end_chan = int(channels.split(',')[1])
chan_range = range(start_chan, end_chan + 1)
data = data.select(freqkeep=chan_range, labelkeep='track')
#return the selected data
return data, compscan_labels
def LoadHDF5(HDF5Filename, header=False):
try:
d = scape.DataSet(HDF5Filename, baseline=opts.baseline)
except ValueError:
print "WARNING:THIS FILE", HDF5Filename.split(
'/'
)[-1], "IS CORRUPTED AND SCAPE WILL NOT PROCESS IT, YOU MAY NEED TO REAUGMENT IT,BUT ITS AN EXPENSIVE TASK..!!"
else:
print "SUCCESSFULLY LOADED: Wellcome to scape Library and scape is busy processing your request"
lo_freq = 4200.0 + d.freqs[len(d.freqs) / 2.0]
# try to check all the rfi channels across all the channels
rfi_chan_across_all = d.identify_rfi_channels()
d = d.select(freqkeep=range(100, 420))
# rfi channels across fringe finder channels ( i.e frequancy range around 100 to 420)
rfi_channels = d.identify_rfi_channels()
freqs = d.freqs
sky_frequency = d.freqs[rfi_channels]
ant = d.antenna.name
data_filename = os.path.splitext(
os.path.basename(HDF5Filename))[0] + '.h5'
# obs_date = os.path.splitext(os.path.basename(HDF5Filename))[0]
#date = time.ctime(float(obs_date))
for compscan in d.compscans:
azimuth = np.hstack(
[scan.pointing['az'] for scan in compscan.scans])
elevation = np.hstack(
[scan.pointing['el'] for scan in compscan.scans])
compscan_times = np.hstack(
[scan.timestamps for scan in compscan.scans])
compscan_start_time = np.hstack(
[scan.timestamps[0] for scan in compscan.scans])
compscan_end_time = np.hstack(
[scan.timestamps[-1] for scan in compscan.scans])
middle_time = np.median(compscan_times, axis=None)
obs_date = katpoint.Timestamp(middle_time)
middle_start_time = np.median(compscan_start_time)
middle_end_time = np.median(compscan_end_time)
end_time = katpoint.Timestamp(middle_end_time)
min_compscan_az = katpoint.rad2deg(azimuth.min())
max_compscan_az = katpoint.rad2deg(azimuth.max())
min_compscan_el = katpoint.rad2deg(elevation.min())
max_compscan_el = katpoint.rad2deg(elevation.max())
start_time = katpoint.Timestamp(middle_start_time)
requested_azel = compscan.target.azel(middle_time)
#ant_az = katpoint.rad2deg(np.array(requested_azel[0]))
#ant_el = katpoint.rad2deg(np.array(requested_azel[1]))
target = compscan.target.name
f = file(opts.outfilebase + '.csv', 'a')
for index in range(0, len(rfi_channels)):
rfi_chan = rfi_channels[index] + 100
rfi_freq = freqs[rfi_channels[index]]
f.write('%s, %s, %s, %s, %s,%f, %f,%f,%f, %f, %d, %f\n' % (data_filename,start_time, end_time, ant,target,min_compscan_az,max_compscan_az,\
min_compscan_el, max_compscan_el,lo_freq, rfi_chan, rfi_freq))
f.close()
def load_cal(filename, baseline, start_freq_channel, end_freq_channel, nd_models):
""" Load the dataset into memory """
d = scape.DataSet(filename, baseline=baseline, nd_models=nd_models)
d = d.select(freqkeep=range(start_freq_channel, end_freq_channel + 1))
d = d.convert_power_to_temperature(min_duration=opts.min_nd, jump_significance=10.0)
d = d.select(flagkeep='~nd_on')
d = d.select(labelkeep='track', copy=False)
d.average()
return d
def extract_cal_dataset(dataset):
"""Build data set from scans in original dataset containing noise diode firings."""
compscanlist = []
for compscan in dataset.compscans:
# Extract scans containing noise diode firings (make copy, as this will be modified by gain cal)
# Don't rely on 'cal' labels, as the KAT-7 system does not produce it anymore
scanlist = [scan.select(copy=True) for scan in compscan.scans
if 'nd_on' in scan.flags.dtype.fields and scan.flags['nd_on'].any()]
if scanlist:
compscanlist.append(scape.CompoundScan(scanlist, compscan.target))
return scape.DataSet(None, compscanlist, dataset.experiment_id, dataset.observer,
dataset.description, dataset.data_unit, dataset.corrconf.select(copy=True),
dataset.antenna, dataset.antenna2, dataset.nd_h_model, dataset.nd_v_model, dataset.enviro)
def reduce_compscan_with_uncertainty(dataset, compscan_index=0, mc_iterations=1, batch=True,
keep_all=True, num_compscans=0, **kwargs):
"""Do complete point source reduction on a compound scan, with uncertainty."""
dataset = scape.DataSet(None, [dataset.compscans[compscan_index]], dataset.experiment_id, dataset.observer,
dataset.description, dataset.data_unit, dataset.corrconf,
dataset.antenna, dataset.antenna2, dataset.nd_h_model, dataset.nd_v_model, dataset.enviro)
scan_dataset = dataset.select(labelkeep='scan', copy=False)
compscan = scan_dataset.compscans[0]
if 'logger' in kwargs:
kwargs['logger'].info("==== Processing compound scan %d of %d: '%s' ====",
compscan_index + 1, num_compscans, ' '.join(compscan_key(compscan)))
# Build data set containing a single compound scan at a time (make copy, as reduction modifies it)
scan_dataset.compscans = [compscan]
# If there are no noise diode models assume that there are no noise diodes
if dataset.nd_h_model is not None and dataset.nd_v_model is not None:
compscan_dataset = scan_dataset.select(flagkeep='~nd_on', copy=True)
else:
compscan_dataset = dataset.select(labelkeep='scan', copy=True)
cal_dataset = extract_cal_dataset(dataset)
# Do first reduction run
main_compscan = compscan_dataset.compscans[0]
fixed, variable = reduce_compscan(main_compscan, cal_dataset, **kwargs)
# Produce data set that has counts converted to Kelvin, but no averaging (for spectral plots)
unavg_compscan_dataset = scan_dataset.select(flagkeep='~nd_on', copy=True)
unavg_compscan_dataset.nd_gain = cal_dataset.nd_gain
unavg_compscan_dataset.convert_power_to_temperature()
# Add data from Monte Carlo perturbations
iter_outputs = [np.rec.fromrecords([tuple(variable.values())], names=list(variable.keys()))]
for m in range(mc_iterations - 1):
if 'logger' in kwargs:
kwargs['logger'].info("---- Monte Carlo iteration %d of %d ----",
m + 2, mc_iterations)
compscan_dataset = scan_dataset.select(flagkeep='~nd_on', copy=True).perturb()
cal_dataset = extract_cal_dataset(dataset).perturb()
fixed, variable = reduce_compscan(compscan_dataset.compscans[0], cal_dataset, **kwargs)
iter_outputs.append(np.rec.fromrecords([tuple(variable.values())], names=list(variable.keys())))
# Get mean and uncertainty of variable part of output data (assumed to be floats)
var_output = np.concatenate(iter_outputs).view(np.float).reshape(mc_iterations, -1)
var_mean = dict(zip(variable.keys(), var_output.mean(axis=0)))
var_std = dict(zip([name + '_std' for name in variable], var_output.std(axis=0)))
# Keep scan only with a valid beam in batch mode (otherwise keep button has to do it explicitly)
keep = batch and main_compscan.beam is not None and (keep_all or main_compscan.beam.is_valid)
if 'logger' in kwargs:
kwargs['logger'].debug("keep_all=%s, main_compscan.beam.is_valid=%s, keep=%s",
keep_all, main_compscan.beam is not None and main_compscan.beam.is_valid, keep)
output_dict = {'keep': keep, 'compscan': main_compscan, 'unavg_dataset': unavg_compscan_dataset}
output_dict.update(fixed)
output_dict.update(var_mean)
output_dict.update(var_std)
return output_dict
def plot_ts(h5, on_ts=None):
import scape
fig = plt.figure(figsize=(20, 5))
a = h5.ants[0]
nd = scape.gaincal.NoiseDiodeModel(freq=[856, 1712], temp=[20, 20])
d = scape.DataSet(h5, nd_h_model=nd, nd_v_model=nd)
scape.plot_xyz(d, 'time', 'amp', label='Average of the data')
if on_ts is not None:
on = on_ts
else:
on = h5.sensor['Antennas/' + a.name + '/nd_coupler']
ts = h5.timestamps - h5.timestamps[0]
plt.plot(ts,
np.array(on).astype(float) * 4000,
'g',
label='katdal ND sensor')
plt.title("Timeseries for antenna %s - %s" % (a.name, git_info()))
plt.legend()
return fig
def main():
# Parse command-line options and arguments
parser = optparse.OptionParser(
usage='%prog [options] <data file> [<data file> ...]',
description='Display a horizon mask from a set of data files.')
parser.add_option(
'-a',
'--baseline',
dest='baseline',
type="string",
metavar='BASELINE',
default='A1A1',
help=
"Baseline to load (e.g. 'A1A1' for antenna 1), default is first single-dish baseline in file"
)
parser.add_option('-o',
'--output',
dest='output',
type="string",
metavar='OUTPUTFILE',
default=None,
help="Write out intermediate h5 file")
parser.add_option('-s',
'--split',
dest='split',
action="store_true",
metavar='SPLIT',
default=False,
help="Whether to split each horizon plot in half")
parser.add_option('-z',
'--azshift',
dest='azshift',
type='float',
metavar='AZIMUTH_SHIFT',
default=45.0,
help="Degrees to rotate azimuth window by.")
parser.add_option(
'--temp-limit',
dest='temp_limit',
type='float',
default=40.0,
help=
"The Tempreture Limit to make the cut-off for the mask. This is calculated "
"as the T_sys at zenith plus the atmospheric noise contrabution at 10 degrees"
"elevation as per R.T. 199 .")
parser.add_option(
"-n",
"--nd-models",
help="Name of optional directory containing noise diode model files")
(opts, args) = parser.parse_args()
# Check arguments
if len(args) < 1:
raise RuntimeError('Please specify the data file to reduce')
# Load data set
gridtemp = []
for filename in args:
print 'Loading baseline', opts.baseline, 'from data file', filename
d = scape.DataSet(filename,
baseline=opts.baseline,
nd_models=opts.nd_models)
if len(d.freqs) > 1:
# Only keep main scans (discard slew and cal scans) a
d = d.select(freqkeep=range(200, 800))
d = remove_rfi(d, width=7, sigma=5)
d = d.convert_power_to_temperature(min_duration=3,
jump_significance=4.0)
d = d.select(flagkeep='~nd_on')
d = d.select(labelkeep='scan', copy=False)
# Average all frequency channels into one band
d.average()
# Extract azimuth and elevation angle from (azel) target associated with scan, in degrees
azimuth, elevation, temp = [], [], []
for s in d.scans:
azimuth.extend(rad2deg(s.pointing['az']))
elevation.extend(rad2deg(s.pointing['el']))
temp.extend(tuple(np.sqrt(s.pol('HH')[:, 0] * s.pol('VV')[:, 0])))
assert len(azimuth) == len(elevation) == len(temp), "sizes don't match"
data = (azimuth, elevation, temp)
np.array(azimuth) < -89
print "Gridding the data"
print "data shape = ", np.shape(data[0] + (
np.array(azimuth)[np.array(azimuth) < -89] + 360.0).tolist())
print np.shape(data[1] +
np.array(elevation)[np.array(azimuth) < -89].tolist())
print np.shape(data[2] +
np.array(temp)[np.array(azimuth) < -89].tolist())
gridtemp.append(
mlab.griddata(
data[0] +
(np.array(azimuth)[np.array(azimuth) < -89] + 360.0).tolist(),
data[1] +
np.array(elevation)[np.array(azimuth) < -89].tolist(),
data[2] + np.array(temp)[np.array(azimuth) < -89].tolist(),
np.arange(-90, 271, 1), np.arange(4, 16, 0.1)))
# The +361 is to ensure that the point are well spaced,
#this offset is not a problem as it is just for sorting out a boundery condition
print "Completed Gridding the data"
print "Making the mask"
mask = gridtemp[0] >= opts.temp_limit
for grid in gridtemp:
mask = mask * (grid >= opts.temp_limit)
maskr = np.zeros((len(np.arange(-90, 271, 1)), 2))
for i, az in enumerate(np.arange(-90, 271, 1)):
print 'at az %f' % (az, )
maskr[i] = az, np.max(elevation)
for j, el in enumerate(np.arange(4, 16, 0.1)):
if ~mask.data[j, i] and ~mask.mask[j, i]:
maskr[i] = az, el
break
np.savetxt('horizon_mask_%s.dat' % (opts.baseline), maskr[1:, :])
import numpy as np
import matplotlib as mpl
import matplotlib.pyplot as plt
import scape
import katpoint
# Load temporary noise diode models
a1h = np.loadtxt('noise_diode_models/T_nd_A1H_coupler.txt', delimiter=',')
a1v = np.loadtxt('noise_diode_models/T_nd_A1V_coupler.txt', delimiter=',')
a2h = np.loadtxt('noise_diode_models/T_nd_A2H_coupler.txt', delimiter=',')
a2v = np.loadtxt('noise_diode_models/T_nd_A2V_coupler.txt', delimiter=',')
# Load data set and do standard continuum reduction
d = scape.DataSet('1268855687.h5', baseline='A1A1')
d.nd_model = scape.gaincal.NoiseDiodeModel(a1h, a1v, std_temp=0.04)
d = d.select(freqkeep=range(95, 380))
d.convert_power_to_temperature()
d = d.select(labelkeep='scan', copy=False)
d.average()
# Edit out some RFI
d.scans = d.compscans[0].scans
d.scans[37] = d.scans[37].select(timekeep=range(76), copy=True)
d.scans[38] = d.scans[38].select(timekeep=range(12,
len(d.scans[38].timestamps)),
copy=True)
d.scans[72] = d.scans[72].select(timekeep=range(2,
len(d.scans[72].timestamps)),
copy=True)
user_logger.info("Waiting for HDF5 file '%s' to appear in archive" % (session.output_file,))
h5file = session.get_archived_product(download_dir=os.path.abspath(os.path.curdir))
if not os.path.isfile(h5file):
raise RuntimeError("Could not download '%s' to %d" % (h5file, os.path.abspath(download_dir)))
if not kat.dry_run:
cfg = kat.system
# Obtain list of antennas and polarisations present in data set
user_logger.info('Loading HDF5 file into scape and reducing the data')
h5 = katfile.open(h5file)
# Iterate through antennas
for ant in h5.ants:
ant_num = int(ant.name[3:])
# Load file and do standard processing
d = scape.DataSet(h5file, baseline='A%dA%d' % (ant_num, ant_num))
d = d.select(freqkeep=range(start_freq_channel, end_freq_channel + 1))
channel_freqs = d.freqs
d.convert_power_to_temperature()
d = d.select(labelkeep='scan', copy=False)
d.average()
# Only use the first compound scan for fitting beam and baseline
compscan = d.compscans[0]
# Calculate average target flux over entire band
flux_spectrum = [compscan.target.flux_density(freq) for freq in channel_freqs]
average_flux = np.mean([flux for flux in flux_spectrum if flux])
# Fit individual polarisation beams first, to get gains and system temperatures
gain_hh, gain_vv = None, None
baseline_hh, baseline_vv = None, None
if (ant.name + 'H') in h5.inputs:
d.fit_beams_and_baselines(pol='HH', circular_beam=False)
def analyse_point_source_scans(filename, opts):
dataset_name = os.path.splitext(os.path.basename(filename))[0]
# Default output file names are based on input file name
if opts.outfilebase is None:
opts.outfilebase = dataset_name + '_point_source_scans'
# Set up logging: logging everything (DEBUG & above), both to console and file
logger = logging.root
logger.setLevel(logging.DEBUG)
fh = logging.FileHandler(opts.outfilebase + '.log', 'w')
fh.setLevel(logging.DEBUG)
fh.setFormatter(logging.Formatter('%(levelname)s: %(message)s'))
logger.addHandler(fh)
# Produce canonical version of baseline string (remove duplicate antennas)
baseline_ants = opts.baseline.split(',')
if len(baseline_ants) == 2 and baseline_ants[0] == baseline_ants[1]:
opts.baseline = baseline_ants[0]
# Load old CSV file used to select compound scans from dataset
keep_scans = keep_datasets = None
if opts.keepfilename:
ant_name = katpoint.Antenna(
file(opts.keepfilename).readline().strip().partition('=')[2]).name
try:
data = np.loadtxt(opts.keepfilename,
dtype='string',
comments='#',
delimiter=', ')
except ValueError:
raise ValueError(
"CSV file '%s' contains rows with a different number of columns/commas"
% opts.keepfilename)
try:
fields = data[0].tolist()
id_fields = [
fields.index('dataset'),
fields.index('target'),
fields.index('timestamp_ut')
]
except (IndexError, ValueError):
raise ValueError("CSV file '%s' do not have the expected columns" %
opts.keepfilename)
keep_scans = set(
[ant_name + ' ' + ' '.join(line) for line in data[1:, id_fields]])
keep_datasets = set(data[1:, id_fields[0]])
# Switch to batch mode if CSV file is given
opts.batch = True
logger.debug(
"Loaded CSV file '%s' containing %d dataset(s) and %d compscan(s) for antenna '%s'"
%
(opts.keepfilename, len(keep_datasets), len(keep_scans), ant_name))
# Ensure we are using antenna found in CSV file (this assumes single dish setup for now)
csv_baseline = ant_name
if opts.baseline != 'sd' and opts.baseline != csv_baseline:
logger.warn(
"Requested baseline '%s' does not match baseline '%s' in CSV file '%s'"
% (opts.baseline, csv_baseline, opts.keepfilename))
logger.warn("Using baseline '%s' found in CSV file '%s'" %
(csv_baseline, opts.keepfilename))
opts.baseline = csv_baseline
# Avoid loading the data set if it does not appear in specified CSV file
if keep_datasets and dataset_name not in keep_datasets:
raise RuntimeError("Skipping dataset '%s' (based on CSV file)" %
(filename, ))
# Load data set
logger.info("Loading dataset '%s'" % (filename, ))
dataset = scape.DataSet(filename,
baseline=opts.baseline,
nd_models=opts.nd_models,
time_offset=opts.time_offset,
katfile=not opts.old_loader)
# Select frequency channels and setup defaults if not specified
num_channels = len(dataset.channel_select)
if opts.freq_chans is None:
# Default is drop first and last 25% of the bandpass
start_chan = num_channels // 4
end_chan = start_chan * 3
else:
start_chan = int(opts.freq_chans.split(',')[0])
end_chan = int(opts.freq_chans.split(',')[1])
chan_range = range(start_chan, end_chan + 1)
dataset = dataset.select(freqkeep=chan_range)
# Check scan count
if len(dataset.compscans) == 0 or len(dataset.scans) == 0:
raise RuntimeError('No scans found in file, skipping data set')
scan_dataset = dataset.select(labelkeep='scan', copy=False)
if len(scan_dataset.compscans) == 0 or len(scan_dataset.scans) == 0:
raise RuntimeError(
'No scans left after standard reduction, skipping data set (no scans labelled "scan", perhaps?)'
)
# Override pointing model if it is specified (useful if it is not in data file, like on early KAT-7)
if opts.pointing_model:
pm = file(opts.pointing_model).readline().strip()
logger.debug("Loaded %d-parameter pointing model from '%s'" %
(len(pm.split(',')), opts.pointing_model))
dataset.antenna.pointing_model = katpoint.PointingModel(pm,
strict=False)
# Initialise the output data cache (None indicates the compscan has not been processed yet)
reduced_data = [{} for n in range(len(scan_dataset.compscans))]
### BATCH MODE ###
# This will cycle through all data sets and stop when done
if opts.batch:
# Go one past the end of compscan list to write the output data out to CSV file
for current_compscan in range(len(scan_dataset.compscans) + 1):
# Look up compscan key in list of compscans to keep (if provided, only applicable to batch mode anyway)
if keep_scans and (current_compscan < len(scan_dataset.compscans)):
cs_key = ' '.join(
compscan_key(scan_dataset.compscans[current_compscan]))
if cs_key not in keep_scans:
logger.info(
"==== Skipping compound scan '%s' (based on CSV file) ===="
% (cs_key, ))
continue
output = reduce_and_plot(dataset,
current_compscan,
reduced_data,
opts,
logger=logger)
return output
### INTERACTIVE MODE ###
else:
if not plt:
raise ImportError(
'Interactive use of this script requires matplotlib - please install it or run in batch mode'
)
# Set up figure with buttons
plt.ion()
fig = plt.figure(1)
plt.clf()
if opts.plot_spectrum:
plt.subplot(311)
plt.subplot(312)
plt.subplot(313)
else:
plt.subplot(211)
plt.subplot(212)
plt.subplots_adjust(bottom=0.2, hspace=0.25)
plt.figtext(0.05, 0.05, '', va='bottom', ha='left')
plt.figtext(0.05, 0.945, '', va='bottom', ha='left')
# Make button context manager that disables buttons during processing and re-enables it afterwards
class DisableButtons(object):
def __init__(self):
"""Start with empty button list."""
self.buttons = []
def append(self, button):
"""Add button to list."""
self.buttons.append(button)
def __enter__(self):
"""Disable buttons on entry."""
if plt.fignum_exists(1):
for button in self.buttons:
button.eventson = False
button.hovercolor = '0.85'
button.label.set_color('gray')
plt.draw()
def __exit__(self, exc_type, exc_value, traceback):
"""Re-enable buttons on exit."""
if plt.fignum_exists(1):
for button in self.buttons:
button.eventson = True
button.hovercolor = '0.95'
button.label.set_color('k')
plt.draw()
all_buttons = DisableButtons()
# Create buttons and their callbacks
spectrogram_button = widgets.Button(plt.axes([0.37, 0.05, 0.1, 0.075]),
'Spectrogram')
def spectrogram_callback(event):
with all_buttons:
plt.figure(2)
plt.clf()
out = reduced_data[fig.current_compscan]
ax = scape.plot_xyz(out['unavg_dataset'],
'time',
'freq',
'amp',
power_in_dB=True)
ax.set_title(out['target'], size='medium')
spectrogram_button.on_clicked(spectrogram_callback)
all_buttons.append(spectrogram_button)
keep_button = widgets.Button(plt.axes([0.48, 0.05, 0.1, 0.075]),
'Keep')
def keep_callback(event):
with all_buttons:
reduced_data[fig.current_compscan]['keep'] = True
fig.current_compscan += 1
reduce_and_plot(dataset,
fig.current_compscan,
reduced_data,
opts,
fig,
logger=logger)
keep_button.on_clicked(keep_callback)
all_buttons.append(keep_button)
discard_button = widgets.Button(plt.axes([0.59, 0.05, 0.1, 0.075]),
'Discard')
def discard_callback(event):
with all_buttons:
reduced_data[fig.current_compscan]['keep'] = False
fig.current_compscan += 1
reduce_and_plot(dataset,
fig.current_compscan,
reduced_data,
opts,
fig,
logger=logger)
discard_button.on_clicked(discard_callback)
all_buttons.append(discard_button)
back_button = widgets.Button(plt.axes([0.7, 0.05, 0.1, 0.075]), 'Back')
def back_callback(event):
with all_buttons:
if fig.current_compscan > 0:
fig.current_compscan -= 1
reduce_and_plot(dataset,
fig.current_compscan,
reduced_data,
opts,
fig,
logger=logger)
back_button.on_clicked(back_callback)
all_buttons.append(back_button)
done_button = widgets.Button(plt.axes([0.81, 0.05, 0.1, 0.075]),
'Done')
def done_callback(event):
with all_buttons:
fig.current_compscan = len(reduced_data)
reduce_and_plot(dataset,
fig.current_compscan,
reduced_data,
opts,
fig,
logger=logger)
done_button.on_clicked(done_callback)
all_buttons.append(done_button)
# Start off the processing on the first compound scan
fig.current_compscan = 0
reduce_and_plot(dataset,
fig.current_compscan,
reduced_data,
opts,
fig,
logger=logger)
# Display plots - this should be called ONLY ONCE, at the VERY END of the script
# The script stops here until you close the plots...
plt.show()
# Only import matplotlib if not in batch mode
if not opts.batch:
import matplotlib.pyplot as plt
import matplotlib.widgets as widgets
# Avoid loading the data set if it does not appear in specified CSV file
if keep_datasets and dataset_name not in keep_datasets:
raise RuntimeError("Skipping dataset '%s' (based on CSV file)" %
(filename, ))
# Load data set
logger.info("Loading dataset '%s'" % (filename, ))
dataset = scape.DataSet(filename,
baseline=opts.baseline,
nd_models=opts.nd_models,
time_offset=opts.time_offset,
katfile=not opts.old_loader)
# Select frequency channels and setup defaults if not specified
num_channels = len(dataset.channel_select)
if opts.freq_chans is None:
# Default is drop first and last 25% of the bandpass
start_chan = num_channels // 4
end_chan = start_chan * 3
else:
start_chan = int(opts.freq_chans.split(',')[0])
end_chan = int(opts.freq_chans.split(',')[1])
chan_range = range(start_chan, end_chan + 1)
dataset = dataset.select(freqkeep=chan_range)
def analyse_point_source_scans(filename, h5file, opts):
# Default output file names are based on input file name
dataset_name = os.path.splitext(os.path.basename(filename))[0]
if opts.outfilebase is None:
opts.outfilebase = dataset_name + '_point_source_scans'
kwargs = {}
#Force centre freqency if ku-band option is set
if opts.ku_band:
kwargs['centre_freq'] = 12.5005e9
# Produce canonical version of baseline string (remove duplicate antennas)
baseline_ants = opts.baseline.split(',')
if len(baseline_ants) == 2 and baseline_ants[0] == baseline_ants[1]:
opts.baseline = baseline_ants[0]
# Load data set
if opts.baseline not in [ant.name for ant in h5file.ants]:
raise RuntimeError('Cannot find antenna %s in dataset' % opts.baseline)
# dataset = scape.DataSet(h5file, baseline=opts.baseline, nd_models=opts.nd_models,
# time_offset=opts.time_offset, **kwargs)
dataset = scape.DataSet(filename,
baseline=opts.baseline,
nd_models=opts.nd_models,
time_offset=opts.time_offset,
**kwargs)
# Select frequency channels and setup defaults if not specified
num_channels = len(dataset.channel_select)
if opts.freq_chans is None:
# Default is drop first and last 25% of the bandpass
start_chan = num_channels // 4
end_chan = start_chan * 3
else:
start_chan = int(opts.freq_chans.split(',')[0])
end_chan = int(opts.freq_chans.split(',')[1])
chan_select = list(range(start_chan, end_chan + 1))
# Check if a channel mask is specified and apply
if opts.channel_mask:
mask_file = open(opts.channel_mask, mode='rb')
chan_select = ~(pickle.load(mask_file))
mask_file.close()
if len(chan_select) != num_channels:
raise ValueError(
'Number of channels in provided mask does not match number of channels in data'
)
chan_select[:start_chan] = False
chan_select[end_chan:] = False
dataset = dataset.select(freqkeep=chan_select)
# Check scan count
if len(dataset.compscans) == 0 or len(dataset.scans) == 0:
raise RuntimeError('No scans found in file, skipping data set')
scan_dataset = dataset.select(labelkeep='scan', copy=False)
if len(scan_dataset.compscans) == 0 or len(scan_dataset.scans) == 0:
raise RuntimeError(
'No scans left after standard reduction, skipping data set (no scans labelled "scan", perhaps?)'
)
# Override pointing model if it is specified (useful if it is not in data file, like on early KAT-7)
if opts.pointing_model:
if opts.pointing_model.split('/')[-2] == 'mkat':
if opts.ku_band: band = 'ku'
else: band = 'l'
pt_file = os.path.join(opts.pointing_model,
'%s.%s.pm.csv' % (opts.baseline, band))
else:
pt_file = os.path.join(opts.pointing_model,
'%s.pm.csv' % (opts.baseline))
if not os.path.isfile(pt_file):
raise RuntimeError('Cannot find file %s' % (pt_file))
pm = file(pt_file).readline().strip()
dataset.antenna.pointing_model = katpoint.PointingModel(pm)
# Remove any noise diode models if the ku band option is set and flag for spikes
if opts.ku_band:
dataset.nd_h_model = None
dataset.nd_v_model = None
for i in range(len(dataset.scans)):
dataset.scans[i].data = scape.stats.remove_spikes(
dataset.scans[i].data, axis=1, spike_width=3, outlier_sigma=5.)
# Initialise the output data cache (None indicates the compscan has not been processed yet)
reduced_data = [{} for n in range(len(scan_dataset.compscans))]
# Go one past the end of compscan list to write the output data out to CSV file
for current_compscan in range(len(scan_dataset.compscans) + 1):
# make things play nice
opts.batch = True
try:
the_compscan = scan_dataset.compscans[current_compscan]
except:
the_compscan = None
fig = plt.figure(1, figsize=(8, 8))
plt.clf()
if opts.plot_spectrum:
plt.subplot(311)
plt.subplot(312)
plt.subplot(313)
else:
plt.subplot(211)
plt.subplot(212)
plt.subplots_adjust(bottom=0.2, hspace=0.25)
plt.figtext(0.05, 0.05, '', va='bottom', ha='left')
plt.figtext(0.05, 0.945, '', va='bottom', ha='left')
# Start off the processing on the first compound scan
logger = logging.root
fig.current_compscan = 0
reduce_and_plot(dataset,
fig.current_compscan,
reduced_data,
opts,
fig,
logger=logger)
# Initialise the output data cache (None indicates the compscan has not been processed yet)
reduced_data = [{} for n in range(len(scan_dataset.compscans))]
# Go one past the end of compscan list to write the output data out to CSV file
for current_compscan in range(len(scan_dataset.compscans) + 1):
# make things play nice
opts.batch = True
try:
the_compscan = scan_dataset.compscans[current_compscan]
except:
the_compscan = None
logger = logging.root
output = local_reduce_and_plot(dataset,
current_compscan,
reduced_data,
opts,
logger=logger)
offsetdata = output[1]
from katpoint import deg2rad
def angle_wrap(angle, period=2.0 * np.pi):
"""wrap angle into the interval -*period* / 2 ... *period* / 2."""
return (angle + 0.5 * period) % period - 0.5 * period
az, el = angle_wrap(deg2rad(offsetdata['azimuth'])), deg2rad(
offsetdata['elevation'])
model_delta_az, model_delta_el = ant.pointing_model.offset(az, el)
measured_delta_az = offsetdata[
'delta_azimuth'] - model_delta_az # pointing model correction
measured_delta_el = offsetdata[
'delta_elevation'] - model_delta_el # pointing model correction
"""determine new residuals from current pointing model"""
residual_az = measured_delta_az - model_delta_az
residual_el = measured_delta_el - model_delta_el
residual_xel = residual_az * np.cos(el)
# Initialise new pointing model and set default enabled parameters
keep = np.ones((len(offsetdata)), dtype=np.bool)
min_rms = np.sqrt(2) * 60. * 1e-12
use_stats = True
new_model = katpoint.PointingModel()
num_params = len(new_model)
default_enabled = np.array([1, 3, 4, 5, 6, 7]) - 1
enabled_params = np.tile(False, num_params)
enabled_params[default_enabled] = True
enabled_params = enabled_params.tolist()
# Fit new pointing model
az, el = angle_wrap(deg2rad(offsetdata['azimuth'])), deg2rad(
offsetdata['elevation'])
measured_delta_az, measured_delta_el = deg2rad(
offsetdata['delta_azimuth']), deg2rad(offsetdata['delta_elevation'])
# Uncertainties are optional
min_std = deg2rad(min_rms / 60. / np.sqrt(2))
std_delta_az = np.clip(deg2rad(offsetdata['delta_azimuth_std']), min_std, np.inf) \
if 'delta_azimuth_std' in offsetdata.dtype.fields and use_stats else np.tile(min_std, len(az))
std_delta_el = np.clip(deg2rad(offsetdata['delta_elevation_std']), min_std, np.inf) \
if 'delta_elevation_std' in offsetdata.dtype.fields and use_stats else np.tile(min_std, len(el))
params, sigma_params = new_model.fit(az[keep], el[keep],
measured_delta_az[keep],
measured_delta_el[keep],
std_delta_az[keep],
std_delta_el[keep], enabled_params)
"""Determine new residuals from new fit"""
newmodel_delta_az, newmodel_delta_el = new_model.offset(az, el)
residual_az = measured_delta_az - newmodel_delta_az
residual_el = measured_delta_el - newmodel_delta_el
residual_xel = residual_az * np.cos(el)
# Show actual scans
h5file.select(scans='scan')
fig1 = plt.figure(2, figsize=(8, 8))
plt.scatter(h5file.ra,
h5file.dec,
s=np.mean(np.abs(h5file.vis[:, 2200:2400, 1]), axis=1))
plt.title('Raster scan over target')
plt.ylabel('Dec [deg]')
plt.xlabel('Ra [deg]')
# Try to fit beam
for c in h5file.compscans():
if not dataset is None:
dataset = dataset.select(flagkeep='~nd_on')
dataset.average()
dataset.fit_beams_and_baselines()
# Generate output report
with PdfPages(opts.outfilebase + '_' + opts.baseline + '.pdf') as pdf:
out = reduced_data[0]
offset_az, offset_el = "%.1f" % (
60. * out['delta_azimuth'], ), "%.1f" % (60. *
out['delta_elevation'], )
beam_width, beam_height = "%.1f" % (
60. * out['beam_width_I'], ), "%.2f" % (out['beam_height_I'], )
baseline_height = "%.1f" % (out['baseline_height_I'], )
pagetext = "\nCheck Point Source Scan"
pagetext += "\n\nDescription: %s\nName: %s\nExperiment ID: %s" % (
h5file.description, h5file.name, h5file.experiment_id)
pagetext = pagetext + "\n"
pagetext += "\n\nTest Setup:"
pagetext += "\nRaster Scan across bright source"
pagetext += "\n\nAntenna %(antenna)s" % out
pagetext += "\n------------"
pagetext += ("\nTarget = '%(target)s', azel=(%(azimuth).1f, %(elevation).1f) deg, " % out) +\
("offset=(%s, %s) arcmin" % (offset_az, offset_el))
pagetext += ("\nBeam height = %s %s") % (beam_height, out['data_unit'])
pagetext += ("\nBeamwidth = %s' (expected %.1f')") % (
beam_width, 60. * out['beam_expected_width_I'])
pagetext += ("\nHH gain = %.3f Jy/%s") % (
out['flux'] / out['beam_height_HH'], out['data_unit'])
pagetext += ("\nVV gain = %.3f Jy/%s") % (
out['flux'] / out['beam_height_VV'], out['data_unit'])
pagetext += ("\nBaseline height = %s %s") % (baseline_height,
out['data_unit'])
pagetext = pagetext + "\n"
pagetext += ("\nCurrent model AzEl=(%.3f, %.3f) deg" %
(model_delta_az[0], model_delta_el[0]))
pagetext += ("\nMeasured coordinates using rough fit")
pagetext += ("\nMeasured AzEl=(%.3f, %.3f) deg" %
(measured_delta_az[0], measured_delta_el[0]))
pagetext = pagetext + "\n"
pagetext += ("\nDetermine residuals from current pointing model")
residual_az = measured_delta_az - model_delta_az
residual_el = measured_delta_el - model_delta_el
pagetext += ("\nResidual AzEl=(%.3f, %.3f) deg" %
(residual_az[0], residual_el[0]))
if dataset.compscans[0].beam is not None:
if not dataset.compscans[0].beam.is_valid:
pagetext += ("\nPossible bad fit!")
if (residual_az[0] < 1.) and (residual_el[0] < 1.):
pagetext += ("\nResiduals withing L-band beam")
else:
pagetext += ("\nMaximum Residual, %.2f, larger than L-band beam" %
(numpy.max(residual_az[0], residual_el[0])))
pagetext = pagetext + "\n"
pagetext += ("\nFitted parameters \n%s" % str(params[:5]))
plt.figure(None, figsize=(16, 8))
plt.axes(frame_on=False)
plt.xticks([])
plt.yticks([])
plt.title("AR1 Report %s" % opts.outfilebase,
fontsize=14,
fontweight="bold")
plt.text(0, 0, pagetext, fontsize=12)
pdf.savefig()
plt.close()
pdf.savefig(fig)
pdf.savefig(fig1)
d = pdf.infodict()
import datetime
d['Title'] = h5file.description
d['Author'] = 'AR1'
d['Subject'] = 'AR1 check point source scan'
d['CreationDate'] = datetime.datetime(2015, 8, 13)
d['ModDate'] = datetime.datetime.today()
gain_hh = np.array(())
gain_vv = np.array(())
timestamps = np.array(())
filename = args[0]
nice_filename = filename.split('/')[-1] + '_' + opts.ant + '_gain_stability'
pp = PdfPages(nice_filename + '.pdf')
for filename in args:
h5 = katdal.open(filename)
if opts.ant == '':
ant = h5.ants[0].name
else:
ant = opts.ant
#h5.select(ants=ant)
d = scape.DataSet(filename, baseline="%s,%s" % (ant, ant))
d = d.select(freqkeep=range(start_freq_channel, end_freq_channel + 1))
d = remove_rfi(d, width=21, sigma=5) # rfi flaging
#Leave the d dataset unchanged after this so that it can be examined interactively if necessary
antenna = d.antenna
d_uncal = d.select(copy=True)
d_uncal.average()
d_cal = d.select(copy=True)
print("do selects")
#extract timestamps from data
timestampfile = np.hstack([scan.timestamps for scan in d_cal.scans])
#get a user-friendly time axis that will plot in the same way as plot_xyz
time = scape.extract_scan_data(d_cal.scans, 'time')
tmin = np.min(np.hstack(time.data))
tmax = np.max(np.hstack(time.data))
#Get the gain from the noise diodes
help="Baseline to calibrate (e.g. 'A1A2'), default is first interferometric baseline in file")
parser.add_option('-p', '--pol', metavar='POL', default='HH',
help="Polarisation term to use ('HH' or 'VV'), default is %default")
parser.add_option('-s', '--max-sigma', type='float', default=0.05,
help="Threshold on std deviation of normalised group delay, default is %default")
(opts, args) = parser.parse_args()
if len(args) < 1:
print 'Please specify the data file to reduce'
sys.exit(1)
# Load data sets
datasets, scans = [], []
for filename in args:
print 'Loading baseline', opts.baseline, 'from data file', filename
d = scape.DataSet(filename, baseline=opts.baseline)
# Discard 'slew' scans and channels outside the Fringe Finder band
d = d.select(labelkeep='scan', freqkeep=range(100, 420), copy=True)
channel_freqs, channel_bw = d.freqs, d.bandwidths[0]
antenna, antenna2 = d.antenna, d.antenna2
datasets.append(d)
scans.extend(d.scans)
time_origin = np.min([scan.timestamps.min() for scan in scans])
# Since the phase slope is sampled, the derived delay exhibits aliasing with a period of 1 / channel_bandwidth
# The maximum delay that can be reliably represented is therefore +- 0.5 / channel_bandwidth
max_delay = 1e-6 / channel_bw / 2
# Maximum standard deviation of delay occurs when delay samples are uniformly distributed between +- max_delay
# In this case, delay is completely random / unknown, and its estimate cannot be trusted
# The division by sqrt(N-1) converts the per-channel standard deviation to a per-snapshot deviation
max_sigma_delay = np.abs(2 * max_delay) / np.sqrt(12) / np.sqrt(len(channel_freqs) - 1)