def file_mime_detection(katfile):
"""Function to instantiate the correct metadata extraction class. The
following file extensions are are currently detected: '.h5' and '.rdb'.
Parameters:
----------
katfile: string : name of file for mime detecetion.
"""
file_ext = os.path.splitext(katfile)[1]
if file_ext == '.h5':
katdata = katdal.open(katfile)
# 'katdata.ants' are sorted alphabetically.
# KAT7 antennas start with 'ant'. MeerKAT (AR1 and onwards) antennas start with 'm'.
# If the first antenna starts with 'ant' consider it KAT7 data.
if katdata.ants[0].name.startswith('ant'):
return KatFileProductMetExtractor(katdata)
# if 'proposal_id' mention RTS at least once then it's an RTS file.
elif 'proposal_id' in katdata.obs_params and katdata.obs_params[
'proposal_id'].count('RTS') >= 1:
return RTSTelescopeProductMetExtractor(katdata)
# everything else that is .h5 must be MeerKAT AR1 and onwards
else:
return MeerKATAR1TelescopeProductMetExtractor(katdata)
# vis trawler uses os.environ['SITENAME'] to differentiate instruments when extracting metadata.
elif file_ext == '.rdb':
if 'sdp_l0' in katfile:
return MeerKATTelescopeProductMetExtractor(katfile)
elif 'sdp_l1_flags' in katfile:
return MeerKATFlagProductMetExtractor(katfile)
raise MetExtractorException("File extension not supported %s" % (file_ext))
def main() -> None:
parser = argparse.ArgumentParser()
parser.add_argument('dataset', type=str, help='Input dataset')
parser.add_argument('output_dir', type=str, help='Parent directory for output')
parser.add_argument('prefix', type=str, help='Prefix for output directories and filenames')
parser.add_argument('stream', type=str, help='Stream name for telescope state inputs')
parser.add_argument('--log-level', type=str, metavar='LEVEL',
help='Logging level [INFO]')
args = parser.parse_args()
katsdpservices.setup_logging()
if args.log_level is not None:
logging.getLogger().setLevel(args.log_level.upper())
dataset = katdal.open(args.dataset, chunk_store=None, upgrade_flags=False)
telstate = dataset.source.telstate.wrapped.root()
telstate = telstate.view(telstate.join(dataset.source.capture_block_id, args.stream))
output_dir = '{}_{}'.format(args.prefix, uuid.uuid4())
output_dir = os.path.join(args.output_dir, output_dir)
filename = args.prefix + '_report.html'
tmp_dir = output_dir + '.writing'
os.mkdir(tmp_dir)
try:
common_stats, target_stats = report.get_stats(dataset, telstate)
report.write_report(common_stats, target_stats, os.path.join(tmp_dir, filename))
report.write_metadata(dataset, common_stats, target_stats,
os.path.join(tmp_dir, 'metadata.json'))
os.rename(tmp_dir, output_dir)
except Exception:
# Make a best effort to clean up
shutil.rmtree(tmp_dir, ignore_errors=True)
raise
def build_offline_pipeline(data, **kwargs):
"""
Decide based on the type of data how to
build an offline pipeline instance and do it.
Parameters
----------
data : str or :class:`katdal.Dataset`
file location or katdal Dataset object
TODO: Allow for different types of data (eg UVFITS)
so that this function actually does something.
Returns
-------
:class:`Pipeline`
A pipeline instance
"""
if isinstance(data, DataSet):
return KatdalOfflinePipeline(data, **kwargs)
if isinstance(data, str):
try:
ds = katdal.open(data)
except (IOError, DataSourceNotFound):
pass
else:
return KatdalOfflinePipeline(ds, **kwargs)
raise ValueError('Data type of %s not recognised for %s' % (type(data), data))
def _test_data(self, *, options=(), channel0=36, channel1=72, max_chunk_vis=None,
missing=set(), channel_mask=None):
loader = LoaderKatdal('file:///fake_filename.rdb', options, 24, None)
bls = len(ANTENNAS) * (len(ANTENNAS) - 1) // 2 # Excludes auto-correlations
vis, weights, uvw, feed_angle1, feed_angle2, chunks = \
self._collect(loader.data_iter(channel0, channel1, max_chunk_vis=max_chunk_vis))
dataset = katdal.open('file:///fake_filename.rdb')
dataset.select(corrprods='cross', channels=np.s_[36:72], scans='track')
if channel_mask is not None:
channel_mask = channel_mask[channel0:channel1]
e_vis, e_weights, e_uvw, e_feed_angle1, e_feed_angle2, e_polarizations = \
self._get_expected(dataset, channel_mask)
# Check the shapes before trying to shuffle data.
N = self.shape[0] * bls
C = e_vis.shape[1]
P = 4
assert_equal(vis.shape, (C, N, P))
assert_equal(weights.shape, (C, N, P))
assert_equal(uvw.shape, (N, 3))
assert_equal(feed_angle1.shape, (N,))
assert_equal(feed_angle2.shape, (N,))
# The expected visibilities are all unique, so we can use them to
# determine the permutation applied. We map each complex visibility to
# its time-frequency-baseline coordinates.
vis_map = {value: index for index, value in np.ndenumerate(e_vis)}
for channel in range(C):
for i in range(N):
# Handling for missing correlation products assumes that the
# first polarization product of a baseline will always be
# present.
baseline = dataset.corr_products[vis_map[vis[channel, i, 0]][2]]
ant1 = baseline[0][:-1]
ant2 = baseline[1][:-1]
for j in range(P):
key = (ant1, ant2, loader.polarizations()[j])
if key in missing:
# Doesn't matter what the values are; it just has to be flagged
assert_equal(weights[channel, i, j], 0.0)
continue
# pop ensures that we don't see the same visibility twice
idx = vis_map.pop(vis[channel, i, j])
assert_equal(idx[1], channel)
assert_equal(loader.polarizations()[j], e_polarizations[idx[2]])
assert_equal(vis[channel, i, j], e_vis[idx])
assert_equal(weights[channel, i, j], e_weights[idx])
np.testing.assert_allclose(
uvw[i].to_value(u.m),
e_uvw[idx[0], idx[2]].to_value(u.m), rtol=1e-7
)
# Won't match exactly because the loader converts to float32
np.testing.assert_allclose(
feed_angle1[i], e_feed_angle1[idx[0], idx[2]], rtol=1e-6)
np.testing.assert_allclose(
feed_angle2[i], e_feed_angle2[idx[0], idx[2]], rtol=1e-6)
assert_equal(len(vis_map), 0)
loader.close()
return chunks
def get_data(file, vSwitch):
"""Read hdf5 data and obtain relevant metadata information."""
h5 = katdal.open(file)
obs = h5.description
sensors = h5.sensor.keys()
tvals = h5.timestamps - h5.timestamps[0] # get updated timestamps (s)
descr = '\n'.join(
h5.__str__().split('\n')[:23]) # human-friendly header information
if (vSwitch == 1):
print '\n', descr # print header
return h5, obs, tvals, sensors
def __init__(self,
filename,
servers=None,
bchan=0,
echan=None,
n_substreams=1):
super().__init__(filename, servers, bchan, echan, n_substreams)
try:
self.file = katdal.open(filename, upgrade_flags=False)
except IOError as error:
raise WrongFileType(str(error)) from error
self.file.select(channels=slice(bchan, echan))
self.num_scans = len(self.file.scan_indices)
def readfile(path):
"""
Read in the RDB file.
Parameters
----------
path : str
RDB file
Returns
-------
output_file : katdal.visdatav4.VisibilityDataV4
katdal data object
"""
vis = katdal.open(path)
return vis
def select_and_average(filename, average_time):
# Read a file into katdal, and average the data to the prescribed averaging time
# Returns the weather data and timestamps with the correct averaging interval
data = katdal.open(filename)
raw_timestamps = data.sensor.timestamps
raw_wind_speed = data.wind_speed
raw_temperature = data.temperature
raw_dumptime = data.dump_period
# Get azel of each antenna and separation of each antenna
sun = katpoint.Target('Sun, special', antenna=data.ants[0])
alltimestamps = data.timestamps[:]
solar_seps = np.zeros_like(alltimestamps)
for dumpnum, timestamp in enumerate(alltimestamps):
azeltarget = katpoint.construct_azel_target(
katpoint.deg2rad(data.az[dumpnum, 0]),
katpoint.deg2rad(data.el[dumpnum, 0]))
azeltarget.antenna = data.ants[0]
solar_seps[dumpnum] = katpoint.rad2deg(
azeltarget.separation(sun, timestamp))
#Determine number of dumps to average
num_average = max(int(np.round(average_time / raw_dumptime)), 1)
#Array of block indices
indices = list(
range(min(num_average, raw_timestamps.shape[0]),
raw_timestamps.shape[0] + 1,
min(num_average, raw_timestamps.shape[0])))
timestamps = np.average(np.array(np.split(raw_timestamps, indices)[:-1]),
axis=1)
wind_speed = np.average(np.array(np.split(raw_wind_speed, indices)[:-1]),
axis=1)
temperature = np.average(np.array(np.split(raw_temperature, indices)[:-1]),
axis=1)
dump_time = raw_dumptime * num_average
return (timestamps, alltimestamps, wind_speed, temperature, dump_time,
solar_seps, data.ants[0])
def readOBS(filename, ant, pol, fc, fnull):
try:
h5 = katdal.open(filename, quicklook=True)
except Exception as err_msg:
raise SystemExit('An error has occured:\n%s' % err_msg)
h5.select(reset='T')
h5.select(ants=ant, pol=pol, corrprods='auto', scans='track')
visibilities = h5.vis[:]
scan_indices = h5.scan_indices
passband = h5.channel_freqs
chan_bw = (h5.spectral_windows[0]).channel_width
the_pointings = []
for idx in scan_indices:
h5.select(reset='T')
h5.select(ants=ant, pol=pol, corrprods='auto', scans=idx)
the_pointings.append(h5.shape[0])
# Channel indices for passband baseline
min_idx = numpy.argmin(numpy.abs(passband - f_start))
max_idx = numpy.argmin(numpy.abs(passband - f_stop))
baseline_idx = list(range(min_idx, max_idx))
# Extract passband only
visibilities = visibilities[:, baseline_idx, :]
passband = passband[baseline_idx]
# Channel indices over target
min_idx = numpy.argmin(numpy.abs(passband - (fc - dft / 2.)))
max_idx = numpy.argmin(numpy.abs(passband - (fc + dft / 2.)))
target_range = list(range(min_idx, max_idx))
# channel indices for null
min_idx = numpy.argmin(numpy.abs(passband - (fnull - dfn / 2)))
max_idx = numpy.argmin(numpy.abs(passband - (fnull + dfn / 2)))
null_range = list(range(min_idx, max_idx))
return [
the_pointings, visibilities, chan_bw, passband, target_range,
null_range
]
def analyse_spectrum(input_file,
output_dir='.',
polarisation='HH,VV',
baseline=None,
target=None,
freqav=None,
timeav=None,
freq_chans=None,
correct='spline',
flags_file=None,
smooth=3):
"""
Plot the mean and standard deviation of the bandpass amplitude for a given target in a file
Inputs
======
polarisation: Comma separated list of polarisations to produce spectrum of, options are I, HH, VV, HV, VH. Default is I.
baseline: Baseline to load (e.g. 'ant1,ant1' for antenna 1 auto-corr), default is first single-dish baseline in file.
target: Target to plot spectrum of, default is the first target in the file.
freqav: Frequency averaging interval in MHz. Default is a bin size that will produce 100 frequency channels.
timeav: Time averageing interval in minutes. Default is the shortest scan length on the selected target.
freq_chans: Range of frequency channels to keep (zero-based, specified as 'start,end', default is 90% of the bandpass.
correct: Method to use to correct the spectrum in each average timestamp. Options are 'spline' - fit a cubic spline,'channels' - use the average at each channel Default: 'spline'
output_dir: Output directory for pdfs. Default is cwd.
flags_file: Name of .h5 file containg flags calculated from 'rfi_report.py'.
debug: make a debug file containing all of the background fits to the dumps
"""
h5data = katdal.open(input_file)
#Get Baseline
if baseline == None:
baseline = h5data.ants[0].name + ',' + h5data.ants[0].name
#Set up plotting.
if type(input_file) == type(list()):
fileprefix = os.path.join(
output_dir,
os.path.splitext(input_file[0].split('/')[-1])[0])
else:
fileprefix = os.path.join(
output_dir,
os.path.splitext(input_file.split('/')[-1])[0])
basename = fileprefix + '_SpecBase_' + baseline.replace(',', '')
pdf = PdfPages(basename + '.pdf')
for this_pol in polarisation.split(','):
print(this_pol, "polarisation.")
# Get data from h5 file and use 'select' to obtain a useable subset of it.
visdata, weightdata, h5data = \
read_and_select_file(h5data, baseline, target=target, channels=freq_chans, polarisation=this_pol, flags_file=flags_file)
# Correct the visibilities by subtracting the average of the channels at each timestamp
#and the average of the timestamps at each channel.
if correct == 'channels':
corr_vis = correct_by_mean(visdata, axis="Channel")
corr_vis = correct_by_mean(corr_vis, axis="Time")
# Correct the background in each time bin by fitting a cubic spline.
else:
bg = onedbackground(smoothing=smooth, background_method=correct)
#Knots will have to satisfy Schoenberg-Whitney conditions for spline else revert to straight mean of channels
try:
print("Fitting background using " + correct + " smoothing.")
corr_vis = np.ma.masked_invalid(
np.ma.masked_array(
[data - bg.getbackground(data) for data in visdata],
mask=visdata.mask,
fill_value=0.0))
#Fill masked values with zero (these will not contribute to the average - and deals with nans returned from the spline fit creeping into the average)
removed_dumps = np.all(corr_vis.mask, axis=1)
print(np.sum(removed_dumps), "out of", len(removed_dumps),
"dumps have been rejected during fitting.")
except ValueError:
print(
"Background fitting failed- using mean deviation instead.")
corr_vis = correct_by_mean(visdata, axis="Channel")
corr_vis = correct_by_mean(corr_vis, axis="Time")
#Get the number of dumps to average
dumpav = max(1, int(np.round(timeav * 60.0 / h5data.dump_period)))
if dumpav > len(h5data.timestamps):
dumpav = 1
print(
"Time averaging interval of %4.1fmin is longer than the observation length. No time averaging will be applied."
% (timeav))
timeav = dumpav * (h5data.dump_period / 60.0)
print("Averaging time to %3d x %4.1fmin (%d dump) intervals." %
(len(h5data.timestamps) // dumpav, timeav, dumpav))
#Get the number of channels to average
chanav = max(1, int(np.round(freqav * 1e6 / h5data.channel_width)))
if chanav > len(h5data.channel_freqs):
chanav = 1
print(
"Frequency averaging interval of %4.1fMHz is wider than available bandwidth. No frequency averaging will be applied."
% (freqav))
freqav = h5data.channel_width / 1e6
print("Averaging frequency to %d x %4.1fMHz intervals." %
(len(h5data.channel_freqs) // chanav, freqav))
#Average the data over all time in chanav channels
av_visdata = averager.average_visibilities(visdata.data,
weightdata,
visdata.mask,
h5data.timestamps,
h5data.channel_freqs,
timeav=len(
h5data.timestamps),
chanav=chanav)
#Average the background subtracted data in dumpav times and chanav channels
av_corr_vis = averager.average_visibilities(corr_vis.filled(),
weightdata,
corr_vis.mask,
h5data.timestamps,
h5data.channel_freqs,
timeav=dumpav,
chanav=chanav)
#Get the averaged weights and channel frequencies
av_weightdata = av_corr_vis[1]
av_channel_freqs = av_corr_vis[4]
#Make a masked array out of the averaged visdata
av_visdata = np.ma.masked_array(np.squeeze(av_visdata[0]),
mask=np.squeeze(av_visdata[2]))
#Make a masked array out of the averaged background subtracted data
av_corr_vis = np.ma.masked_array(av_corr_vis[0], mask=av_corr_vis[2])
#get weighted standard deviation of background subtracted data
corr_vis_mean, corr_vis_std = weighted_avg_and_std(av_corr_vis,
av_weightdata,
axis=0)
obs_duration = np.str(
np.round((h5data.end_time.to_mjd() - h5data.start_time.to_mjd()) *
24 * 60, 2)) + ' min'
h5name = h5data.name.split('/')[-1]
obs_details = h5name + ', start ' + h5data.start_time.to_string(
) + ', duration ' + obs_duration
plot_std_results(corr_vis_std, np.squeeze(av_visdata),
av_channel_freqs, av_corr_vis.mask, baseline,
this_pol, freqav, timeav, obs_details, pdf)
pdf.close()
#!/usr/bin/env python
import katdal
import sys
#
filename=sys.argv[1]
#
h5=katdal.open(filename,refant='m063')
#x=h5.sensor.get('CorrelatorBeamformer/auto-delay')
#print "number of autodelays",x.events[1]
#print "min",min(x),"max",max(x)
for nn in h5.catalogue.targets:
print nn.name
print len(h5.scan_indices), "Scans"
print "antennas",h5.inputs
print "min elevation",h5.el.min()
print "max elevation",h5.el.max()
print "dump period",h5.dump_period
print "band",h5.channel_freqs.min()/1e9,"to",h5.channel_freqs.max()/1e9,"GHz"
print "start",h5.start_time.local(),"end",h5.end_time.local(),\
"duration",(h5.end_time-h5.start_time)/3600,"hrs"
channel_mask = opts.channel_mask #
output_fields = '%(dataset)s, %(target)s, %(timestamp_ut)s, %(azimuth).7f, %(elevation).7f, ' \
'%(delta_azimuth).7f, %(delta_azimuth_std).7f, %(delta_elevation).7f, %(delta_elevation_std).7f, ' \
'%(data_unit)s, %(beam_height_I).7f, %(beam_height_I_std).7f, %(beam_width_I).7f, ' \
'%(beam_width_I_std).7f, %(baseline_height_I).7f, %(baseline_height_I_std).7f, %(refined_I).7f, ' \
'%(beam_height_HH).7f, %(beam_width_HH).7f, %(baseline_height_HH).7f, %(refined_HH).7f, ' \
'%(beam_height_VV).7f, %(beam_width_VV).7f, %(baseline_height_VV).7f, %(refined_VV).7f, ' \
'%(frequency).7f, %(flux).4f, %(temperature).2f, %(pressure).2f, %(humidity).2f, %(wind_speed).2f, ' \
'%(wind_direction).2f , %(sun_az).7f, %(sun_el).7f, %(timestamp)i, %(valid_solutions)i \n'
output_field_names = [
name.partition(')')[0] for name in output_fields[2:].split(', %(')
]
h5 = katdal.open(args[0])
ant_list = [ant.name for ant in h5.ants] # Temp list for input options
if opts.ants is not None:
ant_list = opts.ants.split(',')
if opts.ex_ants is not None:
for ant in opts.ex_ants.split(','):
if ant in ant_list:
ant_list.remove(ant)
h5 = katdal.open(args[0], ref_ant=ant_list[0])
print("Using %s as the reference antenna " % (ant_list[0]))
h5.select(compscans='interferometric_pointing', ants=ant_list)
h5.antlist = [a.name for a in h5.ants]
h5.bls_lookup = calprocs.get_bls_lookup(h5.antlist, h5.corr_products)
if opts.outfilebase is None:
outfilebase = "%s_%s" % (h5.name.split('/')[-1].split('.')[0],
(opts, args) = parser.parse_args()
if len(args) ==0:
raise RuntimeError('Please specify an h5 file to load.')
# frequency channels to keep
start_freq_channel = int(opts.freq_keep.split(',')[0])
end_freq_channel = int(opts.freq_keep.split(',')[1])
start_freq_channel = 200
end_freq_channel = 800
h5 = katdal.open(args)
#h5 = katdal.open('1387000585.h5')
nice_filename = args[0]+ '_phase_stability'
pp = PdfPages(nice_filename+'.pdf')
for pol in ('h','v'):
h5.select(channels=slice(start_freq_channel,end_freq_channel),pol=pol,corrprods='cross',scans='track',dumps=slice(1,600))
# loop over both polarisations
if np.all(h5.sensor['DBE/auto-delay'] == '0') :
print "Need to do fringe stopping "
vis = fringe_stopping(h5)
else:
print "Fringe stopping done in the correlator"
vis = h5.vis[:,:,:]
flaglist = ~h5.flags()[:,:,:].any(axis=0).any(axis=-1)
#flaglist[0:start_freq_channel] = False
help=
"The nuber of bins to use when evaluation the different seperations', default = '%default'"
)
parser.add_option(
"--ant",
default='ant4',
help="The antenna to do the reduction for', default = '%default'")
(opts, args) = parser.parse_args()
if len(args) < 1:
raise RuntimeError('Please specify the data file to reduce')
height = 1.0
bins = opts.bins
h5 = katdal.open(args[0])
#1392246099.h5
nice_filename = args[0].split('/')[-1] + '_' + opts.ant + '_jitter_test'
pp = PdfPages(nice_filename + '.pdf')
h5.select(scans='track',
ants=opts.ant,
channels=slice((h5.channels.shape[0]) // 4,
(h5.channels.shape[0] * 3) // 4))
pos1, pos2 = np.radians(
(h5.az[:, 0],
h5.el[:, 0])), np.array(h5.catalogue.targets[1].azel(h5.timestamps[:]))
hpbw = fwhm = np.degrees(1.02 * (c / h5.channel_freqs) / h5.ants[0].diameter)
pos1, pos2 = np.radians(
def MKContPipeline(files, outputdir, **kwargs):
"""MeerKAT Continuum pipeline.
Parameters
----------
files : list
h5 filenames (note: support for multiple h5 files
i.e. ConcatenatedDataSet is not currently supported)
outputdir : string
Directory location to write output data,
scratchdir : string, optional
The directory location of the aips disk
parmFile : string, optional
Overwrite the default imaging parameters using this parameter file.
"""
if len(files) == 1:
h5file = files[0]
else:
h5file = files
############### Initialize katfile object #########################
OK = False
# Open the h5 file as a katfile object
try:
#open katfile and perform selection according to kwargs
katdal_ref_ant = kwargs.get('katdal_refant', '')
katdal_retries = kwargs.get('katdal_retries', 2)
katdal_timeout = kwargs.get('katdal_timeout', 300)
katdata = katfile.open(h5file,
ref_ant=katdal_ref_ant,
timeout=katdal_timeout,
retries=katdal_retries)
OK = True
except Exception as exception:
print(exception)
if not OK:
raise KATUnimageableError("Unable to read MVF data in " + str(h5file))
# If we are doing polcal- search for the most recent delaycal observation
if kwargs.get('polcal'):
if kwargs.get('delaycal_mvf') is None:
# Automatically determine delay_cal CBID
delay_katdata = KATGetDelayCal(h5file,
katdata,
timeout=katdal_timeout,
retries=katdal_retries)
else:
# Use the user supplied one
delay_katdata = KATGetDelayCal(kwargs.get('delaycal_mvf'))
kwargs["delay_katdata"] = delay_katdata
# Die gracefully if we cannot write to the output area...
if not os.path.exists(outputdir):
print('Specified output directory: ' + outputdir + 'does not exist.')
exit(-1)
# Obit error logging
err = OErr.OErr()
#################### Initialize filenames #######################################################
nameRoot = katdata.obs_params.get('capture_block_id',
katdata.experiment_id)
if type(nameRoot) == list:
nameRoot = nameRoot[0]
fileRoot = os.path.join(outputdir, nameRoot) # root of file name
logFile = fileRoot + ".log" # Processing log file
avgClass = ("UVAv")[0:6] # Averaged data AIPS class
manifestfile = outputdir + '/manifest.pickle'
############################# Initialize OBIT and AIPS ##########################################
noScrat = []
# Logging directly to logFile
OErr.PInit(err, 2, logFile)
EVLAAddOutFile(os.path.basename(logFile), 'project', 'Pipeline log file')
if kwargs.get('reuse'):
ObitSys = AIPSSetup.AIPSSetup(err,
configfile=kwargs.get('configFile'),
scratchdir=kwargs.get('scratchdir'),
aipsdisk=kwargs.get('aipsdisk'),
overwrite=False)
else:
ObitSys = AIPSSetup.AIPSSetup(err,
configfile=kwargs.get('configFile'),
scratchdir=kwargs.get('scratchdir'),
aipsdisk=kwargs.get('aipsdisk'))
# Get the set up AIPS environment.
AIPS_ROOT = os.environ['AIPS_ROOT']
AIPS_VERSION = os.environ['AIPS_VERSION']
nThreads = 72
user = OSystem.PGetAIPSuser()
AIPS.userno = user
disk = 1
fitsdisk = 1
nam = nameRoot[:10]
clss = "Raw"
seq = 1
############### Condition data #########################
#Get calibrator models
fluxcals = katpoint.Catalogue(
open(FITSDir.FITSdisks[1] + "/PERLEY_BUTLER_2013.csv"))
#Condition data (get bpcals, update names for aips conventions etc)
KATh5Condition(katdata, fluxcals, err)
############################# Initialise Parameters ##########################################
####### Initialize parameters dictionary #####
parms = KATInitContParms()
parms['PolCal'] = kwargs.get('polcal')
parms['XYtarg'] = kwargs.get('XYtarg')
# Get default XYtarg if it is not set
targs = [targ.name for targ in katdata.catalogue.targets]
if parms['PolCal']:
if parms['XYtarg'] is None:
GOTTARG = False
for targ in ['1934-638', '0408-65']:
if targ in targs:
parms['XYtarg'] = targ
GOTTARG = True
break
if not GOTTARG:
raise RuntimeError(
'No default targets (1934-638, 0408-65) for XYFix. Cannot run in PolCal mode.'
)
else:
if parms['XYtarg'] not in targs:
raise RuntimeError(
'XYtarg target %s not in observation. Cannot run in PolCal mode.'
% (parms['XYtarg']))
####### User defined parameters ######
if kwargs.get('parmFile'):
print("parmFile", kwargs.get('parmFile'))
exec(open(kwargs.get('parmFile')).read())
EVLAAddOutFile(os.path.basename(kwargs.get('parmFile')), 'project',
'Pipeline input parameters')
###################### Data selection and static edits ############################################
# Select data based on static imageable parameters
KATh5Select(katdata, parms, err, **kwargs)
# General AIPS data parameters at script level
dataClass = ("UVDa")[0:6] # AIPS class of raw uv data
delayClass = "DELA"
band = katdata.spectral_windows[0].product #Correlator product
project = os.path.basename(os.path.splitext(files[0])[0])[
0:10] # Project name (12 char or less, used as AIPS Name)
outIClass = parms["outIClass"] # image AIPS class
debug = parms["debug"]
check = parms["check"]
####################### Import data into AIPS #####################################################
# Reuse or nay?
sw = katdata.spectral_windows[katdata.spw]
# Pick up static flags
if sw.band == 'L':
sflags = FetchObject(ObitTalkUtil.FITSDir.FITSdisks[fitsdisk] +
'maskred.pickle')
if kwargs.get('flag', None):
mess = 'Using static RFI mask in file %s for L-band' % (
ObitTalkUtil.FITSDir.FITSdisks[fitsdisk] + 'maskred.pickle', )
printMess(mess, logFile)
elif sw.band == 'UHF':
sflags = FetchObject(ObitTalkUtil.FITSDir.FITSdisks[fitsdisk] +
'maskredUHF.pickle')
if kwargs.get('flag', None):
mess = 'Using static RFI mask in file %s for UHF-band' % (
ObitTalkUtil.FITSDir.FITSdisks[fitsdisk] +
'maskredUHF.pickle', )
printMess(mess, logFile)
else:
sflags = np.zeros(sw.num_chans, dtype=np.bool)
sflags = sflags[katdata.channels]
# Construct a template uvfits file from master template
mastertemplate = ObitTalkUtil.FITSDir.FITSdisks[
fitsdisk] + 'MKATTemplate.uvtab.gz'
outtemplate = nam + '.uvtemp'
if kwargs.get('reuse'):
uv = UV.newPAUV("AIPS UV DATA", EVLAAIPSName(project), dataClass, disk,
seq, True, err)
obsdata = KATH5toAIPS.GetKATMeta(katdata, err)
# Extract AIPS parameters of the uv data to the metadata
obsdata["Aproject"] = uv.Aname
obsdata["Aclass"] = uv.Aclass
obsdata["Aseq"] = uv.Aseq
obsdata["Adisk"] = disk
obsdata["calInt"] = katdata.dump_period
obsdata["fitsdisk"] = fitsdisk
# TODO: Check if the input data has been Hanned.
doneHann = True
else:
mess = '\nLoading UV data with CBID: %s' % (
katdata.obs_params['capture_block_id'], )
printMess(mess, logFile)
KATH5toAIPS.MakeTemplate(mastertemplate, outtemplate, katdata)
uv = OTObit.uvlod(outtemplate, 0, EVLAAIPSName(project), clss, disk,
seq, err)
obsdata = KATH5toAIPS.KAT2AIPS(katdata,
uv,
disk,
fitsdisk,
err,
calInt=katdata.dump_period,
static=sflags,
**kwargs)
MakeIFs.UVMakeIF(uv, 8, err, solInt=katdata.dump_period)
os.remove(outtemplate)
if parms["PolCal"]:
mess = '\nLoading delay calibration with CBID: %s' % (
delay_katdata.obs_params['capture_block_id'], )
printMess(mess, logFile)
# Load the delay cal observation
KATH5toAIPS.MakeTemplate(mastertemplate, outtemplate, katdata)
delay_uv = OTObit.uvlod(outtemplate, 0, EVLAAIPSName(project),
delayClass, disk, seq, err)
KATH5toAIPS.KAT2AIPS(delay_katdata,
delay_uv,
disk,
fitsdisk,
err,
calInt=katdata.dump_period,
static=sflags,
flag=False)
MakeIFs.UVMakeIF(delay_uv, 8, err, solInt=katdata.dump_period)
os.remove(outtemplate)
# Print the uv data header to screen.
uv.Header(err)
############################# Set Project Processing parameters ###################################
# Parameters derived from obsdata and katdata
KATGetObsParms(obsdata, katdata, parms, logFile)
###### Initialise target parameters #####
KATInitTargParms(katdata, parms, err)
# Load the outputs pickle jar
EVLAFetchOutFiles()
OSystem.PAllowThreads(nThreads) # Allow threads in Obit/oython
retCode = 0
doBand = -1
BPVer = 0
maxgap = max(parms["CalAvgTime"], 160. * katdata.dump_period) / 60.
################### Start processing ###############################################################
mess = "Start project "+parms["project"]+" AIPS user no. "+str(AIPS.userno)+\
", KAT7 configuration "+parms["KAT7Cfg"]
printMess(mess, logFile)
if debug:
pydoc.ttypager = pydoc.plainpager # don't page task input displays
mess = "Using Debug mode "
printMess(mess, logFile)
if check:
mess = "Only checking script"
printMess(mess, logFile)
# Log parameters
printMess("Parameter settings", logFile)
for p in parms:
mess = " " + p + ": " + str(parms[p])
printMess(mess, logFile)
clist = []
for DCal in parms["DCals"]:
if DCal["Source"] not in clist:
clist.append(DCal["Source"])
for PCal in parms["PCals"]:
if PCal["Source"] not in clist:
clist.append(PCal["Source"])
for ACal in parms["ACals"]:
if ACal["Source"] not in clist:
clist.append(ACal["Source"])
if kwargs.get('targets') is not None:
targets = [
targ.name for targ in katdata.catalogue
if (targ.name not in clist) and (
targ.name in kwargs.get('targets').split(','))
]
else:
targets = [
targ.name for targ in katdata.catalogue if (targ.name not in clist)
]
refAnt = kwargs.get('refant')
if refAnt is not None:
try:
SaveObject(obsdata['antLookup'][refAnt],
fileRoot + ".refAnt.pickle", True)
except:
mess = "Select reference antenna " + refAnt + " not in antenna table."
printMess(mess, logFile)
print(mess)
refAnt = FetchObject(fileRoot + ".refAnt.pickle")
# Save parameters to pickle jar, manifest
ParmsPicklefile = fileRoot + ".Parms.pickle" # Where results saved
SaveObject(parms, ParmsPicklefile, True)
EVLAAddOutFile(os.path.basename(ParmsPicklefile), 'project',
'Processing parameters used')
loadClass = dataClass
# Hanning - only if not reusing
doneHann = False
if not kwargs.get('reuse'):
if parms["doHann"]:
uv = KATHann(uv, EVLAAIPSName(project), dataClass, disk, seq, err, \
doDescm=parms["doDescm"], flagVer=-1, logfile=logFile, zapin=True, check=check, debug=debug)
doneHann = True
if parms["PolCal"] and parms["doHann"]:
mess = "Hanning delay calibration scan"
printMess(mess, logFile)
delay_uv = KATHann(delay_uv, EVLAAIPSName(project), delayClass, disk, seq + 1, err, \
doDescm=parms["doDescm"], flagVer=-1, logfile=logFile, zapin=True, check=check, debug=debug)
if doneHann:
# Halve channels after hanning.
parms["selChan"] = int(parms["selChan"] / 2)
parms["BChDrop"] = int(parms["BChDrop"] / 2)
parms["EChDrop"] = int(parms["EChDrop"] / 2)
if uv == None and not check:
raise RuntimeError("Cannot Hann data ")
# Clear any old calibration/editing
if parms["doClearTab"] or kwargs.get('reuse'):
mess = "Clear previous calibration"
printMess(mess, logFile)
EVLAClearCal(uv,
err,
doGain=parms["doClearGain"],
doFlag=parms["doClearFlag"],
doBP=parms["doClearBP"],
check=check)
OErr.printErrMsg(err, "Error resetting calibration")
# Copy FG 1 to FG 2
if parms["doCopyFG"]:
mess = "Copy FG 1 to FG 2"
printMess(mess, logFile)
retCode = KATCopyFG(uv, err, logfile=logFile, check=check, debug=debug)
if retCode != 0:
raise RuntimeError("Error Copying FG table")
# Flag antennas shadowed by others?
if parms["doShad"]:
retCode = EVLAShadow (uv, err, shadBl=parms["shadBl"], \
logfile=logFile, check=check, debug=debug)
if retCode != 0:
raise RuntimeError("Error Shadow flagging data")
# Median window time editing, for RFI impulsive in time
if parms["doMednTD1"]:
mess = "Median window time editing, for RFI impulsive in time:"
printMess(mess, logFile)
retCode = EVLAMedianFlag (uv, clist, err, noScrat=noScrat, nThreads=nThreads, \
avgTime=parms["mednAvgTime"], avgFreq=parms["mednAvgFreq"], chAvg= parms["mednChAvg"], \
timeWind=parms["mednTimeWind"],flagVer=2, flagTab=2,flagSig=parms["mednSigma"], \
logfile=logFile, check=check, debug=False)
if retCode != 0:
raise RuntimeError("Error in MednFlag")
# Median window frequency editing, for RFI impulsive in frequency
if parms["doFD1"]:
mess = "Median window frequency editing, for RFI impulsive in frequency:"
printMess(mess, logFile)
retCode = EVLAAutoFlag (uv, clist, err, flagVer=2, flagTab=2, doCalib=-1, doBand=-1, \
timeAvg=parms["FD1TimeAvg"], \
doFD=True, FDmaxAmp=1.0e20, FDmaxV=1.0e20, FDwidMW=parms["FD1widMW"], \
FDmaxRMS=[1.0e20,0.1], FDmaxRes=parms["FD1maxRes"], \
FDmaxResBL= parms["FD1maxRes"], FDbaseSel=parms["FD1baseSel"],\
nThreads=nThreads, logfile=logFile, check=check, debug=debug)
if retCode != 0:
raise RuntimeError("Error in AutoFlag")
# Parallactic angle correction?
if parms["doPACor"]:
retCode = EVLAPACor(uv, err, \
logfile=logFile, check=check, debug=debug)
if retCode != 0:
raise RuntimeError("Error in Parallactic angle correction")
# Need to find a reference antenna? See if we have saved it?
if (parms["refAnt"] <= 0):
refAnt = FetchObject(fileRoot + ".refAnt.pickle")
if refAnt:
parms["refAnt"] = refAnt
# Use bandpass calibrator and center half of each spectrum
if parms["refAnt"] <= 0:
mess = "Find best reference antenna: run Calib on BP Cal(s) "
printMess(mess, logFile)
parms["refAnt"] = EVLAGetRefAnt(uv, parms["BPCals"], err, flagVer=0, \
solInt=parms["bpsolint1"], nThreads=nThreads, \
logfile=logFile, check=check, debug=debug)
if err.isErr:
raise RuntimeError("Error finding reference antenna")
if parms["refAnts"][0] <= 0:
parms["refAnts"][0] = parms["refAnt"]
mess = "Picked reference antenna " + str(parms["refAnt"])
printMess(mess, logFile)
# Save it
ParmsPicklefile = fileRoot + ".Parms.pickle" # Where results saved
SaveObject(parms, ParmsPicklefile, True)
refAntPicklefile = fileRoot + ".refAnt.pickle" # Where results saved
SaveObject(parms["refAnt"], refAntPicklefile, True)
# Plot Raw, edited data?
if parms["doRawSpecPlot"] and parms["plotSource"]:
mess = "Raw Spectral plot for: " + ' '.join(parms["BPCal"])
printMess(mess, logFile)
plotFile = fileRoot + "_RawSpec.ps"
retCode = EVLASpectrum(uv, parms["BPCal"], parms["plotTime"], maxgap, plotFile, parms["refAnt"], err, \
Stokes=["RR","LL"], doband=-1, \
check=check, debug=debug, logfile=logFile )
if retCode != 0:
raise RuntimeError("Error in Plotting spectrum")
EVLAAddOutFile(plotFile, 'project', 'Pipeline log file')
if parms["PolCal"]:
mess = "XYphase bandpass calibration"
printMess(mess, logFile)
retCode = KATXPhase(delay_uv,
uv,
err,
logfile=logFile,
check=check,
debug=debug,
doCalib=-1,
flagVer=0,
doBand=-1,
refAnt=parms['refAnt'])
doBand = 1
BPVer += 1
if retCode != 0:
raise RuntimeError("Error in Xphase calibration")
# delay calibration
if parms["doDelayCal"] and parms["DCals"] and not check:
plotFile = fileRoot + "_DelayCal.ps"
retCode = EVLADelayCal(uv, parms["DCals"], err, \
BChan=parms["delayBChan"], EChan=parms["delayEChan"], \
doCalib=-1, flagVer=0, doBand=doBand, BPVer=BPVer, \
solInt=parms["delaySolInt"], smoTime=parms["delaySmoo"], \
refAnts=[parms["refAnt"]], doTwo=parms["doTwo"],
doZeroPhs=parms["delayZeroPhs"], \
doAvgIF=parms["delayAvgIF"], doAvgPol=parms["delayAvgPol"], \
doPlot=parms["doSNPlot"], plotFile=plotFile, \
nThreads=nThreads, noScrat=noScrat, \
logfile=logFile, check=check, debug=debug)
if retCode != 0:
raise RuntimeError("Error in delay calibration")
# Plot corrected data?
if parms["doSpecPlot"] and parms["plotSource"]:
plotFile = fileRoot + "_DelaySpec.ps"
retCode = EVLASpectrum(uv, parms["BPCal"], parms["plotTime"], maxgap, \
plotFile, parms["refAnt"], err, \
Stokes=["RR","LL"], doband=doBand, \
check=check, debug=debug, logfile=logFile )
if retCode != 0:
raise RuntimeError("Error in Plotting spectrum")
# Bandpass calibration
if parms["doBPCal"] and parms["BPCals"]:
retCode = KATBPCal(uv, parms["BPCals"], err, doBand=doBand, BPVer=BPVer, newBPVer=0,
noScrat=noScrat, solInt1=parms["bpsolint1"], \
solInt2=parms["bpsolint2"], solMode=parms["bpsolMode"], \
BChan1=parms["bpBChan1"], EChan1=parms["bpEChan1"], \
BChan2=parms["bpBChan2"], EChan2=parms["bpEChan2"], ChWid2=parms["bpChWid2"], \
doCenter1=parms["bpDoCenter1"], refAnt=parms["refAnt"], \
UVRange=parms["bpUVRange"], doCalib=2, gainUse=0, flagVer=0, doPlot=False, \
nThreads=nThreads, logfile=logFile, check=check, debug=debug)
if retCode != 0:
raise RuntimeError("Error in Bandpass calibration")
# Plot corrected data?
if parms["doSpecPlot"] and parms["plotSource"]:
plotFile = fileRoot + "_BPSpec.ps"
retCode = EVLASpectrum(uv, parms["BPCal"], parms["plotTime"], maxgap, plotFile, \
parms["refAnt"], err, Stokes=["RR","LL"], doband=1, \
check=check, debug=debug, logfile=logFile )
if retCode != 0:
raise RuntimeError("Error in Plotting spectrum")
# Amp & phase Calibrate
if parms["doAmpPhaseCal"]:
plotFile = fileRoot + "_APCal.ps"
retCode = KATCalAP (uv, [], parms["ACals"], err, PCals=parms["PCals"],
doCalib=2, doBand=1, BPVer=0, flagVer=0, \
BChan=parms["ampBChan"], EChan=parms["ampEChan"], \
solInt=parms["solInt"], solSmo=parms["solSmo"], ampScalar=parms["ampScalar"], \
doAmpEdit=parms["doAmpEdit"], ampSigma=parms["ampSigma"], \
ampEditFG=parms["ampEditFG"], avgPol=parms["PolCal"], \
doPlot=parms["doSNPlot"], plotFile=plotFile, refAnt=parms["refAnt"], \
nThreads=nThreads, noScrat=noScrat, logfile=logFile, check=check, debug=debug)
if retCode != 0:
raise RuntimeError("Error calibrating")
# More editing
if parms["doAutoFlag"]:
mess = "Post calibration editing:"
printMess(mess, logFile)
# if going to redo then only calibrators
if parms["doRecal"]:
# Only calibrators
clist = []
for DCal in parms["DCals"]:
if DCal["Source"] not in clist:
clist.append(DCal["Source"])
for PCal in parms["PCals"]:
if PCal["Source"] not in clist:
clist.append(PCal["Source"])
for ACal in parms["ACals"]:
if ACal["Source"] not in clist:
clist.append(ACal["Source"])
else:
clist = []
retCode = EVLAAutoFlag (uv, clist, err, flagVer=0, flagTab =2, \
doCalib=2, gainUse=0, doBand=1, BPVer=BPVer, \
IClip=parms["IClip"], minAmp=parms["minAmp"], timeAvg=parms["timeAvg"], \
doFD=parms["doFirstAFFD"], FDmaxAmp=parms["FDmaxAmp"], FDmaxV=parms["FDmaxV"], \
FDwidMW=parms["FDwidMW"], FDmaxRMS=parms["FDmaxRMS"], \
FDmaxRes=parms["FDmaxRes"], FDmaxResBL=parms["FDmaxResBL"], \
FDbaseSel=parms["FDbaseSel"], \
nThreads=nThreads, logfile=logFile, check=check, debug=debug)
if retCode != 0:
raise RuntimeError("Error in AutoFlag")
# Redo the calibration using new flagging?
if parms["doBPCal2"] == None:
parms["doBPCal2"] = parms["doBPCal"]
if parms["doDelayCal2"] == None:
parms["doDelayCal2"] = parms["doDelayCal2"]
if parms["doAmpPhaseCal2"] == None:
parms["doAmpPhaseCal2"] = parms["doAmpPhaseCal"]
if parms["doAutoFlag2"] == None:
parms["doAutoFlagCal2"] = parms["doAutoFlag"]
if parms["doRecal"]:
mess = "Redo calibration:"
printMess(mess, logFile)
EVLAClearCal(uv,
err,
doGain=True,
doFlag=False,
doBP=True,
check=check,
logfile=logFile)
OErr.printErrMsg(err, "Error resetting calibration")
BPVer = 0
# Parallactic angle correction?
if parms["doPACor"]:
retCode = EVLAPACor(uv, err, \
logfile=logFile, check=check, debug=debug)
if retCode != 0:
raise RuntimeError("Error in Parallactic angle correction")
# Run MKXPhase on delaycal data and attach BP table to UV data
if parms["PolCal"]:
mess = "XYphase bandpass calibration"
printMess(mess, logFile)
retCode = KATXPhase(delay_uv,
uv,
err,
logfile=logFile,
check=check,
debug=debug,
doCalib=-1,
flagVer=0,
doBand=-1,
refAnt=parms['refAnt'])
BPVer += 1
if retCode != 0:
raise RuntimeError("Error in Xphase calibration")
# Delay recalibration
if parms["doDelayCal2"] and parms["DCals"] and not check:
plotFile = fileRoot + "_DelayCal2.ps"
retCode = EVLADelayCal(uv, parms["DCals"], err, \
BChan=parms["delayBChan"], EChan=parms["delayEChan"], \
doCalib=-1, flagVer=0, doBand=doBand, BPVer=BPVer, \
solInt=parms["delaySolInt"], smoTime=parms["delaySmoo"], \
refAnts=[parms["refAnt"]], doTwo=parms["doTwo"], \
doZeroPhs=parms["delayZeroPhs"], \
doAvgIF=parms["delayAvgIF"], doAvgPol=parms["delayAvgPol"], \
doPlot=parms["doSNPlot"], plotFile=plotFile, \
nThreads=nThreads, noScrat=noScrat, \
logfile=logFile, check=check, debug=debug)
if retCode != 0:
raise RuntimeError("Error in delay calibration")
# Plot corrected data?
if parms["doSpecPlot"] and parms["plotSource"]:
plotFile = fileRoot + "_DelaySpec2.ps"
retCode = EVLASpectrum(uv, parms["BPCal"], parms["plotTime"], maxgap, plotFile, parms["refAnt"], err, \
Stokes=["RR","LL"], doband=doband, \
check=check, debug=debug, logfile=logFile )
if retCode != 0:
raise RuntimeError("Error in Plotting spectrum")
# Bandpass calibration
if parms["doBPCal2"] and parms["BPCals"]:
retCode = KATBPCal(uv, parms["BPCals"], err, doBand=doBand, BPVer=BPVer, newBPVer=0, \
noScrat=noScrat, solInt1=parms["bpsolint1"], \
solInt2=parms["bpsolint2"], solMode=parms["bpsolMode"], \
BChan1=parms["bpBChan1"], EChan1=parms["bpEChan1"], \
BChan2=parms["bpBChan2"], EChan2=parms["bpEChan2"], ChWid2=parms["bpChWid2"], \
doCenter1=parms["bpDoCenter1"], refAnt=parms["refAnt"], \
UVRange=parms["bpUVRange"], doCalib=2, gainUse=0, flagVer=0, doPlot=False, \
nThreads=nThreads, logfile=logFile, check=check, debug=debug)
if retCode != 0:
raise RuntimeError("Error in Bandpass calibration")
# Plot corrected data?
if parms["doSpecPlot"] and parms["plotSource"]:
plotFile = fileRoot + "_BPSpec2.ps"
retCode = EVLASpectrum(uv, parms["BPCal"], parms["plotTime"], maxgap, plotFile, parms["refAnt"], err, \
Stokes=["RR","LL"], doband=1, \
check=check, debug=debug, logfile=logFile )
if retCode != 0:
raise RuntimeError("Error in Plotting spectrum")
# Amp & phase Recalibrate
if parms["doAmpPhaseCal2"]:
plotFile = fileRoot + "_APCal2.ps"
retCode = KATCalAP (uv, [], parms["ACals"], err, PCals=parms["PCals"], \
doCalib=2, doBand=1, BPVer=0, flagVer=0, \
BChan=parms["ampBChan"], EChan=parms["ampEChan"], \
solInt=parms["solInt"], solSmo=parms["solSmo"], ampScalar=parms["ampScalar"], \
doAmpEdit=True, ampSigma=parms["ampSigma"], \
ampEditFG=parms["ampEditFG"], avgPol=parms["PolCal"], \
doPlot=parms["doSNPlot"], plotFile=plotFile, refAnt=parms["refAnt"], \
noScrat=noScrat, nThreads=nThreads, logfile=logFile, check=check, debug=debug)
if retCode != 0:
raise RuntimeError("Error calibrating")
# More editing
if parms["doAutoFlag2"]:
mess = "Post recalibration editing:"
printMess(mess, logFile)
retCode = EVLAAutoFlag (uv, [], err, flagVer=0, flagTab=2, \
doCalib=2, gainUse=0, doBand=1, BPVer=0, \
IClip=parms["IClip"], minAmp=parms["minAmp"], timeAvg=parms["timeAvg"], \
doFD=parms["doSecAFFD"], FDmaxAmp=parms["FDmaxAmp"], FDmaxV=parms["FDmaxV"], \
FDwidMW=parms["FDwidMW"], FDmaxRMS=parms["FDmaxRMS"], \
FDmaxRes=parms["FDmaxRes"], FDmaxResBL= parms["FDmaxResBL"], \
FDbaseSel=parms["FDbaseSel"], \
nThreads=nThreads, logfile=logFile, check=check, debug=debug)
if retCode != 0:
raise RuntimeError("Error in AutoFlag")
# end recal
# Calibrate and average data
# Overwrite avgStokes from command line
if kwargs.get('halfstokes'):
parms["avgStokes"] = 'HALF'
if parms["doCalAvg"] == 'Splat':
retCode = KATCalAvg (uv, avgClass, parms["seq"], parms["CalAvgTime"], err, \
flagVer=2, doCalib=2, gainUse=0, doBand=1, BPVer=0, doPol=False, \
avgFreq=parms["avgFreq"], chAvg=parms["chAvg"], Stokes=parms["avgStokes"], \
BChan=1, EChan=parms["selChan"] - 1, doAuto=parms["doAuto"], \
BIF=parms["CABIF"], EIF=parms["CAEIF"], Compress=parms["Compress"], \
nThreads=nThreads, logfile=logFile, check=check, debug=debug)
if retCode != 0:
raise RuntimeError("Error in CalAvg")
elif parms["doCalAvg"] == 'BL':
retCode = KATBLCalAvg (uv, avgClass, parms["seq"], err, \
flagVer=2, doCalib=2, gainUse=0, doBand=1, BPVer=0, doPol=False, \
avgFreq=parms["avgFreq"], chAvg=parms["chAvg"], FOV=parms['FOV'], \
maxInt=min(parms["solPInt"],parms["solAInt"]), Stokes=parms["avgStokes"], \
BChan=1, EChan=parms["selChan"] - 1, timeAvg=parms["CalAvgTime"], \
BIF=parms["CABIF"], EIF=parms["CAEIF"], Compress=parms["Compress"], \
logfile=logFile, check=check, debug=debug)
if retCode != 0:
raise RuntimeError("Error in BLCalAvg")
if parms["doSaveTab"]:
filename = project + ".CalTab.uvtab"
_ = EVLAUVFITSTab(uv, filename, 0, err, logfile=logFile)
#Zap unaveraged data if requested
if kwargs.get('zapraw'):
uv.Zap(err)
# Get calibrated/averaged data
if not check:
uv = UV.newPAUV("AIPS UV DATA", EVLAAIPSName(project), avgClass[0:6], \
disk, parms["seq"], True, err)
if err.isErr:
OErr.printErrMsg(err, "Error creating cal/avg AIPS data")
plotFile = fileRoot + "_Spec.ps"
retCode = EVLASpectrum(uv, parms["BPCal"], parms["plotTime"], maxgap, \
plotFile, parms["refAnt"], err, \
Stokes=["I"], doband=-1, docalib=-1, \
check=check, debug=debug, logfile=logFile )
if retCode != 0:
raise RuntimeError("Error in Plotting spectrum")
# KATUVFITS(uv, 'preimage.uvfits', 0, err, exclude=["AIPS HI", "AIPS SL", "AIPS PL"],
# include=["AIPS AN", "AIPS FQ"], compress=parms["Compress"], logfile=logFile)
KATUVFITab(uv, project + '.uvtab', 0, err)
#Gzip the data?
if kwargs.get('gzip'):
os.system('pigz -p %d %s' % (nThreads, project + '.uvtab'))
os.system('rm -f %s' % (project + '.uvtab'))
def main():
tag_to_intent = {
'gaincal': 'CALIBRATE_PHASE,CALIBRATE_AMPLI',
'bpcal': 'CALIBRATE_BANDPASS,CALIBRATE_FLUX',
'target': 'TARGET'
}
usage = "%prog [options] <dataset> [<dataset2>]*"
description = "Convert MVF dataset(s) to CASA MeasurementSet. The datasets may " \
"be local filenames or archive URLs (including access tokens). " \
"If there are multiple datasets they will be concatenated via " \
"katdal before conversion."
parser = optparse.OptionParser(usage=usage, description=description)
parser.add_option("-o",
"--output-ms",
default=None,
help="Name of output MeasurementSet")
parser.add_option(
"-c",
"--circular",
action="store_true",
default=False,
help="Produce quad circular polarisation. (RR, RL, LR, LL) "
"*** Currently just relabels the linear pols ****")
parser.add_option(
"-r",
"--ref-ant",
help="Override the reference antenna used to pick targets "
"and scans (default is the 'array' antenna in MVFv4 "
"and the first antenna in older formats)")
parser.add_option("-t",
"--tar",
action="store_true",
default=False,
help="Tar-ball the MS")
parser.add_option(
"-f",
"--full_pol",
action="store_true",
default=False,
help="Produce a full polarisation MS in CASA canonical order "
"(HH, HV, VH, VV). Default is to produce HH,VV only.")
parser.add_option("-v",
"--verbose",
action="store_true",
default=False,
help="More verbose progress information")
parser.add_option("-w",
"--stop-w",
action="store_true",
default=False,
help="Use W term to stop fringes for each baseline")
parser.add_option("-p",
"--pols-to-use",
default=None,
help="Select polarisation products to include in MS as "
"comma-separated list (from: HH, HV, VH, VV). "
"Default is all available from (HH, VV).")
parser.add_option(
"-u",
"--uvfits",
action="store_true",
default=False,
help="Print command to convert MS to miriad uvfits in casapy")
parser.add_option("-a",
"--no-auto",
action="store_true",
default=False,
help="MeasurementSet will exclude autocorrelation data")
parser.add_option(
"-s",
"--keep-spaces",
action="store_true",
default=False,
help="Keep spaces in source names, default removes spaces")
parser.add_option(
"-C",
"--channel-range",
help="Range of frequency channels to keep (zero-based inclusive "
"'first_chan,last_chan', default is all channels)")
parser.add_option("-e",
"--elevation-range",
help="Flag elevations outside the range "
"'lowest_elevation,highest_elevation'")
parser.add_option(
"-m",
"--model-data",
action="store_true",
default=False,
help="Add MODEL_DATA and CORRECTED_DATA columns to the MS. "
"MODEL_DATA initialised to unity amplitude zero phase, "
"CORRECTED_DATA initialised to DATA.")
flag_names = ', '.join(name for name in FLAG_NAMES
if not name.startswith('reserved'))
parser.add_option("--flags",
default="all",
help="List of online flags to apply "
"(from " + flag_names + ") "
"default is all flags, '' will apply no flags)")
parser.add_option("--dumptime",
type=float,
default=0.0,
help="Output time averaging interval in seconds, "
"default is no averaging")
parser.add_option("--chanbin",
type=int,
default=0,
help="Bin width for channel averaging in channels, "
"default is no averaging")
parser.add_option(
"--flagav",
action="store_true",
default=False,
help="If a single element in an averaging bin is flagged, "
"flag the averaged bin")
parser.add_option("--caltables",
action="store_true",
default=False,
help="Create calibration tables from gain solutions in "
"the dataset (if present)")
parser.add_option("--quack",
type=int,
default=1,
metavar='N',
help="Discard the first N dumps "
"(which are frequently incomplete)")
parser.add_option("--applycal",
default="",
help="List of calibration solutions to apply to data as "
"a string of comma-separated names, e.g. 'l1' or "
"'K,B,G'. Use 'default' for L1 + L2 and 'all' for "
"all available products.")
(options, args) = parser.parse_args()
# Loading is I/O-bound, so give more threads than CPUs
dask.config.set(
pool=multiprocessing.pool.ThreadPool(4 * multiprocessing.cpu_count()))
if len(args) < 1:
parser.print_help()
raise RuntimeError(
"Please provide one or more MVF dataset names as arguments")
if options.elevation_range and len(options.elevation_range.split(',')) < 2:
raise RuntimeError(
"You have selected elevation flagging. Please provide elevation "
"limits in the form 'lowest_elevation,highest_elevation'.")
if len(args) > 1:
print("Concatenating multiple datasets into single MS.")
def antenna_indices(na, no_auto_corr):
"""Get default antenna1 and antenna2 arrays."""
return np.triu_indices(na, 1 if no_auto_corr else 0)
def corrprod_index(dataset):
"""The correlator product index (with -1 representing missing indices)."""
corrprod_to_index = {
tuple(cp): n
for n, cp in enumerate(dataset.corr_products)
}
# ==========================================
# Generate per-baseline antenna pairs and
# correlator product indices
# ==========================================
def _cp_index(a1, a2, pol):
"""Create correlator product index from antenna pair and pol."""
a1 = a1.name + pol[0].lower()
a2 = a2.name + pol[1].lower()
return corrprod_to_index.get((a1, a2), -1)
# Generate baseline antenna pairs
ant1_index, ant2_index = antenna_indices(len(dataset.ants),
options.no_auto)
# Order as similarly to the input as possible, which gives better performance
# in permute_baselines.
bl_indices = list(zip(ant1_index, ant2_index))
bl_indices.sort(key=lambda ants: _cp_index(dataset.ants[ants[
0]], dataset.ants[ants[1]], pols_to_use[0]))
# Undo the zip
ant1_index[:] = [bl[0] for bl in bl_indices]
ant2_index[:] = [bl[1] for bl in bl_indices]
ant1 = [dataset.ants[a1] for a1 in ant1_index]
ant2 = [dataset.ants[a2] for a2 in ant2_index]
# Create actual correlator product index
cp_index = [
_cp_index(a1, a2, p) for a1, a2 in zip(ant1, ant2)
for p in pols_to_use
]
cp_index = np.array(cp_index, dtype=np.int32)
CPInfo = namedtuple(
"CPInfo", ["ant1_index", "ant2_index", "ant1", "ant2", "cp_index"])
return CPInfo(ant1_index, ant2_index, ant1, ant2, cp_index)
# Open dataset
open_args = args[0] if len(args) == 1 else args
# katdal can handle a list of datasets, which get virtually concatenated internally
dataset = katdal.open(open_args,
ref_ant=options.ref_ant,
applycal=options.applycal)
if dataset.applycal_products:
print('The following calibration products will be applied:',
', '.join(dataset.applycal_products))
else:
print('No calibration products will be applied')
# Get list of unique polarisation products in the dataset
pols_in_dataset = np.unique([(cp[0][-1] + cp[1][-1]).upper()
for cp in dataset.corr_products])
# Which polarisation do we want to write into the MS
# select all possible pols if full-pol selected, otherwise the selected polarisations via pols_to_use
# otherwise finally select any of HH,VV present (the default).
pols_to_use = ['HH', 'HV', 'VH', 'VV'] if (options.full_pol or options.circular) else \
list(np.unique(options.pols_to_use.split(','))) if options.pols_to_use else \
[pol for pol in ['HH', 'VV'] if pol in pols_in_dataset]
# Check we have the chosen polarisations
if np.any([pol not in pols_in_dataset for pol in pols_to_use]):
raise RuntimeError(
f"Selected polarisation(s): {', '.join(pols_to_use)} not available. "
f"Available polarisation(s): {','.join(pols_in_dataset)}")
# Set full_pol if this is selected via options.pols_to_use
if set(pols_to_use) == {'HH', 'HV', 'VH', 'VV'} and not options.circular:
options.full_pol = True
# Extract one MS per spectral window in the dataset(s)
for win in range(len(dataset.spectral_windows)):
dataset.select(reset='T')
centre_freq = dataset.spectral_windows[win].centre_freq
print(
f'Extract MS for spw {win}: centre frequency {int(centre_freq)} Hz'
)
# If no output MS directory name supplied, infer it from dataset(s)
if options.output_ms is None:
if len(dataset.spectral_windows) > 1:
# Use frequency label to disambiguate multiple spectral windows
ms_name = default_ms_name(args, centre_freq)
else:
ms_name = default_ms_name(args)
else:
ms_name = options.output_ms
basename = os.path.splitext(ms_name)[0]
# Discard first N dumps which are frequently incomplete
dataset.select(spw=win,
scans='track',
flags=options.flags,
dumps=slice(options.quack, None))
# The first step is to copy the blank template MS to our desired output
# (making sure it's not already there)
if os.path.exists(ms_name):
raise RuntimeError(
f"MS '{ms_name}' already exists - please remove it "
"before running this script")
print("Will create MS output in " + ms_name)
# Instructions to flag by elevation if requested
if options.elevation_range is not None:
emin, emax = options.elevation_range.split(',')
print(
"\nThe MS can be flagged by elevation in casapy v3.4.0 or higher, with the command:"
)
print(
f" tflagdata(vis='{ms_name}', mode='elevation', lowerlimit={emin}, "
f"upperlimit={emax}, action='apply')\n")
# Instructions to create uvfits file if requested
if options.uvfits:
uv_name = basename + ".uvfits"
print(
"\nThe MS can be converted into a uvfits file in casapy, with the command:"
)
print(
f" exportuvfits(vis='{ms_name}', fitsfile='{uv_name}', datacolumn='data')\n"
)
if options.full_pol:
print(
"\n#### Producing a full polarisation MS (HH,HV,VH,VV) ####\n")
else:
print(
f"\n#### Producing MS with {','.join(pols_to_use)} polarisation(s) ####\n"
)
# if fringe stopping is requested, check that it has not already been done in hardware
if options.stop_w:
print(
"W term in UVW coordinates will be used to stop the fringes.")
try:
autodelay = [
int(ad) for ad in dataset.sensor['DBE/auto-delay']
]
if all(autodelay):
print("Fringe-stopping already performed in hardware... "
"do you really want to stop the fringes here?")
except KeyError:
pass
# Select frequency channel range
if options.channel_range is not None:
channel_range = [
int(chan_str) for chan_str in options.channel_range.split(',')
]
first_chan, last_chan = channel_range[0], channel_range[1]
if (first_chan < 0) or (last_chan >= dataset.shape[1]):
raise RuntimeError(
"Requested channel range outside data set boundaries. "
f"Set channels in the range [0,{dataset.shape[1] - 1}]")
if first_chan > last_chan:
raise RuntimeError(
f"First channel ({first_chan}) bigger than last channel "
f"({last_chan}) - did you mean it the other way around?")
chan_range = slice(first_chan, last_chan + 1)
print(f"\nChannel range {first_chan} through {last_chan}.")
dataset.select(channels=chan_range)
# Are we averaging?
average_data = False
# Determine the number of channels
nchan = len(dataset.channels)
# Work out channel average and frequency increment
if options.chanbin > 1:
average_data = True
# Check how many channels we are dropping
chan_remainder = nchan % options.chanbin
avg_nchan = int(nchan / min(nchan, options.chanbin))
print(
f"Averaging {options.chanbin} channels, output ms will have {avg_nchan} channels."
)
if chan_remainder > 0:
print(
f"The last {chan_remainder} channels in the data will be dropped "
f"during averaging ({options.chanbin} does not divide {nchan})."
)
chan_av = options.chanbin
nchan = avg_nchan
else:
# No averaging in channel
chan_av = 1
# Get the frequency increment per averaged channel
channel_freq_width = dataset.channel_width * chan_av
# Work out dump average and dump increment
# Is the desired time bin greater than the dump period?
if options.dumptime > dataset.dump_period:
average_data = True
dump_av = int(np.round(options.dumptime / dataset.dump_period))
time_av = dump_av * dataset.dump_period
print(
f"Averaging {dataset.dump_period} second dumps to {time_av} seconds."
)
else:
# No averaging in time
dump_av = 1
time_av = dataset.dump_period
# Print a message if extending flags to averaging bins.
if average_data and options.flagav and options.flags != '':
print("Extending flags to averaging bins.")
# Optionally keep only cross-correlation products
if options.no_auto:
dataset.select(corrprods='cross')
print("\nCross-correlations only.")
print(
f"\nUsing {dataset.ref_ant} as the reference antenna. All targets and scans "
"will be based on this antenna.\n")
# MS expects timestamps in MJD seconds
start_time = dataset.start_time.to_mjd() * 24 * 60 * 60
end_time = dataset.end_time.to_mjd() * 24 * 60 * 60
# MVF version 1 and 2 datasets are KAT-7; the rest are MeerKAT
telescope_name = 'KAT-7' if dataset.version[0] in '12' else 'MeerKAT'
# increment scans sequentially in the ms
scan_itr = 1
print("\nIterating through scans in dataset(s)...\n")
cp_info = corrprod_index(dataset)
nbl = cp_info.ant1_index.size
npol = len(pols_to_use)
field_names, field_centers, field_times = [], [], []
obs_modes = ['UNKNOWN']
total_size = 0
# Create the MeasurementSet
table_desc, dminfo = ms_extra.kat_ms_desc_and_dminfo(
nbl=nbl, nchan=nchan, ncorr=npol, model_data=options.model_data)
ms_extra.create_ms(ms_name, table_desc, dminfo)
ms_dict = {}
ms_dict['ANTENNA'] = ms_extra.populate_antenna_dict(
[ant.name for ant in dataset.ants],
[ant.position_ecef
for ant in dataset.ants], [ant.diameter for ant in dataset.ants])
ms_dict['FEED'] = ms_extra.populate_feed_dict(len(dataset.ants),
num_receptors_per_feed=2)
ms_dict['DATA_DESCRIPTION'] = ms_extra.populate_data_description_dict()
ms_dict['POLARIZATION'] = ms_extra.populate_polarization_dict(
ms_pols=pols_to_use, circular=options.circular)
ms_dict['OBSERVATION'] = ms_extra.populate_observation_dict(
start_time, end_time, telescope_name, dataset.observer,
dataset.experiment_id)
# before resetting ms_dict, copy subset to caltable dictionary
if options.caltables:
caltable_dict = {}
caltable_dict['ANTENNA'] = ms_dict['ANTENNA']
caltable_dict['OBSERVATION'] = ms_dict['OBSERVATION']
print("Writing static meta data...")
ms_extra.write_dict(ms_dict, ms_name, verbose=options.verbose)
# Pre-allocate memory buffers
tsize = dump_av
in_chunk_shape = (tsize, ) + dataset.shape[1:]
scan_vis_data = np.empty(in_chunk_shape, dataset.vis.dtype)
scan_weight_data = np.empty(in_chunk_shape, dataset.weights.dtype)
scan_flag_data = np.empty(in_chunk_shape, dataset.flags.dtype)
ms_chunk_shape = (SLOTS, tsize // dump_av, nbl, nchan, npol)
raw_vis_data = ms_async.RawArray(ms_chunk_shape, scan_vis_data.dtype)
raw_weight_data = ms_async.RawArray(ms_chunk_shape,
scan_weight_data.dtype)
raw_flag_data = ms_async.RawArray(ms_chunk_shape, scan_flag_data.dtype)
ms_vis_data = raw_vis_data.asarray()
ms_weight_data = raw_weight_data.asarray()
ms_flag_data = raw_flag_data.asarray()
# Need to limit the queue to prevent overwriting slots before they've
# been processed. The -2 allows for the one we're writing and the one
# the writer process is reading.
work_queue = multiprocessing.Queue(maxsize=SLOTS - 2)
result_queue = multiprocessing.Queue()
writer_process = multiprocessing.Process(
target=ms_async.ms_writer_process,
args=(work_queue, result_queue, options, dataset.ants, cp_info,
ms_name, raw_vis_data, raw_weight_data, raw_flag_data))
writer_process.start()
try:
slot = 0
for scan_ind, scan_state, target in dataset.scans():
s = time.time()
scan_len = dataset.shape[0]
prefix = f'scan {scan_ind:3d} ({scan_len:4d} samples)'
if scan_state != 'track':
if options.verbose:
print(f"{prefix} skipped '{scan_state}' - not a track")
continue
if scan_len < 2:
if options.verbose:
print(f'{prefix} skipped - too short')
continue
if target.body_type != 'radec':
if options.verbose:
print(
f"{prefix} skipped - target '{target.name}' not RADEC"
)
continue
print(
f"{prefix} loaded. Target: '{target.name}'. Writing to disk..."
)
# Get the average dump time for this scan (equal to scan length
# if the dump period is longer than a scan)
dump_time_width = min(time_av, scan_len * dataset.dump_period)
# Get UTC timestamps
utc_seconds = dataset.timestamps[:]
# Update field lists if this is a new target
if target.name not in field_names:
# Since this will be an 'radec' target, we don't need antenna
# or timestamp to get the (astrometric) ra, dec
ra, dec = target.radec()
field_names.append(target.name)
field_centers.append((ra, dec))
field_times.append(
katpoint.Timestamp(utc_seconds[0]).to_mjd() * 60 * 60 *
24)
if options.verbose:
print(
f"Added new field {len(field_names) - 1}: '{target.name}' {ra} {dec}"
)
field_id = field_names.index(target.name)
# Determine the observation tag for this scan
obs_tag = ','.join(tag_to_intent[tag] for tag in target.tags
if tag in tag_to_intent)
# add tag to obs_modes list
if obs_tag and obs_tag not in obs_modes:
obs_modes.append(obs_tag)
# get state_id from obs_modes list if it is in the list, else 0 'UNKNOWN'
state_id = obs_modes.index(
obs_tag) if obs_tag in obs_modes else 0
# Iterate over time in some multiple of dump average
ntime = utc_seconds.size
ntime_av = 0
for ltime in range(0, ntime - tsize + 1, tsize):
utime = ltime + tsize
tdiff = utime - ltime
out_freqs = dataset.channel_freqs
# load all visibility, weight and flag data
# for this scan's timestamps.
# Ordered (ntime, nchan, nbl*npol)
load(dataset, np.s_[ltime:utime, :, :], scan_vis_data,
scan_weight_data, scan_flag_data)
# This are updated as we go to point to the current storage
vis_data = scan_vis_data
weight_data = scan_weight_data
flag_data = scan_flag_data
out_utc = utc_seconds[ltime:utime]
# Overwrite the input visibilities with averaged visibilities,
# flags, weights, timestamps, channel freqs
if average_data:
vis_data, weight_data, flag_data, out_utc, out_freqs = \
averager.average_visibilities(vis_data, weight_data, flag_data,
out_utc, out_freqs, timeav=dump_av,
chanav=chan_av, flagav=options.flagav)
# Infer new time dimension from averaged data
tdiff = vis_data.shape[0]
# Select correlator products and permute axes
cp_index = cp_info.cp_index.reshape((nbl, npol))
vis_data, weight_data, flag_data = permute_baselines(
vis_data, weight_data, flag_data, cp_index,
ms_vis_data[slot], ms_weight_data[slot],
ms_flag_data[slot])
# Increment the number of averaged dumps
ntime_av += tdiff
# Check if writer process has crashed and abort if so
try:
result = result_queue.get_nowait()
raise result
except queue.Empty:
pass
work_queue.put(
ms_async.QueueItem(slot=slot,
target=target,
time_utc=out_utc,
dump_time_width=dump_time_width,
field_id=field_id,
state_id=state_id,
scan_itr=scan_itr))
slot += 1
if slot == SLOTS:
slot = 0
work_queue.put(ms_async.EndOfScan())
result = result_queue.get()
if isinstance(result, Exception):
raise result
scan_size = result.scan_size
s1 = time.time() - s
if average_data and utc_seconds.shape != ntime_av:
print(
f'Averaged {np.shape(utc_seconds)[0]} x {dataset.dump_period} second dumps '
f'to {ntime_av} x {dump_time_width} second dumps')
scan_size_mb = float(scan_size) / (1024**2)
print(f'Wrote scan data ({scan_size_mb:.3f} MiB) '
f'in {s1:.3f} s ({scan_size_mb / s1:.3f} MiBps)\n')
scan_itr += 1
total_size += scan_size
finally:
work_queue.put(None)
writer_exc = None
# Drain the result_queue so that we unblock the writer process
while True:
result = result_queue.get()
if isinstance(result, Exception):
writer_exc = result
elif result is None:
break
writer_process.join()
# This raise is deferred to outside the finally block, so that we don't
# raise an exception while unwinding another one.
if isinstance(writer_exc, Exception):
raise writer_exc
if total_size == 0:
raise RuntimeError("No usable data found in MVF dataset "
"(pick another reference antenna, maybe?)")
# Remove spaces from source names, unless otherwise specified
field_names = [f.replace(' ', '') for f in field_names] \
if not options.keep_spaces else field_names
ms_dict = {}
ms_dict['SPECTRAL_WINDOW'] = ms_extra.populate_spectral_window_dict(
out_freqs, channel_freq_width * np.ones(len(out_freqs)))
ms_dict['FIELD'] = ms_extra.populate_field_dict(
field_centers, field_times, field_names)
ms_dict['STATE'] = ms_extra.populate_state_dict(obs_modes)
ms_dict['SOURCE'] = ms_extra.populate_source_dict(
field_centers, field_times, field_names)
print("\nWriting dynamic fields to disk....\n")
# Finally we write the MS as per our created dicts
ms_extra.write_dict(ms_dict, ms_name, verbose=options.verbose)
if options.tar:
tar = tarfile.open(f'{ms_name}.tar', 'w')
tar.add(ms_name, arcname=os.path.basename(ms_name))
tar.close()
# --------------------------------------
# Now write calibration product tables if required
# Open first HDF5 file in the list to extract TelescopeState parameters from
# (can't extract telstate params from contatenated katdal file as it
# uses the hdf5 file directly)
first_dataset = katdal.open(args[0], ref_ant=options.ref_ant)
main_table = ms_extra.open_table(ms_name, verbose=options.verbose)
if options.caltables:
# copy extra subtable dictionary values necessary for caltable
caltable_dict['SPECTRAL_WINDOW'] = ms_dict['SPECTRAL_WINDOW']
caltable_dict['FIELD'] = ms_dict['FIELD']
solution_types = ['G', 'B', 'K']
ms_soltype_lookup = {
'G': 'G Jones',
'B': 'B Jones',
'K': 'K Jones'
}
print("\nWriting calibration solution tables to disk....")
if 'TelescopeState' not in first_dataset.file.keys():
print(
" No TelescopeState in first dataset. Can't create solution tables.\n"
)
else:
# first get solution antenna ordering
# newer files have the cal antlist as a sensor
if 'cal_antlist' in first_dataset.file['TelescopeState'].keys(
):
a0 = first_dataset.file['TelescopeState/cal_antlist'][()]
antlist = telstate_decode(a0[0][1])
# older files have the cal antlist as an attribute
elif 'cal_antlist' in first_dataset.file[
'TelescopeState'].attrs.keys():
antlist = np.safe_eval(
first_dataset.file['TelescopeState'].
attrs['cal_antlist'])
else:
print(" No calibration antenna ordering in first dataset. "
"Can't create solution tables.\n")
continue
antlist_indices = list(range(len(antlist)))
# for each solution type in the file, create a table
for sol in solution_types:
caltable_name = f'{basename}.{sol}'
sol_name = f'cal_product_{sol}'
if sol_name in first_dataset.file['TelescopeState'].keys():
print(
f' - creating {sol} solution table: {caltable_name}\n'
)
# get solution values from the file
solutions = first_dataset.file['TelescopeState'][
sol_name][()]
soltimes, solvals = [], []
for t, s in solutions:
soltimes.append(t)
solvals.append(telstate_decode(s))
solvals = np.array(solvals)
# convert averaged UTC timestamps to MJD seconds.
sol_mjd = np.array([
katpoint.Timestamp(time_utc).to_mjd() * 24 * 60 *
60 for time_utc in soltimes
])
# determine solution characteristics
if len(solvals.shape) == 4:
ntimes, nchans, npols, nants = solvals.shape
else:
ntimes, npols, nants = solvals.shape
nchans = 1
solvals = solvals.reshape(ntimes, nchans, npols,
nants)
# create calibration solution measurement set
caltable_desc = ms_extra.caltable_desc_float \
if sol == 'K' else ms_extra.caltable_desc_complex
caltable = ms_extra.open_table(caltable_name,
tabledesc=caltable_desc)
# add other keywords for main table
if sol == 'K':
caltable.putkeyword('ParType', 'Float')
else:
caltable.putkeyword('ParType', 'Complex')
caltable.putkeyword('MSName', ms_name)
caltable.putkeyword('VisCal', ms_soltype_lookup[sol])
caltable.putkeyword('PolBasis', 'unknown')
# add necessary units
caltable.putcolkeywords(
'TIME', {
'MEASINFO': {
'Ref': 'UTC',
'type': 'epoch'
},
'QuantumUnits': ['s']
})
caltable.putcolkeywords('INTERVAL',
{'QuantumUnits': ['s']})
# specify that this is a calibration table
caltable.putinfo({
'readme': '',
'subType': ms_soltype_lookup[sol],
'type': 'Calibration'
})
# get the solution data to write to the main table
solutions_to_write = solvals.transpose(
0, 3, 1, 2).reshape(ntimes * nants, nchans, npols)
# MS's store delays in nanoseconds
if sol == 'K':
solutions_to_write = 1e9 * solutions_to_write
times_to_write = np.repeat(sol_mjd, nants)
antennas_to_write = np.tile(antlist_indices, ntimes)
# just mock up the scans -- this doesnt actually correspond to scans in the data
scans_to_write = np.repeat(list(range(len(sol_mjd))),
nants)
# write the main table
main_cal_dict = ms_extra.populate_caltable_main_dict(
times_to_write, solutions_to_write,
antennas_to_write, scans_to_write)
ms_extra.write_rows(caltable,
main_cal_dict,
verbose=options.verbose)
# create and write subtables
subtables = [
'OBSERVATION', 'ANTENNA', 'FIELD',
'SPECTRAL_WINDOW', 'HISTORY'
]
subtable_key = [(os.path.join(caltable.name(), st))
for st in subtables]
# Add subtable keywords and create subtables
# ------------------------------------------------------------------------------
# # this gives an error in casapy:
# *** Error *** MSObservation(const Table &) - table is not a valid MSObservation
# for subtable, subtable_location in zip(subtables, subtable_key)
# ms_extra.open_table(subtable_location, tabledesc=ms_extra.ms_desc[subtable])
# caltable.putkeyword(subtable, 'Table: {0}'.format(subtable_location))
# # write the static info for the table
# ms_extra.write_dict(caltable_dict, caltable.name(), verbose=options.verbose)
# ------------------------------------------------------------------------------
# instead try just copying the main table subtables
# this works to plot the data casapy, but the solutions still can't be
# applied in casapy...
for subtable, subtable_location in zip(
subtables, subtable_key):
main_subtable = ms_extra.open_table(
os.path.join(main_table.name(), subtable))
main_subtable.copy(subtable_location, deep=True)
caltable.putkeyword(subtable,
f'Table: {subtable_location}')
if subtable == 'ANTENNA':
caltable.putkeyword('NAME', antlist)
caltable.putkeyword('STATION', antlist)
if sol != 'B':
spw_table = ms_extra.open_table(
os.path.join(caltable.name(),
'SPECTRAL_WINDOW'))
spw_table.removerows(spw_table.rownumbers())
cen_index = len(out_freqs) // 2
# the delay values in the cal pipeline are calculated relative to frequency 0
ref_freq = 0.0 if sol == 'K' else None
spw_dict = {
'SPECTRAL_WINDOW':
ms_extra.populate_spectral_window_dict(
np.atleast_1d(out_freqs[cen_index]),
np.atleast_1d(channel_freq_width),
ref_freq=ref_freq)
}
ms_extra.write_dict(spw_dict,
caltable.name(),
verbose=options.verbose)
# done with this caltable
caltable.flush()
caltable.close()
main_table.close()
def read_and_plot_data(filename,
output_dir='.',
pdf=True,
Ku=False,
verbose=False,
error_bars=False,
target='off1',
write_nd=False,
rfi_mask='/var/kat/katsdpscripts/RTS/rfi_mask.pickle',
**kwargs):
print('inside', kwargs)
file_base = filename.split('/')[-1].split('.')[0]
nice_filename = file_base + '_T_sys_T_nd'
# Set up logging: logging everything (DEBUG & above), both to console and file
logger = logging.root
logger.setLevel(logging.DEBUG)
fh = logging.FileHandler(nice_filename + '.log', 'w')
fh.setLevel(logging.DEBUG)
fh.setFormatter(logging.Formatter('%(levelname)s: %(message)s'))
logger.addHandler(fh)
logger.info('Beginning data processing with:\n%s' % git_info('standard'))
if Ku:
logger.debug("Using Ku band ... unsetting L band RFI flags")
h5 = katfile.open(filename, centre_freq=12500.5e6, **kwargs)
length = h5.shape[1]
# Don't subtract half a channel width as channel 0 is centred on 0 Hz in baseband
rfi_static_flags = np.tile(True, length)
else:
h5 = katfile.open(filename, **kwargs)
length = h5.shape[1]
pickle_file = open(rfi_mask, mode='rb')
rfi_static_flags = pickle.load(pickle_file)
pickle_file.close()
# Now find the edges of the mask
rfi_freqs_width = (856000000.0 / rfi_static_flags.shape[0])
rfi_freqs_min = 856000000.0 - rfi_freqs_width / 2. # True Start of bin
rfi_freqs_max = rfi_freqs_min * 2 - rfi_freqs_width / 2. # Middle of Max-1 bin
rfi_freqs = np.linspace(rfi_freqs_min, rfi_freqs_max,
rfi_static_flags.shape[0])
rfi_function = get_fit(
np.r_[rfi_freqs, rfi_freqs + rfi_freqs_width * 0.9999],
np.r_[rfi_static_flags, rfi_static_flags])
rfi_static_flags = rfi_function(h5.channel_freqs)
#print ( (rfi_static_flags_new-rfi_static_flags)**2).sum()
if verbose: logger.debug(h5.__str__())
edge = np.tile(True, length)
#edge[slice(211,3896)] = False #Old
edge[slice(int(round(length * 0.0515)),
int(round(0.9512 * length)))] = False ###
static_flags = np.logical_or(edge, rfi_static_flags)
ants = h5.ants
colour = ['b', 'g', 'r', 'c', 'm', 'y', 'k']
pols = ['v', 'h']
for a in ants:
ant = a.name
try:
rx_sn = h5.receivers[ant]
except KeyError:
logger.error(
'Receiver serial number for antennna %s not found in the H5 file'
% ant)
rx_sn = 'l.SN_NOT_FOUND'
band, SN = rx_sn.split('.')
if pdf:
pdf_filename = output_dir + '/' + nice_filename + '.' + rx_sn + '.' + a.name + '.pdf'
pp = PdfPages(pdf_filename)
logger.debug("Created output PDF file: %s" % pdf_filename)
#fig0 = plt.figure(0,figsize=(20,5))
h5.select()
h5.select(ants=a.name, channels=~static_flags)
observer = h5.ants[0].observer
observer.date = time.gmtime(
h5.timestamps.mean())[:6] # katdal resets this date to now()!
fig0 = plot_ts(h5)
Tsys, TAc, Tsys_std = {}, {}, {}
eta_A = {}
Tdiode = {}
nd_temp = {}
for pol in pols:
logger.debug("Processing: %s%s" % (a.name, pol))
Tsys_std[pol] = None
if not (Ku):
diode_filename = '/var/kat/katconfig/user/noise-diode-models/mkat/rx.' + rx_sn + '.' + pol + '.csv'
logger.info('Loading noise diode file %s from config' %
diode_filename)
try:
nd = scape.gaincal.NoiseDiodeModel(diode_filename)
except:
logger.error(
"Error reading the noise diode file ... using a constant value of 20k"
)
logger.error(
"Be sure to reprocess the data once the file is in the config"
)
nd = scape.gaincal.NoiseDiodeModel(freq=[856, 1712],
temp=[20, 20])
#cold data
logger.debug('Using off target %s' % target)
h5.select(ants=a.name,
pol=pol,
channels=~static_flags,
targets=target,
scans='track')
freq = h5.channel_freqs
cold_data = h5.vis[:].real
cold_on, cold_off = get_nd_on_off(h5, log=logger)
#hot data
h5.select(ants=a.name,
pol=pol,
channels=~static_flags,
targets='Moon',
scans='track')
hot_on, hot_off = get_nd_on_off(h5, log=logger)
hot_data = h5.vis[:].real
cold_spec = np.mean(cold_data[cold_off, :, 0], 0)
hot_spec = np.mean(hot_data[hot_off, :, 0], 0)
cold_nd_spec = np.mean(cold_data[cold_on, :, 0], 0)
hot_nd_spec = np.mean(hot_data[hot_on, :, 0], 0)
if not (Ku):
nd_temp[pol] = nd.temperature(freq / 1e6)
# antenna temperature on the moon (from diode calibration)
TAh = hot_spec / (hot_nd_spec - hot_spec) * nd_temp[pol]
# antenna temperature on cold sky (from diode calibration) (Tsys)
TAc[pol] = cold_spec / (cold_nd_spec -
cold_spec) * nd_temp[pol]
print("Mean TAh = %f mean TAc = %f " %
(TAh.mean(), TAc[pol].mean()))
Y = hot_spec / cold_spec
D = 13.5 # Efficiency tables are defined for 13.5
lam = 299792458. / freq
HPBW = 1.18 * (lam / D)
Om = 1.133 * HPBW**2 # main beam solid angle for a gaussian beam
R = 0.5 * Dmoon(observer) # radius of the moon
Os = 2 * np.pi * (1 - np.cos(R)) # disk source solid angle
_f_MHz, _eff_pct = np.loadtxt(
"/var/kat/katconfig/user/aperture-efficiency/mkat/ant_eff_%s_%s_AsBuilt.csv"
% (band.upper(), pol.upper()),
skiprows=2,
delimiter="\t",
unpack=True)
eta_A[pol] = np.interp(freq, _f_MHz,
_eff_pct) / 100. # EMSS aperture efficiency
if Ku: eta_A[pol] = 0.7
Ag = np.pi * (D / 2)**2 # Antenna geometric area
Ae = eta_A[pol] * Ag # Effective aperture
x = 2 * R / HPBW # ratio of source to beam
K = ((x / 1.2)**
2) / (1 - np.exp(-((x / 1.2)**2))
) # correction factor for disk source from Baars 1973
TA_moon = 225 * (Os / Om) * (
1 / K
) # contribution from the moon (disk of constant brightness temp)
gamma = 1.0
Thot = TA_moon
Tcold = 0
Tsys[pol] = gamma * (Thot - Tcold) / (
Y - gamma) # Tsys from y-method ... compare with diode TAc
if error_bars:
cold_spec_std = np.std(cold_data[cold_off, :, 0], 0)
hot_spec_std = np.std(hot_data[hot_off, :, 0], 0)
cold_nd_spec_std = np.std(cold_data[cold_on, :, 0], 0)
hot_nd_spec_std = np.std(hot_data[hot_on, :, 0], 0)
Y_std = Y * np.sqrt((hot_spec_std / hot_spec)**2 +
(cold_spec_std / cold_spec)**2)
Thot_std = 2.25
gamma_std = 0.01
# This is not definded
raise NotImplementedError(
"The factor Thot has not been defined ")
Tsys_std[pol] = Tsys[pol] * np.sqrt((Thot_std / Thot)**2 +
(Y_std / Y)**2 +
(gamma_std / gamma)**2)
else:
Tsys_std[pol] = None
if not (Ku):
Ydiode = hot_nd_spec / hot_spec
Tdiode_h = (Tsys[pol] - Tcold + Thot) * (
Ydiode / gamma - 1
) # Tsys as computed above includes Tcold
Ydiode = cold_nd_spec / cold_spec
Tdiode_c = Tsys[pol] * (Ydiode / gamma - 1)
Tdiode[pol] = (
Tdiode_h + Tdiode_c
) / 2. # Average two equally valid, independent results
if write_nd:
outfilename = save_ND(diode_filename, file_base, freq,
Tdiode[pol])
logger.info('Noise temp data written to file %s' % outfilename)
fig2 = plot_nd(freq, Tdiode, nd_temp, ant=ant, file_base=file_base)
fig1 = plot_Tsys_eta_A(freq,
Tsys,
eta_A,
TAc,
Tsys_std=Tsys_std,
ant=ant,
file_base=file_base,
Ku=Ku)
if pdf:
if not (Ku):
fig2.savefig(pp, format='pdf')
fig1.savefig(pp, format='pdf')
fig0.savefig(pp, format='pdf')
pp.close() # close the pdf file
plt.close("all")
logger.info('Processing complete')
parser.add_argument('--time', type=int, default=10, help='Number of times to read per batch')
parser.add_argument('--channels', type=int, help='Number of channels to read')
parser.add_argument('--dumps', type=int, help='Number of times to read')
parser.add_argument('--joint', action='store_true', help='Load vis, weights, flags together')
parser.add_argument('--applycal', help='Calibration solutions to apply')
parser.add_argument('--workers', type=int, help='Number of dask workers')
args = parser.parse_args()
logging.basicConfig(level='INFO', format='%(asctime)s [%(levelname)s] %(message)s')
if args.workers is not None:
dask.config.set(num_workers=args.workers)
logging.info('Starting')
kwargs = {}
if args.applycal is not None:
kwargs['applycal'] = args.applycal
f = katdal.open(args.filename, **kwargs)
logging.info('File loaded, shape %s', f.shape)
if args.channels:
f.select(channels=np.s_[:args.channels])
if args.dumps:
f.select(dumps=np.s_[:args.dumps])
# Trigger creation of the dask graphs, population of sensor cache for applycal etc
_ = (f.vis[0, 0, 0], f.weights[0, 0, 0], f.flags[0, 0, 0])
logging.info('Selection complete')
start = time.time()
last_time = start
for st in range(0, f.shape[0], args.time):
et = st + args.time
if args.joint:
vis, weights, flags = DaskLazyIndexer.get([f.vis, f.weights, f.flags], np.s_[st:et])
else:
parser.add_option('-o', '--out',
action='store',
dest='outfile',
type=str,
default=None,
help='Full path name of output PDF report file.')
(opts, args) = parser.parse_args()
if len(args) < 1: raise SystemExit(parser.print_usage())
datafile = args[0]
if opts.outfile is None: opts.outfile = os.path.splitext(os.path.basename(datafile))[0]
else: opts.outfile = os.path.splitext(os.path.basename(opts.outfile))[0]
# Read data file
try:
h5 = katdal.open(datafile, quicklook=True)
except Exception as err_msg: raise SystemExit('An error as occured:\n%s' % err_msg)
# Read progress output
if opts.aux is None: raise RuntimeError('Please provide progress.out file for gain settings')
progress = opts.aux
f = open(progress)
myfile = f.readlines()
f.close()
# Correlate gain change timestamps from progress to spectra timestamps from observation file
timestamps = []
req_gains = []
gain_idx = []
for line in myfile:
if line.strip().find("Set digital gain on selected DBE") > 0:
import katdal
import numpy as np
import matplotlib.pylab as plt
import katcal
from katcal import calprocs
# ------------------------------------------------------------------------
# open file
file_name = "/data1/NASSP/reduction_script/1355292163.h5"
f = katdal.open(file_name)
# select polarisation
# solver can only work on one polarisation at a time in this framework
f.select(pol="h")
# get info that is needed for the solver
antlist = [a.name for a in f.ants]
corr_products = f.corr_products
antlist1, antlist2 = calprocs.get_solver_antlists(antlist, corr_products)
# ------------------------------------------------------------------------
# solve for gains
glist = []
tlist = []
for scan_ind, scan_state, target in f.scans():
if "track" in scan_state and "gaincal" in target.tags:
vis = f.vis[:]
times = f.timestamps[:]
elif arg.endswith('.h5'):
files.append(arg)
else:
for rootdir, subdirs, dirfiles in os.walk(arg):
files.extend([
os.path.join(rootdir, name) for name in dirfiles
if name.endswith('.h5')
])
# Open each file in turn and print a one-line summary
print(
"Name Ver Observer StartTimeSAST Shape SizeGB "
"DumpHz SPW CFreqMHz Ants Tgts Scans Description")
for f in files:
try:
d = katdal.open(f, quicklook=True)
except Exception, e:
print '%s %s - %s' % (f, e.__class__.__name__, e)
continue
name = os.path.basename(f)
name = (name[:10] + '...') if len(name) > 13 else name
all_ants = ('ant1', 'ant2', 'ant3', 'ant4', 'ant5', 'ant6', 'ant7')
file_ants = [ant.name for ant in d.ants]
ants = ''.join([(ant[3:] if ant in file_ants else '-')
for ant in all_ants])
print '%13s %3s %10s %19s (%6d,%5d,%4d) %6.2f %6.3f %3d %8.3f %s %4d %5d %s' % \
(name, d.version, d.observer.strip()[:10].ljust(10), d.start_time.local()[:19],
d.shape[0], d.shape[1], d.shape[2], d.size / 1024. / 1024. / 1024., 1.0 / d.dump_period,
len(d.spectral_windows), d.spectral_windows[d.spw].centre_freq / 1e6, ants,
len(d.catalogue), len(d.scan_indices), d.description)
del d
help=
"This is the frequency range of the channels to use in the reduction. this is passed as a comma delimatated pair of integer values', default = '%default'"
)
(opts, args) = parser.parse_args()
if len(args) < 1:
raise RuntimeError('Please specify the data file to reduce')
height = 1.0
bins = opts.bins
if opts.ku_band is None:
pickle_fn = open(opts.mask, mode='rb')
rfi_static_flags = pickle.load(pickle_fn)
pickle_fn.close()
h5 = katdal.open(args[0])
else:
centre_freq = 12500.5e6
h5 = katdal.open(args[0], centre_freq=np.float(centre_freq))
rfi_static_flags = rfi.detect_spikes_median(
h5.vis[:], spike_width=3, outlier_sigma=5.0).max(axis=2).max(axis=0)
freqst, freqend = opts.freq.split(',')
rfi_static_flags[0:np.int(freqst)] = True
rfi_static_flags[np.int(freqend):-1] = True
#1392246099.h5
if opts.ant == '':
ant_list = [ant.name for ant in h5.ants]
else:
ant_list = [opts.ant]
type=float,
default=1575e6,
help='Frequency of expected target in Hz, default = \'%default\' Hz.')
parser.add_option('--out',
action='store',
dest='outfile',
type=str,
default=None,
help='Name of output report file.')
(opts, args) = parser.parse_args()
if opts.filename is None: raise SystemExit(parser.print_usage())
try:
h5 = katdal.open(opts.filename, quicklook=True)
except Exception as err_msg:
raise SystemExit('An error as occured:\n%s' % err_msg)
outfile = opts.outfile
if outfile is None:
outfile = os.path.splitext(os.path.basename(opts.filename))[0]
ants = [opts.ant]
pols = [opts.pol]
if opts.ant == 'all':
ants = [ant.name for ant in h5.ants]
outants = 'all_antennas'
else:
outants = opts.ant
if opts.pol == 'all':
def MKContPipeline(files, outputdir, **kwargs):
"""MeerKAT Continuum pipeline.
Parameters
----------
files : list
h5 filenames (note: support for multiple h5 files
i.e. ConcatenatedDataSet is not currently supported)
outputdir : string
Directory location to write output data,
scratchdir : string, optional
The directory location of the aips disk
parmFile : string, optional
Overwrite the default imaging parameters using this parameter file.
"""
#if len(files) > 1:
# raise TooManyKatfilesException('Processing multiple katfiles are not currently supported')
# Onle be concatenated if we have to be
if len(files) == 1:
h5file = files[0]
else:
h5file = files
# Die gracefully if we cannot write to the output area...
if not os.path.exists(outputdir):
print('Specified output directory: ' + outputdir + 'does not exist.')
exit(-1)
# Obit error logging
err = OErr.OErr()
#################### Initialize filenames #######################################################
fileRoot = os.path.join(outputdir,
os.path.basename(os.path.splitext(
files[0])[0])) # root of file name
logFile = fileRoot + ".log" # Processing log file
avgClass = ("UVAv")[0:6] # Averaged data AIPS class
manifestfile = outputdir + '/manifest.pickle'
############################# Initialize OBIT and AIPS ##########################################
noScrat = []
# Logging directly to logFile
OErr.PInit(err, 2, logFile)
EVLAAddOutFile(os.path.basename(logFile), 'project', 'Pipeline log file')
if kwargs.get('reuse'):
ObitSys = AIPSSetup.AIPSSetup(err,
configfile=kwargs.get('configFile'),
scratchdir=kwargs.get('scratchdir'),
aipsdisk=kwargs.get('aipsdisk'),
overwrite=False)
else:
ObitSys = AIPSSetup.AIPSSetup(err,
configfile=kwargs.get('configFile'),
scratchdir=kwargs.get('scratchdir'),
aipsdisk=kwargs.get('aipsdisk'))
# Get the set up AIPS environment.
AIPS_ROOT = os.environ['AIPS_ROOT']
AIPS_VERSION = os.environ['AIPS_VERSION']
nThreads = 72
user = OSystem.PGetAIPSuser()
AIPS.userno = user
disk = 1
fitsdisk = 0
nam = os.path.basename(os.path.splitext(files[0])[0])[0:10]
cls = "Raw"
seq = 1
############################# Initialise Parameters ##########################################
####### Initialize parameters dictionary #####
parms = KATInitContParms()
####### User defined parameters ######
if kwargs.get('parmFile'):
print("parmFile", kwargs.get('parmFile'))
exec(open(kwargs.get('parmFile')).read())
EVLAAddOutFile(os.path.basename(kwargs.get('parmFile')), 'project',
'Pipeline input parameters')
############### Initialize katfile object, uvfits object and condition data #########################
OK = False
# Open the h5 file as a katfile object
try:
#open katfile and perform selection according to kwargs
katdata = katfile.open(h5file)
OK = True
except Exception as exception:
print(exception)
if not OK:
OErr.PSet(err)
OErr.PLog(err, OErr.Fatal,
"Unable to read KAT HDF5 data in " + str(h5file))
raise KATUnimageableError("Unable to read KAT HDF5 data in " +
str(h5file))
#Are we MeerKAT or KAT-7
telescope = katdata.ants[0].name[0]
if telescope == 'm':
sefd = 500.
else:
sefd = 1200.
#Get calibrator models
fluxcals = katpoint.Catalogue(
open(FITSDir.FITSdisks[0] + "/" + parms["fluxModel"]))
#Condition data (get bpcals, update names for aips conventions etc)
KATh5Condition(katdata, fluxcals, err)
###################### Data selection and static edits ############################################
# Select data based on static imageable parameters
MKATh5Select(katdata, parms, err, **kwargs)
# General AIPS data parameters at script level
dataClass = ("UVDa")[0:6] # AIPS class of raw uv data
band = katdata.spectral_windows[0].product #Correlator product
project = os.path.basename(os.path.splitext(files[0])[0])[
0:10] # Project name (12 char or less, used as AIPS Name)
outIClass = parms["outIClass"] # image AIPS class
debug = parms["debug"]
check = parms["check"]
####################### Import data into AIPS #####################################################
# Reuse or nay?
if kwargs.get('reuse'):
uv = UV.newPAUV("AIPS UV DATA", EVLAAIPSName(project), dataClass, disk,
seq, True, err)
obsdata = KATH5toAIPS.GetKATMeta(katdata, err)
# Extract AIPS parameters of the uv data to the metadata
obsdata["Aproject"] = uv.Aname
obsdata["Aclass"] = uv.Aclass
obsdata["Aseq"] = uv.Aseq
obsdata["Adisk"] = disk
obsdata["calInt"] = katdata.dump_period
obsdata["fitsdisk"] = fitsdisk
# TODO: Check if the input data has been Hanned.
doneHann = True
else:
# Number of baselines gives batch size
nbl = len(
np.unique([(cp[0][:-1] + cp[1][:-1]).upper()
for cp in katdata.corr_products]))
# Construct a template uvfits file from master template
mastertemplate = ObitTalkUtil.FITSDir.FITSdisks[
fitsdisk] + 'MKATTemplate.uvtab.gz'
outtemplate = nam + '.uvtemp'
KATH5toAIPS.MakeTemplate(mastertemplate,
outtemplate,
len(katdata.channel_freqs),
nvispio=nbl)
uv = OTObit.uvlod(outtemplate, 0, EVLAAIPSName(project), cls, disk,
seq, err)
obsdata = KATH5toAIPS.KAT2AIPS(katdata,
uv,
disk,
fitsdisk,
err,
calInt=katdata.dump_period,
**kwargs)
MakeIFs.UVMakeIF(uv, 8, err)
os.remove(outtemplate)
# Print the uv data header to screen.
uv.Header(err)
############################# Set Project Processing parameters ###################################
# Parameters derived from obsdata and katdata
MKATGetObsParms(obsdata, katdata, parms, logFile)
###### Initialise target parameters #####
KATInitTargParms(katdata, parms, err)
# Load the outputs pickle jar
EVLAFetchOutFiles()
OSystem.PAllowThreads(nThreads) # Allow threads in Obit/oython
retCode = 0
maxgap = max(parms["CalAvgTime"], 20 * katdata.dump_period) / 60.
################### Start processing ###############################################################
mess = "Start project "+parms["project"]+" AIPS user no. "+str(AIPS.userno)+\
", KAT7 configuration "+parms["KAT7Cfg"]
printMess(mess, logFile)
if debug:
pydoc.ttypager = pydoc.plainpager # don't page task input displays
mess = "Using Debug mode "
printMess(mess, logFile)
if check:
mess = "Only checking script"
printMess(mess, logFile)
# Log parameters
printMess("Parameter settings", logFile)
for p in parms:
mess = " " + p + ": " + str(parms[p])
printMess(mess, logFile)
clist = []
for DCal in parms["DCals"]:
if DCal["Source"] not in clist:
clist.append(DCal["Source"])
for PCal in parms["PCals"]:
if PCal["Source"] not in clist:
clist.append(PCal["Source"])
for ACal in parms["ACals"]:
if ACal["Source"] not in clist:
clist.append(ACal["Source"])
if kwargs.get('targets') is not None:
targets = [
targ.name for targ in katdata.catalogue
if (targ.name not in clist) and (
targ.name in kwargs.get('targets').split(','))
]
else:
targets = [
targ.name for targ in katdata.catalogue if (targ.name not in clist)
]
refAnt = FetchObject(fileRoot + ".refAnt.pickle")
# Save parameters to pickle jar, manifest
ParmsPicklefile = fileRoot + ".Parms.pickle" # Where results saved
SaveObject(parms, ParmsPicklefile, True)
EVLAAddOutFile(os.path.basename(ParmsPicklefile), 'project',
'Processing parameters used')
loadClass = dataClass
# Hanning - No Hanning
parms["doHann"] = False
doneHann = False
if doneHann:
# Halve channels after hanning.
parms["selChan"] = int(parms["selChan"] / 2)
parms["BChDrop"] = int(parms["BChDrop"] / 2)
parms["EChDrop"] = int(parms["EChDrop"] / 2)
if uv == None and not check:
raise RuntimeError("Cannot Hann data ")
# Clear any old calibration/editing
if parms["doClearTab"] or kwargs.get('reuse'):
mess = "Clear previous calibration"
printMess(mess, logFile)
EVLAClearCal(uv,
err,
doGain=parms["doClearGain"],
doFlag=parms["doClearFlag"],
doBP=parms["doClearBP"],
check=check)
OErr.printErrMsg(err, "Error resetting calibration")
# Quack to remove data from start and end of each scan
if parms["doQuack"]:
retCode = EVLAQuack (uv, err, begDrop=parms["quackBegDrop"], endDrop=parms["quackEndDrop"], \
Reason=parms["quackReason"], \
logfile=logFile, check=check, debug=debug)
if retCode != 0:
raise RuntimeError("Error Quacking data")
# Flag antennas shadowed by others?
if parms["doShad"]:
retCode = EVLAShadow (uv, err, shadBl=parms["shadBl"], \
logfile=logFile, check=check, debug=debug)
if retCode != 0:
raise RuntimeError("Error Shadow flagging data")
# Median window time editing, for RFI impulsive in time
if parms["doMednTD1"]:
mess = "Median window time editing, for RFI impulsive in time:"
printMess(mess, logFile)
retCode = EVLAMedianFlag (uv, clist, err, noScrat=noScrat, nThreads=nThreads, \
avgTime=parms["mednAvgTime"], avgFreq=parms["mednAvgFreq"], chAvg= parms["mednChAvg"], \
timeWind=parms["mednTimeWind"],flagVer=2, flagTab=2,flagSig=parms["mednSigma"], \
logfile=logFile, check=check, debug=False)
if retCode != 0:
raise RuntimeError("Error in MednFlag")
# Median window frequency editing, for RFI impulsive in frequency
if parms["doFD1"]:
mess = "Median window frequency editing, for RFI impulsive in frequency:"
printMess(mess, logFile)
retCode = EVLAAutoFlag (uv, clist, err, flagVer=2, flagTab=2, doCalib=-1, doBand=-1, \
timeAvg=parms["FD1TimeAvg"], \
doFD=True, FDmaxAmp=1.0e20, FDmaxV=1.0e20, FDwidMW=parms["FD1widMW"], \
FDmaxRMS=[1.0e20,0.1], FDmaxRes=parms["FD1maxRes"], \
FDmaxResBL= parms["FD1maxRes"], FDbaseSel=parms["FD1baseSel"],\
nThreads=nThreads, logfile=logFile, check=check, debug=debug)
if retCode != 0:
raise RuntimeError("Error in AutoFlag")
# Parallactic angle correction?
if parms["doPACor"]:
retCode = EVLAPACor(uv, err, \
logfile=logFile, check=check, debug=debug)
if retCode != 0:
raise RuntimeError("Error in Parallactic angle correction")
# Need to find a reference antenna? See if we have saved it?
if (parms["refAnt"] <= 0):
refAnt = FetchObject(fileRoot + ".refAnt.pickle")
if refAnt:
parms["refAnt"] = refAnt
# Use bandpass calibrator and center half of each spectrum
if parms["refAnt"] <= 0:
mess = "Find best reference antenna: run Calib on BP Cal(s) "
printMess(mess, logFile)
parms["refAnt"] = EVLAGetRefAnt(uv, parms["BPCals"], err, flagVer=0, \
solInt=parms["bpsolint1"], nThreads=nThreads, \
logfile=logFile, check=check, debug=debug)
if err.isErr:
raise RuntimeError("Error finding reference antenna")
if parms["refAnts"][0] <= 0:
parms["refAnts"][0] = parms["refAnt"]
mess = "Picked reference antenna " + str(parms["refAnt"])
printMess(mess, logFile)
# Save it
ParmsPicklefile = fileRoot + ".Parms.pickle" # Where results saved
SaveObject(parms, ParmsPicklefile, True)
refAntPicklefile = fileRoot + ".refAnt.pickle" # Where results saved
SaveObject(parms["refAnt"], refAntPicklefile, True)
# Plot Raw, edited data?
parms["doRawSpecPlot"] = False
parms["doSpecPlot"] = False
if parms["doRawSpecPlot"] and parms["plotSource"]:
mess = "Raw Spectral plot for: " + ' '.join(parms["BPCal"])
printMess(mess, logFile)
plotFile = fileRoot + "_RawSpec.ps"
retCode = EVLASpectrum(uv, parms["BPCal"], parms["plotTime"], maxgap, plotFile, parms["refAnt"], err, \
Stokes=["RR","LL"], doband=-1, \
check=check, debug=debug, logfile=logFile )
if retCode != 0:
raise RuntimeError("Error in Plotting spectrum")
EVLAAddOutFile(plotFile, 'project', 'Pipeline log file')
# delay calibration
if parms["doDelayCal"] and parms["DCals"] and not check:
plotFile = fileRoot + "_DelayCal.ps"
retCode = EVLADelayCal(uv, parms["DCals"], err, \
BChan=parms["delayBChan"], EChan=parms["delayEChan"], \
doCalib=2, flagVer=0, doBand=-1, \
solInt=parms["delaySolInt"], smoTime=parms["delaySmoo"], \
refAnts=[parms["refAnt"]], doTwo=parms["doTwo"],
doZeroPhs=parms["delayZeroPhs"], \
doPlot=parms["doSNPlot"], plotFile=plotFile, \
nThreads=nThreads, noScrat=noScrat, \
logfile=logFile, check=check, debug=debug)
if retCode != 0:
raise RuntimeError("Error in delay calibration")
# Plot corrected data?
if parms["doSpecPlot"] and parms["plotSource"]:
plotFile = fileRoot + "_DelaySpec.ps"
retCode = EVLASpectrum(uv, parms["BPCal"], parms["plotTime"], maxgap, \
plotFile, parms["refAnt"], err, \
Stokes=["RR","LL"], doband=-1, \
check=check, debug=debug, logfile=logFile )
if retCode != 0:
raise RuntimeError("Error in Plotting spectrum")
print(parms["bpBChan1"], parms["bpEChan1"], parms["bpBChan2"],
parms["bpEChan2"], parms["bpChWid2"])
# Bandpass calibration
if parms["doBPCal"] and parms["BPCals"]:
retCode = KATBPCal(uv, parms["BPCals"], err, noScrat=noScrat, solInt1=parms["bpsolint1"], \
solInt2=parms["bpsolint2"], solMode=parms["bpsolMode"], \
BChan1=parms["bpBChan1"], EChan1=parms["bpEChan1"], \
BChan2=parms["bpBChan2"], EChan2=parms["bpEChan2"], ChWid2=parms["bpChWid2"], \
doCenter1=parms["bpDoCenter1"], refAnt=parms["refAnt"], \
UVRange=parms["bpUVRange"], doCalib=2, gainUse=0, flagVer=0, doPlot=False, \
nThreads=nThreads, logfile=logFile, check=check, debug=debug)
if retCode != 0:
raise RuntimeError("Error in Bandpass calibration")
# Plot corrected data?
if parms["doSpecPlot"] and parms["plotSource"]:
plotFile = fileRoot + "_BPSpec.ps"
retCode = EVLASpectrum(uv, parms["BPCal"], parms["plotTime"], maxgap, plotFile, \
parms["refAnt"], err, Stokes=["RR","LL"], doband=2, \
check=check, debug=debug, logfile=logFile )
if retCode != 0:
raise RuntimeError("Error in Plotting spectrum")
# Amp & phase Calibrate
if parms["doAmpPhaseCal"]:
plotFile = fileRoot + "_APCal.ps"
retCode = KATCalAP (uv, [], parms["ACals"], err, PCals=parms["PCals"],
doCalib=2, doBand=2, BPVer=1, flagVer=0, \
BChan=parms["ampBChan"], EChan=parms["ampEChan"], \
solInt=parms["solInt"], solSmo=parms["solSmo"], ampScalar=parms["ampScalar"], \
doAmpEdit=parms["doAmpEdit"], ampSigma=parms["ampSigma"], \
ampEditFG=parms["ampEditFG"], \
doPlot=parms["doSNPlot"], plotFile=plotFile, refAnt=parms["refAnt"], \
nThreads=nThreads, noScrat=noScrat, logfile=logFile, check=check, debug=debug)
#print parms["ACals"],parms["PCals"]
if retCode != 0:
raise RuntimeError("Error calibrating")
# More editing
if parms["doAutoFlag"]:
mess = "Post calibration editing:"
printMess(mess, logFile)
# if going to redo then only calibrators
if parms["doRecal"]:
# Only calibrators
clist = []
for DCal in parms["DCals"]:
if DCal["Source"] not in clist:
clist.append(DCal["Source"])
for PCal in parms["PCals"]:
if PCal["Source"] not in clist:
clist.append(PCal["Source"])
for ACal in parms["ACals"]:
if ACal["Source"] not in clist:
clist.append(ACal["Source"])
else:
clist = []
retCode = EVLAAutoFlag (uv, clist, err, flagVer=0, flagTab =2, \
doCalib=2, gainUse=0, doBand=2, BPVer=1, \
IClip=parms["IClip"], minAmp=parms["minAmp"], timeAvg=parms["timeAvg"], \
doFD=parms["doFirstAFFD"], FDmaxAmp=parms["FDmaxAmp"], FDmaxV=parms["FDmaxV"], \
FDwidMW=parms["FDwidMW"], FDmaxRMS=parms["FDmaxRMS"], \
FDmaxRes=parms["FDmaxRes"], FDmaxResBL=parms["FDmaxResBL"], \
FDbaseSel=parms["FDbaseSel"], \
nThreads=nThreads, logfile=logFile, check=check, debug=debug)
if retCode != 0:
raise RuntimeError("Error in AutoFlag")
# Redo the calibration using new flagging?
if parms["doBPCal2"] == None:
parms["doBPCal2"] = parms["doBPCal"]
if parms["doDelayCal2"] == None:
parms["doDelayCal2"] = parms["doDelayCal2"]
if parms["doAmpPhaseCal2"] == None:
parms["doAmpPhaseCal2"] = parms["doAmpPhaseCal"]
if parms["doAutoFlag2"] == None:
parms["doAutoFlagCal2"] = parms["doAutoFlag"]
if parms["doRecal"]:
mess = "Redo calibration:"
printMess(mess, logFile)
EVLAClearCal(uv,
err,
doGain=True,
doFlag=False,
doBP=True,
check=check,
logfile=logFile)
OErr.printErrMsg(err, "Error resetting calibration")
# Parallactic angle correction?
if parms["doPACor"]:
retCode = EVLAPACor(uv, err, \
logfile=logFile, check=check, debug=debug)
if retCode != 0:
raise RuntimeError("Error in Parallactic angle correction")
# Delay recalibration
if parms["doDelayCal2"] and parms["DCals"] and not check:
plotFile = fileRoot + "_DelayCal2.ps"
retCode = EVLADelayCal(uv, parms["DCals"], err, \
BChan=parms["delayBChan"], EChan=parms["delayEChan"], \
doCalib=2, flagVer=0, doBand=-1, \
solInt=parms["delaySolInt"], smoTime=parms["delaySmoo"], \
refAnts=[parms["refAnt"]], doTwo=parms["doTwo"], \
doZeroPhs=parms["delayZeroPhs"], \
doPlot=parms["doSNPlot"], plotFile=plotFile, \
nThreads=nThreads, noScrat=noScrat, \
logfile=logFile, check=check, debug=debug)
if retCode != 0:
raise RuntimeError("Error in delay calibration")
# Plot corrected data?
if parms["doSpecPlot"] and parms["plotSource"]:
plotFile = fileRoot + "_DelaySpec2.ps"
retCode = EVLASpectrum(uv, parms["BPCal"], parms["plotTime"], maxgap, plotFile, parms["refAnt"], err, \
Stokes=["RR","LL"], doband=-1, \
check=check, debug=debug, logfile=logFile )
if retCode != 0:
raise RuntimeError("Error in Plotting spectrum")
# Bandpass calibration
if parms["doBPCal2"] and parms["BPCals"]:
retCode = KATBPCal(uv, parms["BPCals"], err, noScrat=noScrat, solInt1=parms["bpsolint1"], \
solInt2=parms["bpsolint2"], solMode=parms["bpsolMode"], \
BChan1=parms["bpBChan1"], EChan1=parms["bpEChan1"], \
BChan2=parms["bpBChan2"], EChan2=parms["bpEChan2"], ChWid2=parms["bpChWid2"], \
doCenter1=parms["bpDoCenter1"], refAnt=parms["refAnt"], \
UVRange=parms["bpUVRange"], doCalib=2, gainUse=0, flagVer=0, doPlot=False, \
nThreads=nThreads, logfile=logFile, check=check, debug=debug)
if retCode != 0:
raise RuntimeError("Error in Bandpass calibration")
# Plot corrected data?
if parms["doSpecPlot"] and parms["plotSource"]:
plotFile = fileRoot + "_BPSpec2.ps"
retCode = EVLASpectrum(uv, parms["BPCal"], parms["plotTime"], maxgap, plotFile, parms["refAnt"], err, \
Stokes=["RR","LL"], doband=2, \
check=check, debug=debug, logfile=logFile )
if retCode != 0:
raise RuntimeError("Error in Plotting spectrum")
# Amp & phase Recalibrate
if parms["doAmpPhaseCal2"]:
plotFile = fileRoot + "_APCal2.ps"
retCode = KATCalAP (uv, [], parms["ACals"], err, PCals=parms["PCals"], \
doCalib=2, doBand=2, BPVer=1, flagVer=0, \
BChan=parms["ampBChan"], EChan=parms["ampEChan"], \
solInt=parms["solInt"], solSmo=parms["solSmo"], ampScalar=parms["ampScalar"], \
doAmpEdit=True, ampSigma=parms["ampSigma"], \
ampEditFG=parms["ampEditFG"], \
doPlot=parms["doSNPlot"], plotFile=plotFile, refAnt=parms["refAnt"], \
noScrat=noScrat, nThreads=nThreads, logfile=logFile, check=check, debug=debug)
if retCode != 0:
raise RuntimeError("Error calibrating")
# More editing
parms["doAutoFlag2"] = False
if parms["doAutoFlag2"]:
mess = "Post recalibration editing:"
printMess(mess, logFile)
retCode = EVLAAutoFlag (uv, [], err, flagVer=0, flagTab=2, \
doCalib=2, gainUse=0, doBand=2, BPVer=1, \
IClip=parms["IClip"], minAmp=parms["minAmp"], timeAvg=parms["timeAvg"], \
doFD=parms["doSecAFFD"], FDmaxAmp=parms["FDmaxAmp"], FDmaxV=parms["FDmaxV"], \
FDwidMW=parms["FDwidMW"], FDmaxRMS=parms["FDmaxRMS"], \
FDmaxRes=parms["FDmaxRes"], FDmaxResBL= parms["FDmaxResBL"], \
FDbaseSel=parms["FDbaseSel"], \
nThreads=nThreads, logfile=logFile, check=check, debug=debug)
if retCode != 0:
raise RuntimeError("Error in AutoFlag")
# end recal
# Calibrate and average data
# Overwrite avgStokes from command line
parms["avgFreq"] = 0
parms["chAvg"] = 1
parms["doCalAvg"] = 'Splat'
if kwargs.get('halfstokes'):
parms["avgStokes"] = 'HALF'
if parms["doCalAvg"] == 'Splat':
retCode = KATCalAvg (uv, avgClass, parms["seq"], parms["CalAvgTime"], err, \
flagVer=2, doCalib=2, gainUse=0, doBand=2, BPVer=1, doPol=False, \
avgFreq=parms["avgFreq"], chAvg=parms["chAvg"], Stokes=parms["avgStokes"], \
BChan=1, EChan=parms["selChan"] - 1, doAuto=parms["doAuto"], \
BIF=parms["CABIF"], EIF=parms["CAEIF"], Compress=parms["Compress"], \
nThreads=nThreads, logfile=logFile, check=check, debug=debug)
if retCode != 0:
raise RuntimeError("Error in CalAvg")
elif parms["doCalAvg"] == 'BL':
retCode = KATBLCalAvg (uv, avgClass, parms["seq"], err, \
flagVer=2, doCalib=2, gainUse=0, doBand=2, BPVer=1, doPol=False, \
avgFreq=parms["avgFreq"], chAvg=parms["chAvg"], FOV=parms['FOV'], \
maxInt=min(parms["solPInt"],parms["solAInt"]), Stokes=parms["avgStokes"], \
BChan=1, EChan=parms["selChan"] - 1, timeAvg=parms["CalAvgTime"], \
BIF=parms["CABIF"], EIF=parms["CAEIF"], Compress=parms["Compress"], \
logfile=logFile, check=check, debug=debug)
if retCode != 0:
raise RuntimeError("Error in BLCalAvg")
if parms["doSaveTab"]:
filename = project + ".CalTab.uvtab"
_ = EVLAUVFITSTab(uv, filename, 0, err, logfile=logFile)
#Zap unaveraged data if requested
if kwargs.get('zapraw'):
uv.Zap(err)
# Get calibrated/averaged data
if not check:
uv = UV.newPAUV("AIPS UV DATA", EVLAAIPSName(project), avgClass[0:6], \
disk, parms["seq"], True, err)
if err.isErr:
OErr.printErrMsg(err, "Error creating cal/avg AIPS data")
plotFile = fileRoot + "_Spec.ps"
retCode = EVLASpectrum(uv, parms["BPCal"], parms["plotTime"], maxgap, \
plotFile, parms["refAnt"], err, \
Stokes=["I"], doband=-1, docalib=-1, \
check=check, debug=debug, logfile=logFile )
if retCode != 0:
raise RuntimeError("Error in Plotting spectrum")
# KATUVFITS(uv, 'preimage.uvfits', 0, err, exclude=["AIPS HI", "AIPS SL", "AIPS PL"],
# include=["AIPS AN", "AIPS FQ"], compress=parms["Compress"], logfile=logFile)
KATUVFITab(uv, project + '.uvtab', 0, err)
#Gzip the data?
if kwargs.get('gzip'):
os.system('pigz -p %d %s' % (nThreads, project + '.uvtab'))
os.system('rm -f %s' % (project + '.uvtab'))
res = get_reduction_metadata(
filename.split('/')[-1],
reduction_name='AR1 Generate RFI Flags')
flag_results = res[0]['CAS.ReferenceDatastore']
for fl in flag_results:
if fl.endswith('flags.h5'):
flag_filename = fl.split(':')[-1]
print flag_filename
h5_flags = h5py.File(flag_filename, 'r')
cal_flags = True
except:
print "no calibration pipeline flags found"
cal_flags = False
#Loading data
data = katdal.open(filename)
if cal_flags:
data._flags = h5_flags['flags']
#Nice name for pdf
h5name = data.name.split('/')[-1]
obs_details = h5name + '_AR1_observation_report'
pp = PdfPages(obs_details + '.pdf')
N = data.shape[0]
ext = 930
step = max(int(math.floor(N / ext)), 1)
#Loading flags
data.select()
M = 4 * np.shape(data)[1] / 4096
if len(args) > 1:
print "Concatenating multiple h5 files into single MS."
if not ms_extra.casacore_binding:
raise RuntimeError(
"Failed to find casacore binding. You need to install both "
"casacore and pyrap, or run the script from within a modified "
"casapy containing h5py and katpoint.")
else:
print "Using '%s' casacore binding to produce MS" % (
ms_extra.casacore_binding, )
# Open HDF5 file
# if len(args) == 1: args = args[0]
# katdal can handle a list of files, which get virtually concatenated internally
h5 = katdal.open(args, ref_ant=options.ref_ant)
#Get list of unique polarisation products in the file
pols_in_file = np.unique([(cp[0][-1] + cp[1][-1]).upper()
for cp in h5.corr_products])
#Which polarisation do we want to write into the MS
#select all possible pols if full-pol selected, otherwise the selected polarisations via pols_to_use
#otherwise finally select any of HH,VV present (the default).
pols_to_use = ['HH', 'HV', 'VH', 'VV'] if (options.full_pol or options.circular) else \
list(np.unique(options.pols_to_use.split(','))) if options.pols_to_use else \
[pol for pol in ['HH','VV'] if pol in pols_in_file]
#Check we have the chosen polarisations
if np.any([pol not in pols_in_file for pol in pols_to_use]):
raise RuntimeError("Selected polarisation(s): %s not available. "
default=10.,
help=
"Keep jumps that are bigger than margin by this factor (default %default)")
parser.add_option(
"-c",
"--channel-mask",
default='/var/kat/katsdpscripts/RTS/rfi_mask.pickle',
help=
"Optional pickle file with boolean array specifying channels to mask (default is no mask)"
)
(opts, args) = parser.parse_args()
if len(args) < 1:
raise RuntimeError('Please specify an HDF5 file to check')
data = katdal.open(args[0])
n_chan = np.shape(data.channels)[0]
if opts.freq_chans is not None:
start_f = int(opts.freq_chans.split(',')[0])
end_f = int(opts.freq_chans.split(',')[1])
chan_range = slice(start_f, end_f)
else:
chan_range = slice(data.shape[1] // 4, 3 * data.shape[1] // 4)
# load static flags if pickle file is given
channel_mask = opts.channel_mask
if channel_mask > 0:
pickle_file = open(channel_mask)
rfi_static_flags = pickle.load(pickle_file)
pickle_file.close()
if n_chan > rfi_static_flags.shape[0]:
#command-line parameters
parser = optparse.OptionParser(
usage="Please specify the input file\n\
USAGE: python analyse_self_generated_rfi.py <inputfile.h5> ",
description=
"Evaluate the auto & cross correlation spectra to Find the RFI spikes\
that appear consistently in all observations/pointings.")
opts, args = parser.parse_args()
# if no enough arguments, raise the runtimeError
if len(args) < 1:
raise RuntimeError("No File passed as argument to script")
filename = args[0]
h5 = katdal.open(filename)
# user defined variables
print("Please wait while analysis is in progress...")
pdf = PdfPages(filename.split('/')[-1] + '_RFI.pdf')
page_length = 70
#antennas = [ant.name for ant in fileopened.ants]
#targets = [('%s' % (i.name)) for i in fileopened.catalogue.targets]
chan_range = slice(1, -1) # remove dc spike
#freqs = fileopened.channel_freqs*1.0e-6
detection_function = detect_spikes_sumthreshold
#plot_horizontal_selection_per_antenna
for pol in ['H', 'V']:
(all_text, figlist) = plot_selection_per_antenna(h5, pol, chan_range,
detection_function)
output = None
for scan, _, target in kd.scans():
scan_length = len(kd.dumps)
if scan_length > max_scan:
max_scan, scan_select = scan_length, scan
if scan_select >= 0:
output = f'{kd.obs_params["capture_block_id"]:13s} {scan:<5d} {max_scan:<6d} {target.name}'
return output
parser = argparse.ArgumentParser()
parser.add_argument("katdata", help="URL of observation for which to find the associated delaycal CBID.")
args = parser.parse_args()
base_katdata = katdal.open(args.katdata)
sb_cbids = base_katdata.source.telstate.get_range('sdp_capture_block_id', st=0)
delay_cbids = [cbid[0] for cbid in sb_cbids[::-1]
if 'calibrate_delays.py' in
base_katdata.source.telstate.view(cbid[0])['obs_params']['script_name']]
# Check each candidate delaycal CBID for an appropriate noise-diode scan
out = []
for cbid in delay_cbids:
dc_katdata = katdal.open(args.katdata, capture_block_id=cbid)
dc_search = _get_noise_diode_scan(dc_katdata)
if dc_search is not None:
out += [dc_search]
if out == []:
print(f'No valid delay calibration observations found.')
else:
#open ouput files
text_log = open(text_log_filename, 'w')
pp = PdfPages(pdf_filename)
print 'Searching the data file from the mounted archive'
if opts.noarchive:
d = [opts.filename]
else:
import katarchive
d = katarchive.get_archived_products(datafile)
while len(d) == 0:
time.sleep(10)
d = katarchive.get_archived_products(datafile)
print "Opening %s using katfile, this might take a while" % (datafile, )
f = katfile.open(d[0], quicklook=True)
#start a figure
figure(figsize=(13.5, 6.5))
axes(frame_on=False)
xticks([])
yticks([])
title(datafile + " Observation Report", fontsize=16, fontweight="bold")
frontpage = make_frontpage(f)
text_log.write(frontpage)
text(0, 0, frontpage, fontsize=12)
savefig(pp, format='pdf')
print f
lst_time, loc_datetime = lst_date(f)
def __init__(self, cbid_stream_rdb_file):
katdata = katdal.open(cbid_stream_rdb_file)
metfilename = '{}.met'.format(katdata.source.data.name)
super(MeerKATTelescopeProductMetExtractor,
self).__init__(katdata, metfilename)
self.product_type = 'MeerKATTelescopeProduct'
