def main(argv=None, interactive=True):
"""
Entry point for setup.py.
Wrapper for a profiler if requested otherwise just call run() directly.
If profiling is enabled we disable interactivity as it would wait for user
input and influence the statistics. However the -r option still works.
"""
# catch and ignore a NumPy deprecation warning
with warnings.catch_warnings(record=True):
warnings.filterwarnings(
"ignore", 'The compiler package is deprecated and removed in '
'Python 3.x.', DeprecationWarning)
np.safe_eval('1')
if '-p' in sys.argv or '--profile' in sys.argv:
try:
import cProfile as Profile
except ImportError:
import Profile
Profile.run('from obspy.scripts.runtests import run; run()',
'obspy.pstats')
import pstats
stats = pstats.Stats('obspy.pstats')
print()
print("Profiling:")
stats.sort_stats('cumulative').print_stats('obspy.', 20)
print(PSTATS_HELP)
else:
errors = run(argv, interactive)
if errors:
sys.exit(1)
def dataframe_inputs_to_dict(dfs, df_misc, df_settings):
# write the dataframes back to one dictionary
input_dictionary = dict()
input_dictionary = df_misc.to_dict()[0]
for df in dfs:
for col in df.columns:
if "units" not in col:
input_dictionary[col] = df[col].tolist()
else:
input_dictionary[col] = df[col].unique()[0]
input_dictionary["settings_dictionary"] = df_settings.to_dict()["settings"]
# set the format back for the edge cases
input_dictionary["settings_dictionary"]["model"] = np.safe_eval(
input_dictionary["settings_dictionary"]["model"]
)
input_dictionary["settings_dictionary"]["default_absorption_times"] = (
np.safe_eval(
input_dictionary["settings_dictionary"]["default_absorption_times"]
)
)
input_dictionary["offset_applied_to_dates"] = (
int(input_dictionary["offset_applied_to_dates"])
)
return input_dictionary
def main(argv=None, interactive=True):
"""
Entry point for setup.py.
Wrapper for a profiler if requested otherwise just call run() directly.
If profiling is enabled we disable interactivity as it would wait for user
input and influence the statistics. However the -r option still works.
"""
# catch and ignore a NumPy deprecation warning
with warnings.catch_warnings(record=True):
warnings.filterwarnings(
"ignore", 'The compiler package is deprecated and removed in '
'Python 3.x.', DeprecationWarning)
np.safe_eval('1')
if '-p' in sys.argv or '--profile' in sys.argv:
try:
import cProfile as Profile
except ImportError:
import Profile
Profile.run('from obspy.scripts.runtests import run; run()',
'obspy.pstats')
import pstats
stats = pstats.Stats('obspy.pstats')
print()
print("Profiling:")
stats.sort_stats('cumulative').print_stats('obspy.', 20)
print(PSTATS_HELP)
else:
errors = run(argv, interactive)
if errors:
sys.exit(1)
def input_table_to_dict(input_df):
dict_ = dict()
# first parse and format the settings
all_settings = input_df["settings"].dropna()
dict_["settings_dictionary"] = all_settings.to_dict()
for k in dict_["settings_dictionary"].keys():
if k in ["dynamic_carb_absorption_enabled", "retrospective_correction_enabled"]:
dict_["settings_dictionary"][k] = str2bool(dict_["settings_dictionary"][k])
else:
dict_["settings_dictionary"][k] = np.safe_eval(
dict_["settings_dictionary"][k]
)
if "suspend_threshold" not in dict_["settings_dictionary"].keys():
dict_["settings_dictionary"]["suspend_threshold"] = None
# then parse and format the rest
input_df_T = input_df.drop(columns=["settings"]).dropna(axis=0, how="all").T
input_df_columns = input_df_T.columns
for col in input_df_columns:
if "units" in col:
dict_[col] = input_df_T[col].dropna().unique()[0]
elif "offset" in col:
dict_[col] = int(np.safe_eval(input_df_T[col].dropna()[0]))
elif "time_to_calculate" in col:
dict_[col] = datetime.datetime.fromisoformat(
pd.to_datetime(input_df_T[col].dropna()[0]).isoformat()
)
else:
temp_df = input_df_T[col].dropna()
temp_array = []
for v in temp_df.values:
if ":" in v:
if len(v) == 7:
obj = datetime.time.fromisoformat(
pd.to_datetime(v).strftime("%H:%M:%S")
)
elif len(v) == 8:
obj = datetime.time.fromisoformat(v)
elif len(v) > 8:
obj = datetime.datetime.fromisoformat(
pd.to_datetime(v).isoformat()
)
else:
obj = np.safe_eval(v)
elif "DoseType" in v:
obj = DoseType.from_str(v[9:])
else:
obj = np.safe_eval(v)
temp_array = np.append(temp_array, obj)
dict_[col] = list(temp_array)
return dict_
def _h5_telstate_unpack(s):
"""Unpack a telstate value from its encoded representation."""
try:
# Since 2016-05-09 the HDF5 TelescopeState contains encoded values
return telstate_decode(s)
except katsdptelstate.DecodeError:
try:
# Before 2016-05-09 the telstate values were str() representations
# This cannot be unpacked in general but works for numbers at least
return np.safe_eval(s)
except (ValueError, SyntaxError):
# When unsure, return the string itself (correct for string sensors)
return s
def _telstate_unpack(s):
"""This unpacks a telstate value from its string representation."""
try:
# Since 2016-05-09 the HDF5 TelescopeState contains pickled values
return pickle.loads(s)
except (pickle.UnpicklingError, ValueError, EOFError):
try:
# Before 2016-05-09 the telstate values were str() representations
# This cannot be unpacked in general but works for numbers at least
return np.safe_eval(s)
except (ValueError, SyntaxError):
# When unsure, return the string itself (correct for string sensors)
return s
def test_chunk_retrieved_matches_values_injected(self):
''' add invalid data - less than 240 values '''
hour_offset = 0
''' one second = 1/60 minute '''
two_seconds = 2 / 60
values_list = [1, 2, 3]
window = self.ecg.inject_reading(offset_hours=hour_offset,
interval_minutes=two_seconds,
values_list=values_list)
#
injected_values = np.safe_eval(window.iloc[-1]['values'])
self.assertEqual(
injected_values, values_list,
"Oops! injected values do not match retrieved values")
def __init__(self, metadata):
self.metadata = metadata
if metadata is None or metadata['container'] != 'numpy':
raise NotANumpyArray
# The try except is a backwards compatability hack for the old way of
# serializing ndarray dtype which was used prior to 0.7.2. This means
# the dtype 'descr' was serialized directly to json and not via 'repr'.
# As such, it does not need to be evaluated, but instead is already a
# string that can be passed to the constructor. Note that this never
# worked for nested dtype objects and that this required it to be
# changed in the first place.
try:
dtype_ = numpy.safe_eval(metadata['dtype'])
except SyntaxError:
dtype_ = metadata['dtype']
self.ndarray = numpy.empty(metadata['shape'],
dtype=numpy.dtype(dtype_),
order=metadata['order'])
self.ptr = self.ndarray.__array_interface__['data'][0]
solution_types = ['G', 'B', 'K']
ms_soltype_lookup = {'G': 'G Jones', 'B': 'B Jones', 'K': 'K Jones'}
print "\nWriting calibration solution tables to disk...."
if 'TelescopeState' not in first_h5.file.keys():
print " No TelescopeState in first H5 file. Can't create solution tables.\n"
else:
# first get solution antenna ordering
# newer h5 files have the cal antlist as a sensor
if 'cal_antlist' in first_h5.file['TelescopeState'].keys():
a0 = first_h5.file['TelescopeState']['cal_antlist'].value
antlist = pickle.loads(a0[0][1])
# older h5 files have the cal antlist as an attribute
elif 'cal_antlist' in first_h5.file['TelescopeState'].attrs.keys():
antlist = np.safe_eval(
first_h5.file['TelescopeState'].attrs['cal_antlist'])
else:
print " No calibration antenna ordering in first H5 file. Can't create solution tables.\n"
continue
antlist_indices = range(len(antlist))
# for each solution type in the h5 file, create a table
for sol in solution_types:
caltable_name = '{0}.{1}'.format(caltable_basename, sol)
sol_name = 'cal_product_{0}'.format(sol, )
if sol_name in first_h5.file['TelescopeState'].keys():
print ' - creating {0} solution table: {1}\n'.format(
sol, caltable_name)
# get solution values from the h5 file
from io import StringIO
from typing import Any, Dict
import numpy as np
AR: np.ndarray[Any, np.dtype[np.float64]]
AR_DICT: Dict[str, np.ndarray[Any, np.dtype[np.float64]]]
FILE: StringIO
def func(a: int) -> bool: ...
reveal_type(np.deprecate(func)) # E: def (a: builtins.int) -> builtins.bool
reveal_type(np.deprecate()) # E: _Deprecate
reveal_type(np.deprecate_with_doc("test")) # E: _Deprecate
reveal_type(np.deprecate_with_doc(None)) # E: _Deprecate
reveal_type(np.byte_bounds(AR)) # E: Tuple[builtins.int, builtins.int]
reveal_type(np.byte_bounds(np.float64())) # E: Tuple[builtins.int, builtins.int]
reveal_type(np.who(None)) # E: None
reveal_type(np.who(AR_DICT)) # E: None
reveal_type(np.info(1, output=FILE)) # E: None
reveal_type(np.source(np.interp, output=FILE)) # E: None
reveal_type(np.lookfor("binary representation", output=FILE)) # E: None
reveal_type(np.safe_eval("1 + 1")) # E: Any
def main():
tag_to_intent = {
'gaincal': 'CALIBRATE_PHASE,CALIBRATE_AMPLI',
'bpcal': 'CALIBRATE_BANDPASS,CALIBRATE_FLUX',
'target': 'TARGET'
}
usage = "%prog [options] <dataset> [<dataset2>]*"
description = "Convert MVF dataset(s) to CASA MeasurementSet. The datasets may " \
"be local filenames or archive URLs (including access tokens). " \
"If there are multiple datasets they will be concatenated via " \
"katdal before conversion."
parser = optparse.OptionParser(usage=usage, description=description)
parser.add_option("-o",
"--output-ms",
default=None,
help="Name of output MeasurementSet")
parser.add_option(
"-c",
"--circular",
action="store_true",
default=False,
help="Produce quad circular polarisation. (RR, RL, LR, LL) "
"*** Currently just relabels the linear pols ****")
parser.add_option(
"-r",
"--ref-ant",
help="Override the reference antenna used to pick targets "
"and scans (default is the 'array' antenna in MVFv4 "
"and the first antenna in older formats)")
parser.add_option("-t",
"--tar",
action="store_true",
default=False,
help="Tar-ball the MS")
parser.add_option(
"-f",
"--full_pol",
action="store_true",
default=False,
help="Produce a full polarisation MS in CASA canonical order "
"(HH, HV, VH, VV). Default is to produce HH,VV only.")
parser.add_option("-v",
"--verbose",
action="store_true",
default=False,
help="More verbose progress information")
parser.add_option("-w",
"--stop-w",
action="store_true",
default=False,
help="Use W term to stop fringes for each baseline")
parser.add_option("-p",
"--pols-to-use",
default=None,
help="Select polarisation products to include in MS as "
"comma-separated list (from: HH, HV, VH, VV). "
"Default is all available from (HH, VV).")
parser.add_option(
"-u",
"--uvfits",
action="store_true",
default=False,
help="Print command to convert MS to miriad uvfits in casapy")
parser.add_option("-a",
"--no-auto",
action="store_true",
default=False,
help="MeasurementSet will exclude autocorrelation data")
parser.add_option(
"-s",
"--keep-spaces",
action="store_true",
default=False,
help="Keep spaces in source names, default removes spaces")
parser.add_option(
"-C",
"--channel-range",
help="Range of frequency channels to keep (zero-based inclusive "
"'first_chan,last_chan', default is all channels)")
parser.add_option("-e",
"--elevation-range",
help="Flag elevations outside the range "
"'lowest_elevation,highest_elevation'")
parser.add_option(
"-m",
"--model-data",
action="store_true",
default=False,
help="Add MODEL_DATA and CORRECTED_DATA columns to the MS. "
"MODEL_DATA initialised to unity amplitude zero phase, "
"CORRECTED_DATA initialised to DATA.")
flag_names = ', '.join(name for name in FLAG_NAMES
if not name.startswith('reserved'))
parser.add_option("--flags",
default="all",
help="List of online flags to apply "
"(from " + flag_names + ") "
"default is all flags, '' will apply no flags)")
parser.add_option("--dumptime",
type=float,
default=0.0,
help="Output time averaging interval in seconds, "
"default is no averaging")
parser.add_option("--chanbin",
type=int,
default=0,
help="Bin width for channel averaging in channels, "
"default is no averaging")
parser.add_option(
"--flagav",
action="store_true",
default=False,
help="If a single element in an averaging bin is flagged, "
"flag the averaged bin")
parser.add_option("--caltables",
action="store_true",
default=False,
help="Create calibration tables from gain solutions in "
"the dataset (if present)")
parser.add_option("--quack",
type=int,
default=1,
metavar='N',
help="Discard the first N dumps "
"(which are frequently incomplete)")
parser.add_option("--applycal",
default="",
help="List of calibration solutions to apply to data as "
"a string of comma-separated names, e.g. 'l1' or "
"'K,B,G'. Use 'default' for L1 + L2 and 'all' for "
"all available products.")
(options, args) = parser.parse_args()
# Loading is I/O-bound, so give more threads than CPUs
dask.config.set(
pool=multiprocessing.pool.ThreadPool(4 * multiprocessing.cpu_count()))
if len(args) < 1:
parser.print_help()
raise RuntimeError(
"Please provide one or more MVF dataset names as arguments")
if options.elevation_range and len(options.elevation_range.split(',')) < 2:
raise RuntimeError(
"You have selected elevation flagging. Please provide elevation "
"limits in the form 'lowest_elevation,highest_elevation'.")
if len(args) > 1:
print("Concatenating multiple datasets into single MS.")
def antenna_indices(na, no_auto_corr):
"""Get default antenna1 and antenna2 arrays."""
return np.triu_indices(na, 1 if no_auto_corr else 0)
def corrprod_index(dataset):
"""The correlator product index (with -1 representing missing indices)."""
corrprod_to_index = {
tuple(cp): n
for n, cp in enumerate(dataset.corr_products)
}
# ==========================================
# Generate per-baseline antenna pairs and
# correlator product indices
# ==========================================
def _cp_index(a1, a2, pol):
"""Create correlator product index from antenna pair and pol."""
a1 = a1.name + pol[0].lower()
a2 = a2.name + pol[1].lower()
return corrprod_to_index.get((a1, a2), -1)
# Generate baseline antenna pairs
ant1_index, ant2_index = antenna_indices(len(dataset.ants),
options.no_auto)
# Order as similarly to the input as possible, which gives better performance
# in permute_baselines.
bl_indices = list(zip(ant1_index, ant2_index))
bl_indices.sort(key=lambda ants: _cp_index(dataset.ants[ants[
0]], dataset.ants[ants[1]], pols_to_use[0]))
# Undo the zip
ant1_index[:] = [bl[0] for bl in bl_indices]
ant2_index[:] = [bl[1] for bl in bl_indices]
ant1 = [dataset.ants[a1] for a1 in ant1_index]
ant2 = [dataset.ants[a2] for a2 in ant2_index]
# Create actual correlator product index
cp_index = [
_cp_index(a1, a2, p) for a1, a2 in zip(ant1, ant2)
for p in pols_to_use
]
cp_index = np.array(cp_index, dtype=np.int32)
CPInfo = namedtuple(
"CPInfo", ["ant1_index", "ant2_index", "ant1", "ant2", "cp_index"])
return CPInfo(ant1_index, ant2_index, ant1, ant2, cp_index)
# Open dataset
open_args = args[0] if len(args) == 1 else args
# katdal can handle a list of datasets, which get virtually concatenated internally
dataset = katdal.open(open_args,
ref_ant=options.ref_ant,
applycal=options.applycal)
if dataset.applycal_products:
print('The following calibration products will be applied:',
', '.join(dataset.applycal_products))
else:
print('No calibration products will be applied')
# Get list of unique polarisation products in the dataset
pols_in_dataset = np.unique([(cp[0][-1] + cp[1][-1]).upper()
for cp in dataset.corr_products])
# Which polarisation do we want to write into the MS
# select all possible pols if full-pol selected, otherwise the selected polarisations via pols_to_use
# otherwise finally select any of HH,VV present (the default).
pols_to_use = ['HH', 'HV', 'VH', 'VV'] if (options.full_pol or options.circular) else \
list(np.unique(options.pols_to_use.split(','))) if options.pols_to_use else \
[pol for pol in ['HH', 'VV'] if pol in pols_in_dataset]
# Check we have the chosen polarisations
if np.any([pol not in pols_in_dataset for pol in pols_to_use]):
raise RuntimeError(
f"Selected polarisation(s): {', '.join(pols_to_use)} not available. "
f"Available polarisation(s): {','.join(pols_in_dataset)}")
# Set full_pol if this is selected via options.pols_to_use
if set(pols_to_use) == {'HH', 'HV', 'VH', 'VV'} and not options.circular:
options.full_pol = True
# Extract one MS per spectral window in the dataset(s)
for win in range(len(dataset.spectral_windows)):
dataset.select(reset='T')
centre_freq = dataset.spectral_windows[win].centre_freq
print(
f'Extract MS for spw {win}: centre frequency {int(centre_freq)} Hz'
)
# If no output MS directory name supplied, infer it from dataset(s)
if options.output_ms is None:
if len(dataset.spectral_windows) > 1:
# Use frequency label to disambiguate multiple spectral windows
ms_name = default_ms_name(args, centre_freq)
else:
ms_name = default_ms_name(args)
else:
ms_name = options.output_ms
basename = os.path.splitext(ms_name)[0]
# Discard first N dumps which are frequently incomplete
dataset.select(spw=win,
scans='track',
flags=options.flags,
dumps=slice(options.quack, None))
# The first step is to copy the blank template MS to our desired output
# (making sure it's not already there)
if os.path.exists(ms_name):
raise RuntimeError(
f"MS '{ms_name}' already exists - please remove it "
"before running this script")
print("Will create MS output in " + ms_name)
# Instructions to flag by elevation if requested
if options.elevation_range is not None:
emin, emax = options.elevation_range.split(',')
print(
"\nThe MS can be flagged by elevation in casapy v3.4.0 or higher, with the command:"
)
print(
f" tflagdata(vis='{ms_name}', mode='elevation', lowerlimit={emin}, "
f"upperlimit={emax}, action='apply')\n")
# Instructions to create uvfits file if requested
if options.uvfits:
uv_name = basename + ".uvfits"
print(
"\nThe MS can be converted into a uvfits file in casapy, with the command:"
)
print(
f" exportuvfits(vis='{ms_name}', fitsfile='{uv_name}', datacolumn='data')\n"
)
if options.full_pol:
print(
"\n#### Producing a full polarisation MS (HH,HV,VH,VV) ####\n")
else:
print(
f"\n#### Producing MS with {','.join(pols_to_use)} polarisation(s) ####\n"
)
# if fringe stopping is requested, check that it has not already been done in hardware
if options.stop_w:
print(
"W term in UVW coordinates will be used to stop the fringes.")
try:
autodelay = [
int(ad) for ad in dataset.sensor['DBE/auto-delay']
]
if all(autodelay):
print("Fringe-stopping already performed in hardware... "
"do you really want to stop the fringes here?")
except KeyError:
pass
# Select frequency channel range
if options.channel_range is not None:
channel_range = [
int(chan_str) for chan_str in options.channel_range.split(',')
]
first_chan, last_chan = channel_range[0], channel_range[1]
if (first_chan < 0) or (last_chan >= dataset.shape[1]):
raise RuntimeError(
"Requested channel range outside data set boundaries. "
f"Set channels in the range [0,{dataset.shape[1] - 1}]")
if first_chan > last_chan:
raise RuntimeError(
f"First channel ({first_chan}) bigger than last channel "
f"({last_chan}) - did you mean it the other way around?")
chan_range = slice(first_chan, last_chan + 1)
print(f"\nChannel range {first_chan} through {last_chan}.")
dataset.select(channels=chan_range)
# Are we averaging?
average_data = False
# Determine the number of channels
nchan = len(dataset.channels)
# Work out channel average and frequency increment
if options.chanbin > 1:
average_data = True
# Check how many channels we are dropping
chan_remainder = nchan % options.chanbin
avg_nchan = int(nchan / min(nchan, options.chanbin))
print(
f"Averaging {options.chanbin} channels, output ms will have {avg_nchan} channels."
)
if chan_remainder > 0:
print(
f"The last {chan_remainder} channels in the data will be dropped "
f"during averaging ({options.chanbin} does not divide {nchan})."
)
chan_av = options.chanbin
nchan = avg_nchan
else:
# No averaging in channel
chan_av = 1
# Get the frequency increment per averaged channel
channel_freq_width = dataset.channel_width * chan_av
# Work out dump average and dump increment
# Is the desired time bin greater than the dump period?
if options.dumptime > dataset.dump_period:
average_data = True
dump_av = int(np.round(options.dumptime / dataset.dump_period))
time_av = dump_av * dataset.dump_period
print(
f"Averaging {dataset.dump_period} second dumps to {time_av} seconds."
)
else:
# No averaging in time
dump_av = 1
time_av = dataset.dump_period
# Print a message if extending flags to averaging bins.
if average_data and options.flagav and options.flags != '':
print("Extending flags to averaging bins.")
# Optionally keep only cross-correlation products
if options.no_auto:
dataset.select(corrprods='cross')
print("\nCross-correlations only.")
print(
f"\nUsing {dataset.ref_ant} as the reference antenna. All targets and scans "
"will be based on this antenna.\n")
# MS expects timestamps in MJD seconds
start_time = dataset.start_time.to_mjd() * 24 * 60 * 60
end_time = dataset.end_time.to_mjd() * 24 * 60 * 60
# MVF version 1 and 2 datasets are KAT-7; the rest are MeerKAT
telescope_name = 'KAT-7' if dataset.version[0] in '12' else 'MeerKAT'
# increment scans sequentially in the ms
scan_itr = 1
print("\nIterating through scans in dataset(s)...\n")
cp_info = corrprod_index(dataset)
nbl = cp_info.ant1_index.size
npol = len(pols_to_use)
field_names, field_centers, field_times = [], [], []
obs_modes = ['UNKNOWN']
total_size = 0
# Create the MeasurementSet
table_desc, dminfo = ms_extra.kat_ms_desc_and_dminfo(
nbl=nbl, nchan=nchan, ncorr=npol, model_data=options.model_data)
ms_extra.create_ms(ms_name, table_desc, dminfo)
ms_dict = {}
ms_dict['ANTENNA'] = ms_extra.populate_antenna_dict(
[ant.name for ant in dataset.ants],
[ant.position_ecef
for ant in dataset.ants], [ant.diameter for ant in dataset.ants])
ms_dict['FEED'] = ms_extra.populate_feed_dict(len(dataset.ants),
num_receptors_per_feed=2)
ms_dict['DATA_DESCRIPTION'] = ms_extra.populate_data_description_dict()
ms_dict['POLARIZATION'] = ms_extra.populate_polarization_dict(
ms_pols=pols_to_use, circular=options.circular)
ms_dict['OBSERVATION'] = ms_extra.populate_observation_dict(
start_time, end_time, telescope_name, dataset.observer,
dataset.experiment_id)
# before resetting ms_dict, copy subset to caltable dictionary
if options.caltables:
caltable_dict = {}
caltable_dict['ANTENNA'] = ms_dict['ANTENNA']
caltable_dict['OBSERVATION'] = ms_dict['OBSERVATION']
print("Writing static meta data...")
ms_extra.write_dict(ms_dict, ms_name, verbose=options.verbose)
# Pre-allocate memory buffers
tsize = dump_av
in_chunk_shape = (tsize, ) + dataset.shape[1:]
scan_vis_data = np.empty(in_chunk_shape, dataset.vis.dtype)
scan_weight_data = np.empty(in_chunk_shape, dataset.weights.dtype)
scan_flag_data = np.empty(in_chunk_shape, dataset.flags.dtype)
ms_chunk_shape = (SLOTS, tsize // dump_av, nbl, nchan, npol)
raw_vis_data = ms_async.RawArray(ms_chunk_shape, scan_vis_data.dtype)
raw_weight_data = ms_async.RawArray(ms_chunk_shape,
scan_weight_data.dtype)
raw_flag_data = ms_async.RawArray(ms_chunk_shape, scan_flag_data.dtype)
ms_vis_data = raw_vis_data.asarray()
ms_weight_data = raw_weight_data.asarray()
ms_flag_data = raw_flag_data.asarray()
# Need to limit the queue to prevent overwriting slots before they've
# been processed. The -2 allows for the one we're writing and the one
# the writer process is reading.
work_queue = multiprocessing.Queue(maxsize=SLOTS - 2)
result_queue = multiprocessing.Queue()
writer_process = multiprocessing.Process(
target=ms_async.ms_writer_process,
args=(work_queue, result_queue, options, dataset.ants, cp_info,
ms_name, raw_vis_data, raw_weight_data, raw_flag_data))
writer_process.start()
try:
slot = 0
for scan_ind, scan_state, target in dataset.scans():
s = time.time()
scan_len = dataset.shape[0]
prefix = f'scan {scan_ind:3d} ({scan_len:4d} samples)'
if scan_state != 'track':
if options.verbose:
print(f"{prefix} skipped '{scan_state}' - not a track")
continue
if scan_len < 2:
if options.verbose:
print(f'{prefix} skipped - too short')
continue
if target.body_type != 'radec':
if options.verbose:
print(
f"{prefix} skipped - target '{target.name}' not RADEC"
)
continue
print(
f"{prefix} loaded. Target: '{target.name}'. Writing to disk..."
)
# Get the average dump time for this scan (equal to scan length
# if the dump period is longer than a scan)
dump_time_width = min(time_av, scan_len * dataset.dump_period)
# Get UTC timestamps
utc_seconds = dataset.timestamps[:]
# Update field lists if this is a new target
if target.name not in field_names:
# Since this will be an 'radec' target, we don't need antenna
# or timestamp to get the (astrometric) ra, dec
ra, dec = target.radec()
field_names.append(target.name)
field_centers.append((ra, dec))
field_times.append(
katpoint.Timestamp(utc_seconds[0]).to_mjd() * 60 * 60 *
24)
if options.verbose:
print(
f"Added new field {len(field_names) - 1}: '{target.name}' {ra} {dec}"
)
field_id = field_names.index(target.name)
# Determine the observation tag for this scan
obs_tag = ','.join(tag_to_intent[tag] for tag in target.tags
if tag in tag_to_intent)
# add tag to obs_modes list
if obs_tag and obs_tag not in obs_modes:
obs_modes.append(obs_tag)
# get state_id from obs_modes list if it is in the list, else 0 'UNKNOWN'
state_id = obs_modes.index(
obs_tag) if obs_tag in obs_modes else 0
# Iterate over time in some multiple of dump average
ntime = utc_seconds.size
ntime_av = 0
for ltime in range(0, ntime - tsize + 1, tsize):
utime = ltime + tsize
tdiff = utime - ltime
out_freqs = dataset.channel_freqs
# load all visibility, weight and flag data
# for this scan's timestamps.
# Ordered (ntime, nchan, nbl*npol)
load(dataset, np.s_[ltime:utime, :, :], scan_vis_data,
scan_weight_data, scan_flag_data)
# This are updated as we go to point to the current storage
vis_data = scan_vis_data
weight_data = scan_weight_data
flag_data = scan_flag_data
out_utc = utc_seconds[ltime:utime]
# Overwrite the input visibilities with averaged visibilities,
# flags, weights, timestamps, channel freqs
if average_data:
vis_data, weight_data, flag_data, out_utc, out_freqs = \
averager.average_visibilities(vis_data, weight_data, flag_data,
out_utc, out_freqs, timeav=dump_av,
chanav=chan_av, flagav=options.flagav)
# Infer new time dimension from averaged data
tdiff = vis_data.shape[0]
# Select correlator products and permute axes
cp_index = cp_info.cp_index.reshape((nbl, npol))
vis_data, weight_data, flag_data = permute_baselines(
vis_data, weight_data, flag_data, cp_index,
ms_vis_data[slot], ms_weight_data[slot],
ms_flag_data[slot])
# Increment the number of averaged dumps
ntime_av += tdiff
# Check if writer process has crashed and abort if so
try:
result = result_queue.get_nowait()
raise result
except queue.Empty:
pass
work_queue.put(
ms_async.QueueItem(slot=slot,
target=target,
time_utc=out_utc,
dump_time_width=dump_time_width,
field_id=field_id,
state_id=state_id,
scan_itr=scan_itr))
slot += 1
if slot == SLOTS:
slot = 0
work_queue.put(ms_async.EndOfScan())
result = result_queue.get()
if isinstance(result, Exception):
raise result
scan_size = result.scan_size
s1 = time.time() - s
if average_data and utc_seconds.shape != ntime_av:
print(
f'Averaged {np.shape(utc_seconds)[0]} x {dataset.dump_period} second dumps '
f'to {ntime_av} x {dump_time_width} second dumps')
scan_size_mb = float(scan_size) / (1024**2)
print(f'Wrote scan data ({scan_size_mb:.3f} MiB) '
f'in {s1:.3f} s ({scan_size_mb / s1:.3f} MiBps)\n')
scan_itr += 1
total_size += scan_size
finally:
work_queue.put(None)
writer_exc = None
# Drain the result_queue so that we unblock the writer process
while True:
result = result_queue.get()
if isinstance(result, Exception):
writer_exc = result
elif result is None:
break
writer_process.join()
# This raise is deferred to outside the finally block, so that we don't
# raise an exception while unwinding another one.
if isinstance(writer_exc, Exception):
raise writer_exc
if total_size == 0:
raise RuntimeError("No usable data found in MVF dataset "
"(pick another reference antenna, maybe?)")
# Remove spaces from source names, unless otherwise specified
field_names = [f.replace(' ', '') for f in field_names] \
if not options.keep_spaces else field_names
ms_dict = {}
ms_dict['SPECTRAL_WINDOW'] = ms_extra.populate_spectral_window_dict(
out_freqs, channel_freq_width * np.ones(len(out_freqs)))
ms_dict['FIELD'] = ms_extra.populate_field_dict(
field_centers, field_times, field_names)
ms_dict['STATE'] = ms_extra.populate_state_dict(obs_modes)
ms_dict['SOURCE'] = ms_extra.populate_source_dict(
field_centers, field_times, field_names)
print("\nWriting dynamic fields to disk....\n")
# Finally we write the MS as per our created dicts
ms_extra.write_dict(ms_dict, ms_name, verbose=options.verbose)
if options.tar:
tar = tarfile.open(f'{ms_name}.tar', 'w')
tar.add(ms_name, arcname=os.path.basename(ms_name))
tar.close()
# --------------------------------------
# Now write calibration product tables if required
# Open first HDF5 file in the list to extract TelescopeState parameters from
# (can't extract telstate params from contatenated katdal file as it
# uses the hdf5 file directly)
first_dataset = katdal.open(args[0], ref_ant=options.ref_ant)
main_table = ms_extra.open_table(ms_name, verbose=options.verbose)
if options.caltables:
# copy extra subtable dictionary values necessary for caltable
caltable_dict['SPECTRAL_WINDOW'] = ms_dict['SPECTRAL_WINDOW']
caltable_dict['FIELD'] = ms_dict['FIELD']
solution_types = ['G', 'B', 'K']
ms_soltype_lookup = {
'G': 'G Jones',
'B': 'B Jones',
'K': 'K Jones'
}
print("\nWriting calibration solution tables to disk....")
if 'TelescopeState' not in first_dataset.file.keys():
print(
" No TelescopeState in first dataset. Can't create solution tables.\n"
)
else:
# first get solution antenna ordering
# newer files have the cal antlist as a sensor
if 'cal_antlist' in first_dataset.file['TelescopeState'].keys(
):
a0 = first_dataset.file['TelescopeState/cal_antlist'][()]
antlist = telstate_decode(a0[0][1])
# older files have the cal antlist as an attribute
elif 'cal_antlist' in first_dataset.file[
'TelescopeState'].attrs.keys():
antlist = np.safe_eval(
first_dataset.file['TelescopeState'].
attrs['cal_antlist'])
else:
print(" No calibration antenna ordering in first dataset. "
"Can't create solution tables.\n")
continue
antlist_indices = list(range(len(antlist)))
# for each solution type in the file, create a table
for sol in solution_types:
caltable_name = f'{basename}.{sol}'
sol_name = f'cal_product_{sol}'
if sol_name in first_dataset.file['TelescopeState'].keys():
print(
f' - creating {sol} solution table: {caltable_name}\n'
)
# get solution values from the file
solutions = first_dataset.file['TelescopeState'][
sol_name][()]
soltimes, solvals = [], []
for t, s in solutions:
soltimes.append(t)
solvals.append(telstate_decode(s))
solvals = np.array(solvals)
# convert averaged UTC timestamps to MJD seconds.
sol_mjd = np.array([
katpoint.Timestamp(time_utc).to_mjd() * 24 * 60 *
60 for time_utc in soltimes
])
# determine solution characteristics
if len(solvals.shape) == 4:
ntimes, nchans, npols, nants = solvals.shape
else:
ntimes, npols, nants = solvals.shape
nchans = 1
solvals = solvals.reshape(ntimes, nchans, npols,
nants)
# create calibration solution measurement set
caltable_desc = ms_extra.caltable_desc_float \
if sol == 'K' else ms_extra.caltable_desc_complex
caltable = ms_extra.open_table(caltable_name,
tabledesc=caltable_desc)
# add other keywords for main table
if sol == 'K':
caltable.putkeyword('ParType', 'Float')
else:
caltable.putkeyword('ParType', 'Complex')
caltable.putkeyword('MSName', ms_name)
caltable.putkeyword('VisCal', ms_soltype_lookup[sol])
caltable.putkeyword('PolBasis', 'unknown')
# add necessary units
caltable.putcolkeywords(
'TIME', {
'MEASINFO': {
'Ref': 'UTC',
'type': 'epoch'
},
'QuantumUnits': ['s']
})
caltable.putcolkeywords('INTERVAL',
{'QuantumUnits': ['s']})
# specify that this is a calibration table
caltable.putinfo({
'readme': '',
'subType': ms_soltype_lookup[sol],
'type': 'Calibration'
})
# get the solution data to write to the main table
solutions_to_write = solvals.transpose(
0, 3, 1, 2).reshape(ntimes * nants, nchans, npols)
# MS's store delays in nanoseconds
if sol == 'K':
solutions_to_write = 1e9 * solutions_to_write
times_to_write = np.repeat(sol_mjd, nants)
antennas_to_write = np.tile(antlist_indices, ntimes)
# just mock up the scans -- this doesnt actually correspond to scans in the data
scans_to_write = np.repeat(list(range(len(sol_mjd))),
nants)
# write the main table
main_cal_dict = ms_extra.populate_caltable_main_dict(
times_to_write, solutions_to_write,
antennas_to_write, scans_to_write)
ms_extra.write_rows(caltable,
main_cal_dict,
verbose=options.verbose)
# create and write subtables
subtables = [
'OBSERVATION', 'ANTENNA', 'FIELD',
'SPECTRAL_WINDOW', 'HISTORY'
]
subtable_key = [(os.path.join(caltable.name(), st))
for st in subtables]
# Add subtable keywords and create subtables
# ------------------------------------------------------------------------------
# # this gives an error in casapy:
# *** Error *** MSObservation(const Table &) - table is not a valid MSObservation
# for subtable, subtable_location in zip(subtables, subtable_key)
# ms_extra.open_table(subtable_location, tabledesc=ms_extra.ms_desc[subtable])
# caltable.putkeyword(subtable, 'Table: {0}'.format(subtable_location))
# # write the static info for the table
# ms_extra.write_dict(caltable_dict, caltable.name(), verbose=options.verbose)
# ------------------------------------------------------------------------------
# instead try just copying the main table subtables
# this works to plot the data casapy, but the solutions still can't be
# applied in casapy...
for subtable, subtable_location in zip(
subtables, subtable_key):
main_subtable = ms_extra.open_table(
os.path.join(main_table.name(), subtable))
main_subtable.copy(subtable_location, deep=True)
caltable.putkeyword(subtable,
f'Table: {subtable_location}')
if subtable == 'ANTENNA':
caltable.putkeyword('NAME', antlist)
caltable.putkeyword('STATION', antlist)
if sol != 'B':
spw_table = ms_extra.open_table(
os.path.join(caltable.name(),
'SPECTRAL_WINDOW'))
spw_table.removerows(spw_table.rownumbers())
cen_index = len(out_freqs) // 2
# the delay values in the cal pipeline are calculated relative to frequency 0
ref_freq = 0.0 if sol == 'K' else None
spw_dict = {
'SPECTRAL_WINDOW':
ms_extra.populate_spectral_window_dict(
np.atleast_1d(out_freqs[cen_index]),
np.atleast_1d(channel_freq_width),
ref_freq=ref_freq)
}
ms_extra.write_dict(spw_dict,
caltable.name(),
verbose=options.verbose)
# done with this caltable
caltable.flush()
caltable.close()
main_table.close()
async def get_sensors(self) -> List[Sensor]:
"""Get list of sensors to be collected from CAM.
Returns
-------
sensors : list of `Sensor`
"""
# Tell mypy that these must have been initialised
assert self._sub_name is not None
assert self._cbf_name is not None
assert self._sdp_name is not None
receptors = await self.get_receptors()
input_labels = await self.get_sensor_value('{}_input_labels'.format(
self._cbf_name))
input_labels = input_labels.split(',')
band = await self.get_sensor_value('{}_band'.format(self._sub_name))
rx_name = 'rsc_rx{}'.format(band)
dig_name = 'dig_{}_band'.format(band)
# Build table of names for expanding sensor templates
substitutions: Dict[str, List[Tuple[str, List[str]]]] = {
'receptor': [(name, [name]) for name in receptors],
'receiver': [(rx_name, [rx_name])],
'digitiser': [(dig_name, [dig_name])],
'subarray': [(self._sub_name, ['sub'])],
'cbf': [(self._cbf_name, ['cbf'])],
'sdp': [(self._sdp_name, ['sdp'])],
'inputn': [],
'instrument': [],
'stream': [],
'sub_stream': [],
'stream.cbf.tied_array_channelised_voltage.inputn': []
}
for stream_type in STREAM_TYPES:
substitutions['stream.' + stream_type] = []
substitutions['sub_stream.' + stream_type] = []
cam_prefix = self._cbf_name
for (number, name) in enumerate(input_labels):
# input{} is the old version, input name is the new version. For
# now try with both and one of them won't exist.
substitutions['inputn'].append(('input{}'.format(number), [name]))
substitutions['inputn'].append((name, [name]))
# Add the per instrument specific sensors for every instrument we know about
for instrument in self._instruments:
cam_instrument = "{}_{}".format(cam_prefix, instrument)
sdp_instruments = [instrument]
substitutions['instrument'].append(
(cam_instrument, sdp_instruments))
# For each stream we add type specific sensors
for (full_stream_name, stream_type) in self._streams_with_type.items():
if stream_type not in STREAM_TYPES:
self._logger.warning('Skipping stream %s with unknown type %s',
full_stream_name, stream_type)
cam_stream = "{}_{}".format(cam_prefix, full_stream_name)
cam_sub_stream = "{}_streams_{}".format(self._sub_name,
full_stream_name)
sdp_streams = [full_stream_name]
substitutions['stream'].append((cam_stream, sdp_streams))
substitutions['stream.' + stream_type].append(
(cam_stream, sdp_streams))
substitutions['sub_stream'].append((cam_sub_stream, sdp_streams))
substitutions['sub_stream.' + stream_type].append(
(cam_sub_stream, sdp_streams))
# tied-array-channelised-voltage per-input sensors are special:
# only a subset of the inputs are used and only the corresponding
# sensors exist.
if stream_type == 'cbf.tied_array_channelised_voltage':
source_indices = await self.get_sensor_value(cam_stream +
'_source_indices')
source_indices = np.safe_eval(source_indices)
sublist = substitutions['stream.{}.inputn'.format(stream_type)]
for index in source_indices:
if 0 <= index < len(input_labels):
name = '{}_{}'.format(full_stream_name,
input_labels[index])
sublist.append(('{}_input{}'.format(cam_stream,
index), [name]))
else:
self._logger.warning(
'Out of range source index %d on %s', index,
full_stream_name)
sensors: List[Sensor] = []
for template in SENSORS:
expanded = template.expand(substitutions)
sensors.extend(expanded)
return sensors
output = np.squeeze(a=output)
output_cat = np.argmax(a=output, axis=1)
labels_cat = np.argmax(a=labels, axis=1)
acc_with_reinit = np.mean(np.equal(output_cat, labels_cat))
print("Accuracy: %s" % str(acc_with_reinit))
if __name__ == "__main__":
base_dir = "."
categories = {0: 0, 5: 1, 7: 2}
hdf5_fn = "MNISTDemo.h5"
index_fn = "MNISTDemo_indexSplit.pkl"
# Load layout
with open(os.path.join(base_dir, "dnn_layout"), "r") as layout_file:
dnn_layout = np.safe_eval("[" + layout_file.read() + "]")
print("Training verbosity has been turned off for the tests to make the "
"output easier \nto parse. These tests will take several minutes "
"to complete.")
print("------------------------------------------------------------------"
"--------------")
print("TEST: test_from_scratch()")
test_from_scratch()
print("TEST: test_continue_training()")
test_continue_training()
print("TEST: test_apply_dnn()")
test_apply_dnn()
import numpy as np np.deprecate(1) # E: No overload variant np.deprecate_with_doc(1) # E: incompatible type np.byte_bounds(1) # E: incompatible type np.who(1) # E: incompatible type np.lookfor(None) # E: incompatible type np.safe_eval(None) # E: incompatible type
