class DeviceExampleServer(katcp.DeviceServer):
## Interface version information.
VERSION_INFO = ("example-server", 0, 1)
## Device server build / instance information.
BUILD_INFO = ("my-example-server", 0, 1, "rc1")
#pylint: disable-msg=R0904
def setup_sensors(self):
pass
def request_echo(self, sock, msg):
"""Echo the arguments of the message sent."""
return katcp.Message.reply(msg.name, "ok", *msg.arguments)
@request(Str(), Int())
@return_reply(Str())
def request_repeat(self, sock, txt, n):
"""Repeat txt n times."""
return ("ok", txt * n)
@request(Float(), Float())
@return_reply(Float())
def request_add(self, sock, x, y):
"""Add x and y."""
return ("ok", x + y)
@request(Int(), Int())
@return_reply(Int())
def request_intdiv(self, sock, x, y):
"""Perform integer division of x and y."""
return ("ok", x // y)
def setUp(self):
basic = Int()
default = Int(default=11)
optional = Int(optional=True)
default_optional = Int(default=11, optional=True)
self.minmax = Int(min=5, max=6)
self._pack = [
(basic, 5, "5"),
(basic, -5, "-5"),
(basic, "a", TypeError),
(basic, None, ValueError),
(self.minmax, 5, "5"),
(self.minmax, 6, "6"),
(self.minmax, 4, ValueError),
(self.minmax, 7, ValueError),
(default, None, "11"),
(default_optional, None, "11"),
(optional, None, ValueError),
]
self._unpack = [
(basic, "5", 5),
(basic, "-5", -5),
(basic, "a", ValueError),
(basic, None, ValueError),
(self.minmax, "5", 5),
(self.minmax, "6", 6),
(self.minmax, "4", ValueError),
(self.minmax, "7", ValueError),
(default, None, 11),
(default_optional, None, 11),
(optional, None, None),
]
class FakeHandlers(object):
@request(Int(), Int())
@return_reply(Int())
def request_add_test(self, req, a, b):
"Add numbers"
req.inform(a * 2, b * 3)
return ('ok', a + b)
@request(Int(), Int())
@return_reply(Float())
@tornado.gen.coroutine
def request_async_divide(self, req, a, b):
"Divide numbers"
req.inform(a / 2, b / 10)
req.inform('polony-is-real meat')
raise tornado.gen.Return(('ok', a / b))
@tornado.testing.gen_test
def test_request_handlers(self):
yield self.fake_inspecting_client.connect()
test_handlers = FakeHandlers()
self.fake_inspecting_manager.add_request_handlers_object(test_handlers)
reply, informs = yield self.fake_inspecting_client.simple_request(
'add-test', 1, 5, mid='123')
self.assertEqual(len(informs), 1)
self.assertEqual(str(informs[0]), '#add-test[123] 2 15')
self.assertEqual(str(reply), '!add-test[123] ok 6')
reply, informs = yield self.fake_inspecting_client.simple_request(
'async-divide', 7, 2, mid='112')
self.assertEqual(len(informs), 2)
self.assertEqual(str(informs[0]),
'#async-divide[112] {} {}'.format(7 / 2, 2 / 10))
self.assertEqual(str(informs[1]),
'#async-divide[112] polony-is-real\\_meat')
class ManagementNode(NodeServer):
"""Katcp server for cluster head nodes.
Notes: This is the basis of the top level
interface for FBF/APSUSE.
"""
VERSION_INFO = ("reynard-managementnode-api", 0, 1)
BUILD_INFO = ("reynard-managementnode-implementation", 0, 1, "rc1")
def __init__(self, server_host, server_port, config):
self._clients = {}
super(ManagementNode, self).__init__(server_host, server_port, config)
def start(self):
"""Start the server
Based on the passed configuration object this is
where the clients for suboridnates nodes will be
set up.
"""
super(ManagementNode, self).start()
self.ioloop.add_callback(self._setup_clients)
def _setup_clients(self):
"""Setup clients based on configuration object."""
for node, port in self._config.NODES:
name = '{node}-client'.format(node=node)
self._add_client(name, '127.0.0.1', port)
def _add_client(self, name, ip, port):
"""Add a named client."""
if name in self._clients.keys():
raise KeyError(
"Client already exists with name '{name}'".format(name=name))
client = KATCPClientResource(
dict(name=name, address=(ip, port), controlled=True))
client.start()
self._clients[name] = client
def _remove_client(self, name):
"""Remove a client by name."""
if name not in self._clients.keys():
raise KeyError(
"No client exists with name '{name}'".format(name=name))
self._clients[name].stop()
self._clients[name].join()
del self._clients[name]
@request(Str(), Str(), Int())
@return_reply(Str())
def request_client_add(self, req, name, ip, port):
"""Add a new client."""
try:
self._add_client(name, ip, port)
except KeyError, e:
return ("fail", str(e))
return ("ok", "added client")
class PafBackendController(AsyncDeviceServer):
VERSION_INFO = ("paf-backend-controller-api", 0, 1)
BUILD_INFO = ("paf-backend-controller-implementation", 0, 1, "rc1")
DEVICE_STATUSES = ["ok", "degraded", "fail"]
def __init__(self, ip, port):
super(PafBackendController, self).__init__(ip, port)
def setup_sensors(self):
self._device_status = Sensor.discrete(
"device-status",
description="Health status of PafBackendController",
params=self.DEVICE_STATUSES,
default="ok",
initial_status=Sensor.UNKNOWN)
self.add_sensor(self._device_status)
def start(self):
super(PafBackendController, self).start()
@request(Int(), Float(), Bool())
@return_reply(Int(), Float())
def request_myreq(self, req, my_int, my_float, my_bool):
'''?myreq my_int my_float my_bool'''
return ("ok", my_int + 1, my_float / 2.0)
@request(Str())
@return_reply(Str())
def request_echo(self, req, message):
"""
@brief A request that echos a message
"""
return ("ok", message)
@request(Str())
@return_reply(Str())
def request_echomine(self, req, message):
"""
@brief A request that echos a message
"""
return ("ok", message)
def setUp(self):
basic = Int()
default = Int(default=11)
optional = Int(optional=True)
default_optional = Int(default=11, optional=True)
minmax = Int(min=5, max=6)
big_minmax = Int(min=-2**64, max=2**64)
self._pack = [
(basic, 5, b"5"),
(basic, -5, b"-5"),
(basic, "a", TypeError),
(basic, None, ValueError),
(minmax, 5, b"5"),
(minmax, 6, b"6"),
(minmax, 4, ValueError),
(minmax, 7, ValueError),
(big_minmax, 2**64, b"18446744073709551616"),
(big_minmax, -2**64, b"-18446744073709551616"),
(big_minmax, 2**64 + 1, ValueError),
(big_minmax, -2**64 - 1, ValueError),
(default, None, b"11"),
(default_optional, None, b"11"),
(optional, None, ValueError),
]
self._unpack = [
(basic, b"5", 5),
(basic, b"-5", -5),
(basic, b"a", ValueError),
(basic, None, ValueError),
(minmax, b"5", 5),
(minmax, b"6", 6),
(minmax, b"4", ValueError),
(minmax, b"7", ValueError),
(default, None, 11),
(default_optional, None, 11),
(optional, None, None),
]
class TestDevice(object):
def __init__(self):
self.sent_messages = []
@request(Int(min=1, max=10), Discrete(("on", "off")), Bool())
@return_reply(Int(min=1, max=10), Discrete(("on", "off")), Bool())
def request_one(self, req, i, d, b):
if i == 3:
return ("fail", "I failed!")
if i == 5:
return ("bananas", "This should never be sent")
if i == 6:
return ("ok", i, d, b, "extra parameter")
if i == 9:
self.finish_request_one(req, i, d, b)
raise AsyncReply()
return ("ok", i, d, b)
@send_reply(Int(min=1, max=10), Discrete(("on", "off")), Bool())
def finish_request_one(self, req, i, d, b):
return (req, "ok", i, d, b)
def reply(self, req, msg, orig_msg):
self.sent_messages.append([req, msg])
@request(Int(min=1, max=3, default=2),
Discrete(("on", "off"), default="off"), Bool(default=True))
@return_reply(Int(min=1, max=3), Discrete(("on", "off")), Bool())
def request_two(self, req, i, d, b):
return ("ok", i, d, b)
@return_reply(Int(min=1, max=3), Discrete(("on", "off")), Bool())
@request(Int(min=1, max=3), Discrete(("on", "off")), Bool())
def request_three(self, req, i, d, b):
return ("ok", i, d, b)
@return_reply()
@request()
def request_four(self, req):
return ["ok"]
@inform(Int(min=1, max=3), Discrete(("on", "off")), Bool())
def inform_one(self, i, d, b):
pass
@request(Timestamp(), Timestamp(optional=True), major=4)
@return_reply(Timestamp(), Timestamp(default=321), major=4)
def request_katcpv4_time(self, req, timestamp1, timestamp2):
self.katcpv4_time1 = timestamp1
self.katcpv4_time2 = timestamp2
if timestamp2:
return ('ok', timestamp1, timestamp2)
else:
return ('ok', timestamp1)
@request(Timestamp(multiple=True), major=4)
@return_reply(Timestamp(multiple=True), major=4)
def request_katcpv4_time_multi(self, req, *timestamps):
self.katcpv4_time_multi = timestamps
return ('ok', ) + timestamps
@return_reply(Int(), Str())
@request(Int(), include_msg=True)
def request_eight(self, req, msg, i):
return ("ok", i, msg.name)
@request(Int(), Float(multiple=True))
@return_reply(Int(), Float(multiple=True))
def request_int_multifloat(self, req, i, *floats):
return ('ok', i) + floats
class FbfWorkerServer(AsyncDeviceServer):
VERSION_INFO = ("fbf-control-server-api", 0, 1)
BUILD_INFO = ("fbf-control-server-implementation", 0, 1, "rc1")
DEVICE_STATUSES = ["ok", "degraded", "fail"]
STATES = [
"idle", "preparing", "ready", "starting", "capturing", "stopping",
"error"
]
IDLE, PREPARING, READY, STARTING, CAPTURING, STOPPING, ERROR = STATES
def __init__(self, ip, port, capture_interface, numa_node, exec_mode=FULL):
"""
@brief Construct new FbfWorkerServer instance
@params ip The interface address on which the server should listen
@params port The port that the server should bind to
@params de_ip The IP address of the delay engine server
@params de_port The port number for the delay engine server
"""
self._dc_ip = None
self._dc_port = None
self._delay_client = None
self._delays = None
self._numa = numa_node
self._exec_mode = exec_mode
self._dada_input_key = "dada"
self._dada_coh_output_key = "caca"
self._dada_incoh_output_key = "baba"
self._capture_interface = capture_interface
self._capture_monitor = None
self._input_level = 10.0
self._output_level = 10.0
self._partition_bandwidth = None
self._centre_frequency = None
super(FbfWorkerServer, self).__init__(ip, port)
@coroutine
def start(self):
"""Start FbfWorkerServer server"""
super(FbfWorkerServer, self).start()
@coroutine
def stop(self):
yield super(FbfWorkerServer, self).stop()
def setup_sensors(self):
"""
@brief Set up monitoring sensors.
Sensor list:
- device-status
- local-time-synced
- fbf0-status
- fbf1-status
@note The following sensors are made available on top of default
sensors implemented in AsynDeviceServer and its base classes.
device-status: Reports the health status of the FBFUSE
and associated devices:
Among other things report HW failure, SW
failure and observation failure.
"""
self._device_status_sensor = Sensor.discrete(
"device-status",
description="Health status of FbfWorkerServer instance",
params=self.DEVICE_STATUSES,
default="ok",
initial_status=Sensor.NOMINAL)
self.add_sensor(self._device_status_sensor)
self._state_sensor = LoggingSensor.discrete(
"state",
params=self.STATES,
description="The current state of this worker instance",
default=self.IDLE,
initial_status=Sensor.NOMINAL)
self._state_sensor.set_logger(log)
self.add_sensor(self._state_sensor)
self._capture_interface_sensor = Sensor.string(
"capture-interface",
description="The IP address of the NIC to be used for data capture",
default=self._capture_interface,
initial_status=Sensor.NOMINAL)
self.add_sensor(self._capture_interface_sensor)
self._delay_client_sensor = Sensor.string(
"delay-engine-server",
description="The address of the currently set delay engine",
default="",
initial_status=Sensor.UNKNOWN)
self.add_sensor(self._delay_client_sensor)
self._antenna_capture_order_sensor = Sensor.string(
"antenna-capture-order",
description=
"The order in which the worker will capture antennas internally",
default="",
initial_status=Sensor.UNKNOWN)
self.add_sensor(self._antenna_capture_order_sensor)
self._mkrecv_header_sensor = Sensor.string(
"mkrecv-capture-header",
description=
"The MKRECV/DADA header used for configuring capture with MKRECV",
default="",
initial_status=Sensor.UNKNOWN)
self.add_sensor(self._mkrecv_header_sensor)
self._mksend_coh_header_sensor = Sensor.string(
"mksend-coherent-beam-header",
description=
"The MKSEND/DADA header used for configuring transmission of coherent beam data",
default="",
initial_status=Sensor.UNKNOWN)
self.add_sensor(self._mksend_coh_header_sensor)
self._mksend_incoh_header_sensor = Sensor.string(
"mksend-incoherent-beam-header",
description=
"The MKSEND/DADA header used for configuring transmission of incoherent beam data",
default="",
initial_status=Sensor.UNKNOWN)
self.add_sensor(self._mksend_incoh_header_sensor)
self._psrdada_cpp_args_sensor = Sensor.string(
"psrdada-cpp-arguments",
description="The command line arguments used to invoke psrdada_cpp",
default="",
initial_status=Sensor.UNKNOWN)
self.add_sensor(self._psrdada_cpp_args_sensor)
self._mkrecv_heap_loss = Sensor.float(
"feng-heap-loss",
description=("The percentage if F-engine heaps lost "
"(within MKRECV statistics window)"),
default=0.0,
initial_status=Sensor.UNKNOWN,
unit="%")
self.add_sensor(self._mkrecv_heap_loss)
self._ingress_buffer_percentage = Sensor.float(
"ingress-buffer-fill-level",
description=("The percentage fill level for the capture"
"buffer between MKRECV and PSRDADA_CPP"),
default=0.0,
initial_status=Sensor.UNKNOWN,
unit="%")
self.add_sensor(self._ingress_buffer_percentage)
self._cb_egress_buffer_percentage = Sensor.float(
"cb-egress-buffer-fill-level",
description=("The percentage fill level for the transmission"
"buffer between PSRDADA_CPP and MKSEND (for "
"coherent beams)"),
default=0.0,
initial_status=Sensor.UNKNOWN,
unit="%")
self.add_sensor(self._cb_egress_buffer_percentage)
self._ib_egress_buffer_percentage = Sensor.float(
"ib-egress-buffer-fill-level",
description=("The percentage fill level for the transmission"
"buffer between PSRDADA_CPP and MKSEND (for "
"incoherent beams)"),
default=0.0,
initial_status=Sensor.UNKNOWN,
unit="%")
self.add_sensor(self._ib_egress_buffer_percentage)
@property
def capturing(self):
return self.state == self.CAPTURING
@property
def idle(self):
return self.state == self.IDLE
@property
def starting(self):
return self.state == self.STARTING
@property
def stopping(self):
return self.state == self.STOPPING
@property
def ready(self):
return self.state == self.READY
@property
def preparing(self):
return self.state == self.PREPARING
@property
def error(self):
return self.state == self.ERROR
@property
def state(self):
return self._state_sensor.value()
def _system_call_wrapper(self, cmd):
log.debug("System call: '{}'".format(" ".join(cmd)))
check_call(cmd)
@coroutine
def _make_db(self, key, block_size, nblocks, timeout=120):
try:
yield self._destroy_db(key, timeout=20)
except Exception as error:
log.debug("Could not clean previous buffer (key={}): {}".format(
key, str(error)))
log.debug(("Building DADA buffer: key={}, block_size={}, "
"nblocks={}").format(key, block_size, nblocks))
if self._exec_mode == FULL:
cmdline = map(str, [
"dada_db", "-k", key, "-b", block_size, "-n", nblocks, "-l",
"-p"
])
proc = Popen(cmdline,
stdout=PIPE,
stderr=PIPE,
shell=False,
close_fds=True)
yield process_watcher(proc,
name="make_db({})".format(key),
timeout=timeout)
else:
log.warning(("Current execution mode disables "
"DADA buffer creation/destruction"))
@coroutine
def _destroy_db(self, key, timeout=20.0):
log.debug("Destroying DADA buffer with key={}".format(key))
if self._exec_mode == FULL:
cmdline = map(str, ["dada_db", "-k", key, "-d"])
proc = Popen(cmdline,
stdout=PIPE,
stderr=PIPE,
shell=False,
close_fds=True)
yield process_watcher(proc,
name="destroy_db({})".format(key),
timeout=timeout)
else:
log.warning(("Current execution mode disables "
"DADA buffer creation/destruction"))
@coroutine
def _reset_db(self, key, timeout=5.0):
log.debug("Resetting DADA buffer with key={}".format(key))
if self._exec_mode == FULL:
cmdline = map(str, ["dbreset", "-k", key])
proc = Popen(cmdline,
stdout=PIPE,
stderr=PIPE,
shell=False,
close_fds=True)
yield process_watcher(proc,
name="reset_db({})".format(key),
timeout=timeout)
else:
log.warning(("Current execution mode disables "
"DADA buffer reset"))
def set_affinity(self, pid, core_spec):
log.debug("Setting affinity for PID {} to {}".format(pid, core_spec))
os.system("taskset -cp {} {}".format(core_spec, pid))
@request(Float(), Float())
@return_reply()
def request_set_levels(self, req, input_level, output_level):
"""
@brief Set the input and output levels for FBFUSE
@param req A katcp request object
@param input_level The standard deviation of the data
from the F-engines.
@param output_level The standard deviation of the data
output from FBFUSE.
"""
self._input_level = input_level
self._output_level = output_level
return ("ok", )
@request(Str(), Int(), Int(), Float(), Float(), Str(), Str(), Str(), Str(),
Int())
@return_reply()
def request_prepare(self, req, feng_groups, nchans_per_group, chan0_idx,
chan0_freq, chan_bw, feng_config, coherent_beam_config,
incoherent_beam_config, dc_ip, dc_port):
"""
@brief Prepare FBFUSE to receive and process data from a subarray
@detail REQUEST ?configure feng_groups, nchans_per_group, chan0_idx, chan0_freq,
chan_bw, mcast_to_beam_map, antenna_to_feng_id_map, coherent_beam_config,
incoherent_beam_config
Configure FBFUSE for the particular data products
@param req A katcp request object
@param feng_groups The contiguous range of multicast groups to capture F-engine data from,
the parameter is formatted in stream notation, e.g.: spead://239.11.1.150+3:7148
@param nchans_per_group The number of frequency channels per multicast group
@param chan0_idx The index of the first channel in the set of multicast groups
@param chan0_freq The frequency in Hz of the first channel in the set of multicast groups
@param chan_bw The channel bandwidth in Hz
@param feng_config JSON dictionary containing general F-engine parameters.
@code
{
'bandwidth': 856e6,
'centre-frequency': 1200e6,
'sideband': 'upper',
'feng-antenna-map': {...},
'sync-epoch': 12353524243.0,
'nchans': 4096
}
@param coherent_beam_config A JSON object specifying the coherent beam configuration in the form:
@code
{
'tscrunch':16,
'fscrunch':1,
'nbeams': 400,
'antennas':'m007,m008,m009',
'destination': 'spead://239.11.1.0+127:7148'
}
@endcode
@param incoherent_beam_config A JSON object specifying the incoherent beam configuration in the form:
@code
{
'tscrunch':16,
'fscrunch':1,
'antennas':'m007,m008,m009',
'destination': 'spead://239.11.1.150:7148'
}
@endcode
@return katcp reply object [[[ !configure ok | (fail [error description]) ]]]
"""
if not self.idle:
return ("fail", "FBF worker not in IDLE state")
log.info("Preparing worker server instance")
try:
feng_config = json.loads(feng_config)
except Exception as error:
msg = ("Unable to parse F-eng config with "
"error: {}").format(str(error))
log.error("Prepare failed: {}".format(msg))
return ("fail", msg)
log.info("F-eng config: {}".format(feng_config))
try:
coherent_beam_config = json.loads(coherent_beam_config)
except Exception as error:
msg = ("Unable to parse coherent beam "
"config with error: {}").format(str(error))
log.error("Prepare failed: {}".format(msg))
return ("fail", msg)
log.info("Coherent beam config: {}".format(coherent_beam_config))
try:
incoherent_beam_config = json.loads(incoherent_beam_config)
except Exception as error:
msg = ("Unable to parse incoherent beam "
"config with error: {}").format(str(error))
log.error("Prepare failed: {}".format(msg))
return ("fail", msg)
log.info("Incoherent beam config: {}".format(incoherent_beam_config))
@coroutine
def configure():
self._state_sensor.set_value(self.PREPARING)
log.debug("Starting delay configuration server client")
self._delay_client = KATCPClientResource(
dict(name="delay-configuration-client",
address=(dc_ip, dc_port),
controlled=True))
self._delay_client.start()
log.info("Determining F-engine capture order")
feng_capture_order_info = determine_feng_capture_order(
feng_config['feng-antenna-map'], coherent_beam_config,
incoherent_beam_config)
log.info("F-engine capture order info: {}".format(
feng_capture_order_info))
feng_to_antenna_map = {
value: key
for key, value in feng_config['feng-antenna-map'].items()
}
antenna_capture_order_csv = ",".join([
feng_to_antenna_map[feng_id]
for feng_id in feng_capture_order_info['order']
])
self._antenna_capture_order_sensor.set_value(
antenna_capture_order_csv)
log.debug("Parsing F-engines to capture: {}".format(feng_groups))
capture_range = ip_range_from_stream(feng_groups)
ngroups = capture_range.count
partition_nchans = nchans_per_group * ngroups
worker_idx = chan0_idx / partition_nchans
partition_bandwidth = partition_nchans * chan_bw
self._partition_bandwidth = partition_bandwidth
sample_clock = feng_config['bandwidth'] * 2
timestamp_step = feng_config['nchans'] * 2 * 256
frequency_ids = [
chan0_idx + nchans_per_group * ii for ii in range(ngroups)
]
nantennas = len(feng_capture_order_info['order'])
heap_size = nchans_per_group * PACKET_PAYLOAD_SIZE
heap_group_size = ngroups * heap_size * nantennas
ngroups_data = int(MAX_DADA_BLOCK_SIZE / heap_group_size)
ngroups_data = 2**((ngroups_data - 1).bit_length())
centre_frequency = chan0_freq + self._partition_bandwidth / 2.0
self._centre_frequency = centre_frequency
# Coherent beam timestamps
coh_heap_size = 8192
nsamps_per_coh_heap = (
coh_heap_size /
(partition_nchans * coherent_beam_config['fscrunch']))
coh_timestamp_step = (coherent_beam_config['tscrunch'] *
nsamps_per_coh_heap * 2 *
feng_config["nchans"])
# Incoherent beam timestamps
incoh_heap_size = 8192
nsamps_per_incoh_heap = (
incoh_heap_size /
(partition_nchans * incoherent_beam_config['fscrunch']))
incoh_timestamp_step = (incoherent_beam_config['tscrunch'] *
nsamps_per_incoh_heap * 2 *
feng_config["nchans"])
timestamp_modulus = lcm(
timestamp_step, lcm(incoh_timestamp_step, coh_timestamp_step))
if self._exec_mode == FULL:
dada_mode = 4
else:
dada_mode = 0
mkrecv_config = {
'dada_mode':
dada_mode,
'dada_key':
self._dada_input_key,
'sync_epoch':
feng_config['sync-epoch'],
'sample_clock':
sample_clock,
'mcast_sources':
",".join([str(group) for group in capture_range]),
'mcast_port':
capture_range.port,
'interface':
self._capture_interface,
'timestamp_step':
timestamp_step,
'timestamp_modulus':
timestamp_modulus,
'ordered_feng_ids_csv':
",".join(map(str, feng_capture_order_info['order'])),
'frequency_partition_ids_csv':
",".join(map(str, frequency_ids)),
'ngroups_data':
ngroups_data,
'heap_size':
heap_size
}
mkrecv_header = make_mkrecv_header(mkrecv_config,
outfile=MKRECV_CONFIG_FILENAME)
self._mkrecv_header_sensor.set_value(mkrecv_header)
log.info(
"Determined MKRECV configuration:\n{}".format(mkrecv_header))
coh_ip_range = ip_range_from_stream(
coherent_beam_config['destination'])
nbeams = coherent_beam_config['nbeams']
nbeams_per_group = nbeams / coh_ip_range.count
msg = "nbeams is not a mutliple of the IP range"
assert nbeams % coh_ip_range.count == 0, msg
"""
Note on data rates:
For both the coherent and incoherent beams, we set the sending
rate in MKSEND equal to 110% of the required data rate. This
is a fudge to ensure that we send rapidly enough that MKSEND
does not limit performance while at the same time ensuring that
the burst rate out of the instrument is limited. This number
may need to be tuned.
"""
coh_data_rate = (partition_bandwidth /
coherent_beam_config['tscrunch'] /
coherent_beam_config['fscrunch'] *
nbeams_per_group * 1.1)
heap_id_start = worker_idx * coh_ip_range.count
log.debug("Determining MKSEND configuration for coherent beams")
dada_mode = int(self._exec_mode == FULL)
coherent_mcast_dest = coherent_beam_config['destination'].lstrip(
"spead://").split(":")[0]
mksend_coh_config = {
'dada_key': self._dada_coh_output_key,
'dada_mode': dada_mode,
'interface': self._capture_interface,
'data_rate': coh_data_rate,
'mcast_port': coh_ip_range.port,
'mcast_destinations': coherent_mcast_dest,
'sync_epoch': feng_config['sync-epoch'],
'sample_clock': sample_clock,
'heap_size': coh_heap_size,
'heap_id_start': heap_id_start,
'timestamp_step': coh_timestamp_step,
'beam_ids': "0:{}".format(nbeams),
'multibeam': True,
'subband_idx': chan0_idx,
'heap_group': nbeams_per_group
}
mksend_coh_header = make_mksend_header(
mksend_coh_config, outfile=MKSEND_COHERENT_CONFIG_FILENAME)
log.info(("Determined MKSEND configuration for coherent beams:\n{}"
).format(mksend_coh_header))
self._mksend_coh_header_sensor.set_value(mksend_coh_header)
log.debug("Determining MKSEND configuration for incoherent beams")
incoh_data_rate = (partition_bandwidth /
incoherent_beam_config['tscrunch'] /
incoherent_beam_config['fscrunch'] * 1.1)
dada_mode = int(self._exec_mode == FULL)
incoh_ip_range = ip_range_from_stream(
incoherent_beam_config['destination'])
coherent_mcast_dest = incoherent_beam_config['destination'].lstrip(
"spead://").split(":")[0]
mksend_incoh_config = {
'dada_key': self._dada_incoh_output_key,
'dada_mode': dada_mode,
'interface': self._capture_interface,
'data_rate': incoh_data_rate,
'mcast_port': incoh_ip_range.port,
'mcast_destinations': coherent_mcast_dest,
'sync_epoch': feng_config['sync-epoch'],
'sample_clock': sample_clock,
'heap_size': incoh_heap_size,
'heap_id_start': worker_idx,
'timestamp_step': incoh_timestamp_step,
'beam_ids': 0,
'multibeam': False,
'subband_idx': chan0_idx,
'heap_group': 1
}
mksend_incoh_header = make_mksend_header(
mksend_incoh_config, outfile=MKSEND_INCOHERENT_CONFIG_FILENAME)
log.info(
"Determined MKSEND configuration for incoherent beam:\n{}".
format(mksend_incoh_header))
self._mksend_incoh_header_sensor.set_value(mksend_incoh_header)
"""
Tasks:
- compile kernels
- create shared memory banks
"""
# Here we create a future object for the psrdada_cpp compilation
# this is the longest running setup task and so intermediate steps
# such as dada buffer generation
fbfuse_pipeline_params = {
'total_nantennas':
len(feng_capture_order_info['order']),
'fbfuse_nchans':
partition_nchans,
'total_nchans':
feng_config['nchans'],
'coherent_tscrunch':
coherent_beam_config['tscrunch'],
'coherent_fscrunch':
coherent_beam_config['fscrunch'],
'coherent_nantennas':
len(coherent_beam_config['antennas'].split(",")),
'coherent_antenna_offset':
feng_capture_order_info["coherent_span"][0],
'coherent_nbeams':
nbeams,
'incoherent_tscrunch':
incoherent_beam_config['tscrunch'],
'incoherent_fscrunch':
incoherent_beam_config['fscrunch']
}
psrdada_compilation_future = compile_psrdada_cpp(
fbfuse_pipeline_params)
log.info("Creating all DADA buffers")
# Create capture data DADA buffer
capture_block_size = ngroups_data * heap_group_size
capture_block_count = int(AVAILABLE_CAPTURE_MEMORY /
capture_block_size)
log.debug("Creating dada buffer for input with key '{}'".format(
"%s" % self._dada_input_key))
input_make_db_future = self._make_db(self._dada_input_key,
capture_block_size,
capture_block_count)
# Create coherent beam output DADA buffer
coh_output_channels = (ngroups * nchans_per_group) / \
coherent_beam_config['fscrunch']
coh_output_samples = ngroups_data * \
256 / coherent_beam_config['tscrunch']
coherent_block_size = (nbeams * coh_output_channels *
coh_output_samples)
coherent_block_count = 32
log.debug(
("Creating dada buffer for coherent beam output "
"with key '{}'").format("%s" % self._dada_coh_output_key))
coh_output_make_db_future = self._make_db(
self._dada_coh_output_key, coherent_block_size,
coherent_block_count)
# Create incoherent beam output DADA buffer
incoh_output_channels = ((ngroups * nchans_per_group) /
incoherent_beam_config['fscrunch'])
incoh_output_samples = ((ngroups_data * 256) /
incoherent_beam_config['tscrunch'])
incoherent_block_size = incoh_output_channels * incoh_output_samples
incoherent_block_count = 32
log.debug(("Creating dada buffer for incoherent beam "
"output with key '{}'").format(
"%s" % self._dada_incoh_output_key))
incoh_output_make_db_future = self._make_db(
self._dada_incoh_output_key, incoherent_block_size,
incoherent_block_count)
# Need to pass the delay buffer controller the F-engine capture
# order but only for the coherent beams
cstart, cend = feng_capture_order_info['coherent_span']
coherent_beam_feng_capture_order = feng_capture_order_info[
'order'][cstart:cend]
coherent_beam_antenna_capture_order = [
feng_to_antenna_map[idx]
for idx in coherent_beam_feng_capture_order
]
# Start DelayBufferController instance
# Here we are going to make the assumption that the server and processing all run in
# one docker container that will be preallocated with the right CPU set, GPUs, memory
# etc. This means that the configurations need to be unique by NUMA node... [Note: no
# they don't, we can use the container IPC channel which isolates
# the IPC namespaces.]
#
# Here we recreate the beam keys as they are handled by the BeamManager
# instance in the product controller
#
beam_idxs = ["cfbf%05d" % (i) for i in range(nbeams)]
self._delay_buf_ctrl = DelayBufferController(
self._delay_client, beam_idxs,
coherent_beam_antenna_capture_order, 1)
yield self._delay_buf_ctrl.start()
# By this point we require psrdada_cpp to have been compiled
# as such we can yield on the future we created earlier
yield psrdada_compilation_future
# Now we can yield on dada buffer generation
yield input_make_db_future
yield coh_output_make_db_future
yield incoh_output_make_db_future
self._state_sensor.set_value(self.READY)
log.info("Prepare request successful")
req.reply("ok", )
@coroutine
def safe_configure():
try:
yield configure()
except Exception as error:
log.exception(str(error))
req.reply("fail", str(error))
self.ioloop.add_callback(safe_configure)
raise AsyncReply
@request()
@return_reply()
def request_deconfigure(self, req):
"""
@brief Deconfigure the FBFUSE instance.
@note Deconfigure the FBFUSE instance. If FBFUSE uses katportalclient to get information
from CAM, then it should disconnect at this time.
@param req A katcp request object
@return katcp reply object [[[ !deconfigure ok | (fail [error description]) ]]]
"""
# Need to make sure everything is stopped
# Call self.stop?
# Need to delete all allocated DADA buffers:
log.info("Received deconfigure request")
@coroutine
def deconfigure():
log.info("Destroying allocated DADA buffers")
try:
yield self._destroy_db(self._dada_input_key)
yield self._destroy_db(self._dada_coh_output_key)
yield self._destroy_db(self._dada_incoh_output_key)
except Exception as error:
log.warning("Error while destroying DADA buffers: {}".format(
str(error)))
log.info("Destroying delay buffers")
del self._delay_buf_ctrl
self._delay_buf_ctrl = None
self._state_sensor.set_value(self.IDLE)
log.info("Deconfigure request successful")
req.reply("ok", )
self.ioloop.add_callback(deconfigure)
raise AsyncReply
@request()
@return_reply()
def request_capture_start(self, req):
"""
@brief Prepare FBFUSE ingest process for data capture.
@note A successful return value indicates that FBFUSE is ready for data capture and
has sufficient resources available. An error will indicate that FBFUSE is not
in a position to accept data
@param req A katcp request object
@return katcp reply object [[[ !capture-init ok | (fail [error description]) ]]]
"""
log.info("Received capture-start request")
try:
self.capture_start()
except Exception as error:
log.exception("Error during capture start")
return ("fail", str(error))
else:
log.info("Capture-start successful")
return ("ok", )
def capture_start(self):
if not self.ready:
raise Exception("FBF worker not in READY state")
self._state_sensor.set_value(self.STARTING)
# Create SPEAD transmitter for coherent beams
if self._numa == 0:
mksend_cpu_set = "7"
psrdada_cpp_cpu_set = "6"
mkrecv_cpu_set = "0-5"
else:
mksend_cpu_set = "14"
psrdada_cpp_cpu_set = "15"
mkrecv_cpu_set = "8-13"
self._mksend_coh_proc = ManagedProcess([
"taskset", "-c", mksend_cpu_set, "mksend", "--header",
MKSEND_COHERENT_CONFIG_FILENAME, "--quiet"
])
self._mksend_incoh_proc = ManagedProcess([
"taskset", "-c", mksend_cpu_set, "mksend", "--header",
MKSEND_INCOHERENT_CONFIG_FILENAME, "--quiet"
])
# Start beamforming pipeline
log.info("Starting PSRDADA_CPP beamforming pipeline")
delay_buffer_key = self._delay_buf_ctrl.shared_buffer_key
# Start beamformer instance
psrdada_cpp_cmdline = [
"taskset", "-c", psrdada_cpp_cpu_set, "fbfuse", "--input_key",
self._dada_input_key, "--cb_key", self._dada_coh_output_key,
"--ib_key", self._dada_incoh_output_key, "--delay_key_root",
delay_buffer_key, "--cfreq", self._centre_frequency, "--bandwidth",
self._partition_bandwidth, "--input_level", self._input_level,
"--output_level", self._output_level, "--log_level", "info"
]
self._psrdada_cpp_args_sensor.set_value(" ".join(
map(str, psrdada_cpp_cmdline)))
log.debug(" ".join(map(str, psrdada_cpp_cmdline)))
self._psrdada_cpp_proc = ManagedProcess(psrdada_cpp_cmdline)
def update_heap_loss_sensor(curr, total, avg, window):
self._mkrecv_heap_loss.set_value(100.0 - avg)
# Create SPEAD receiver for incoming antenna voltages
self._mkrecv_proc = ManagedProcess(
[
"taskset", "-c", mkrecv_cpu_set, "mkrecv_nt", "--header",
MKRECV_CONFIG_FILENAME, "--quiet"
],
stdout_handler=MkrecvStdoutHandler(
callback=update_heap_loss_sensor))
def exit_check_callback():
if not self._mkrecv_proc.is_alive():
log.error("mkrecv_nt exited unexpectedly")
self.ioloop.add_callback(self.capture_stop)
if not self._psrdada_cpp_proc.is_alive():
log.error("fbfuse pipeline exited unexpectedly")
self.ioloop.add_callback(self.capture_stop)
if not self._mksend_coh_proc.is_alive():
log.error("mksend coherent exited unexpectedly")
self.ioloop.add_callback(self.capture_stop)
if not self._mksend_incoh_proc.is_alive():
log.error("mksend incoherent exited unexpectedly")
self.ioloop.add_callback(self.capture_stop)
self._capture_monitor.stop()
self._capture_monitor = PeriodicCallback(exit_check_callback, 1000)
self._capture_monitor.start()
def dada_callback(params):
self._ingress_buffer_percentage.set_value(params["fraction-full"])
# start DB monitors
self._ingress_buffer_monitor = DbMonitor(self._dada_input_key,
callback=dada_callback)
self._ingress_buffer_monitor.start()
self._cb_egress_buffer_monitor = DbMonitor(
self._dada_input_key,
callback=lambda params: self._cb_egress_buffer_percentage.
set_value(params["fraction-full"]))
self._cb_egress_buffer_monitor.start()
self._ib_egress_buffer_monitor = DbMonitor(
self._dada_input_key,
callback=lambda params: self._ib_egress_buffer_percentage.
set_value(params["fraction-full"]))
self._ib_egress_buffer_monitor.start()
self._state_sensor.set_value(self.CAPTURING)
@request()
@return_reply()
def request_capture_stop(self, req):
"""
@brief Terminate the FBFUSE ingest process for the particular FBFUSE instance
@note This writes out any remaining metadata, closes all files, terminates any remaining processes and
frees resources for the next data capture.
@param req A katcp request object
@param product_id This is a name for the data product, used to track which subarray is being told to stop capture.
For example "array_1_bc856M4k".
@return katcp reply object [[[ !capture-done ok | (fail [error description]) ]]]
"""
log.info("Received capture-stop request")
@coroutine
def capture_stop_wrapper():
try:
yield self.capture_stop()
except Exception as error:
log.exception("Capture-stop request failed")
req.reply("fail", str(error))
else:
log.info("Capture-stop request successful")
req.reply("ok", )
self.ioloop.add_callback(capture_stop_wrapper)
raise AsyncReply
@coroutine
def capture_stop(self):
if not self.capturing and not self.error:
return
log.info("Stopping capture")
self._state_sensor.set_value(self.STOPPING)
self._capture_monitor.stop()
self._ingress_buffer_monitor.stop()
self._cb_egress_buffer_monitor.stop()
self._ib_egress_buffer_monitor.stop()
log.info("Stopping MKRECV instance")
self._mkrecv_proc.terminate()
log.info("Stopping PSRDADA_CPP instance")
self._psrdada_cpp_proc.terminate()
log.info("Stopping MKSEND instances")
self._mksend_incoh_proc.terminate()
self._mksend_coh_proc.terminate()
log.info("Resetting DADA buffers")
reset_tasks = []
reset_tasks.append(self._reset_db(self._dada_input_key, timeout=7.0))
reset_tasks.append(
self._reset_db(self._dada_coh_output_key, timeout=4.0))
reset_tasks.append(
self._reset_db(self._dada_incoh_output_key, timeout=5.0))
for task in reset_tasks:
try:
yield task
except Exception as error:
log.warning("Error raised on DB reset: {}".format(str(error)))
self._state_sensor.set_value(self.READY)
class BLBackendInterface(AsyncDeviceServer):
"""Breakthrough Listen's KATCP Server Backend Interface
This server responds to requests sent from CAM, most notably:
@ configue
@ capture-init
@ capture-start
@ capture-stop
@ capture-done
@ deconfigure
But because it inherits from AsyncDeviceServer, also responds to:
* halt
* help
* log-level
* restart [#restartf1]_
* client-list
* sensor-list
* sensor-sampling
* sensor-value
* watchdog
* version-list (only standard in KATCP v5 or later)
* request-timeout-hint (pre-standard only if protocol flags indicates
timeout hints, supported for KATCP v5.1 or later)
* sensor-sampling-clear (non-standard)
"""
VERSION_INFO = ("BLUSE-katcp-interface", 1, 0)
BUILD_INFO = ("BLUSE-katcp-implementation", 1, 0, "rc?")
DEVICE_STATUSES = ["ok", "fail", "degraded"]
def __init__(self, server_host, server_port):
self.port = server_port
self.redis_server = redis.StrictRedis()
super(BLBackendInterface, self).__init__(server_host, server_port)
def start(self):
"""Start the server
Based on the passed configuration object this is
where the clients for suboridnates nodes will be
set up.
"""
super(BLBackendInterface, self).start()
print(R"""
,'''''-._
; ,. <> `-._
; \' _,--'"
; (
; , ` \
;, , \
; | | MeerKAT BL Backend Interface:
; |, | |\ KATCP Server
; | | | \ Version: {}
|.-\ ,\ |\ : Port: {}
|.| `. `-. | ||
:.| `-. \ ';;
.- , \;;|
; , | ,\
; , ; \ https://github.com/ejmichaud/meerkat-backend-interface
; , /`. , )
__,;, ,' \ ,|
_,--''__,| / \ :
,'_,-'' | ,/ | :
/ / | ; ; |
| | __,-| |--..__,--| |---.--....___
___,-| |----'' / | `._`-. `----
\ \ `''' ''' --
`.`. --'
`.`-._ _, ,- __,-
`-.`.
--' `;
""".format("{}.{}".format(self.VERSION_INFO[1], self.VERSION_INFO[2]),
self.port))
@request(Str(), Str(), Int(), Str(), Str())
@return_reply()
def request_configure(self, req, product_id, antennas_csv, n_channels,
streams_json, proxy_name):
"""Receive metadata for upcoming observation.
In order to allow BLUSE to make an estimate of its ability
to process a particular data product, this command should
be used to configure a BLUSE instance when a new subarray is activated.
Args:
product_id (str): This is a name for the data product,
which is a useful tag to include in the data,
but should not be analysed further.
For example "array_1_bc856M4k". This value will
be unique across all subarrays. However, it is
not a globally unique identifier for the lifetime
of the telescope. The exact same value may be provided
at a later time when the same subarray is activated again.
antennas_csv (str): A comma separated list of physical antenna names
used in particular sub-array to which the data products belongs.
n_channels (int): The integer number of frequency channels provided by the CBF.
streams_json (str) is a JSON struct containing config keys and
values describing the streams. For example:
{'stream_type1': {
'stream_name1': 'stream_address1',
'stream_name2': 'stream_address2',
...},
'stream_type2': {
'stream_name1': 'stream_address1',
'stream_name2': 'stream_address2',
...},
...}
The steam type keys indicate the source of the data and the type, e.g. cam.http.
stream_address will be a URI. For SPEAD streams, the format will be
spead://<ip>[+<count>]:<port>, representing SPEAD stream multicast groups.
When a single logical stream requires too much bandwidth to accommodate
as a single multicast group, the count parameter indicates the number of
additional consecutively numbered multicast group ip addresses, and
sharing the same UDP port number.
stream_name is the name used to identify the stream in CAM.
A Python example is shown below, for five streams:
One CAM stream, with type cam.http. The camdata stream provides the
connection string for katportalclient (for the subarray that this
BLUSE instance is being configured on).
One F-engine stream, with type: cbf.antenna_channelised_voltage.
One X-engine stream, with type: cbf.baseline_correlation_products.
Two beam streams, with type: cbf.tied_array_channelised_voltage.
The stream names ending in x are horizontally polarised, and those
ending in y are vertically polarised.
proxy_name (str): The CAM name for the instance of the BLUSE data
proxy that is being configured. For example, "BLUSE_3". This
can be used to query sensors on the correct proxy. Note that for
BLUSE there will only be a single instance of the proxy in a subarray.
Returns:
None... but replies with "ok" or "fail" and logs either info or error
Writes:
- subbarry1_abc65555:timestamp" -> "1534657577373.23423" :: Redis String
- subarray1_abc65555:antennas" -> [1,2,3,4] :: Redis List
- subarray1_abc65555:n_channels" -> "4096" :: Redis String
- subarray1_abc65555:proxy_name "-> "BLUSE_whatever" :: Redis String
- subarray1_abc65555:streams" -> {....} :: Redis Hash !!!CURRENTLY A STRING!!!
- current:obs:id -> "subbary1_abc65555"
Publishes:
redis-channel: 'alerts' <-- "configure"
Examples:
> ?configure array_1_bc856M4k a1,a2,a3,a4 128000 {"cam.http":{"camdata":"http://monctl.devnmk.camlab.kat.ac.za/api/client/2"},"stream_type2":{"stream_name1":"stream_address1","stream_name2":"stream_address2"}} BLUSE_3
"""
try:
antennas_list = antennas_csv.split(",")
json_dict = unpack_dict(streams_json)
cam_url = json_dict['cam.http']['camdata']
except Exception as e:
log.error(e)
return ("fail", e)
statuses = []
statuses.append(
write_pair_redis(self.redis_server,
"{}:timestamp".format(product_id), time.time()))
statuses.append(
write_list_redis(self.redis_server,
"{}:antennas".format(product_id), antennas_list))
statuses.append(
write_pair_redis(self.redis_server,
"{}:n_channels".format(product_id), n_channels))
statuses.append(
write_pair_redis(self.redis_server,
"{}:proxy_name".format(product_id), proxy_name))
statuses.append(
write_pair_redis(self.redis_server,
"{}:streams".format(product_id),
json.dumps(json_dict)))
statuses.append(
write_pair_redis(self.redis_server,
"{}:cam:url".format(product_id), cam_url))
statuses.append(
write_pair_redis(self.redis_server, "current:obs:id", product_id))
msg = "configure:{}".format(product_id)
statuses.append(
publish_to_redis(self.redis_server, REDIS_CHANNELS.alerts, msg))
if all(statuses):
return ("ok", )
else:
return ("fail", "Failed to publish to our local redis server")
@request(Str())
@return_reply()
def request_capture_init(self, req, product_id):
"""Signals that an observation will start soon
Publishes a message to the 'alerts' channel of the form:
capture-init:product_id
The product_id should match what what was sent in the ?configure request
This alert should notify all backend processes (such as beamformer)
to get ready for data
"""
msg = "capture-init:{}".format(product_id)
success = publish_to_redis(self.redis_server, REDIS_CHANNELS.alerts,
msg)
if success:
return ("ok", )
else:
return ("fail", "Failed to publish to our local redis server")
@request(Str())
@return_reply()
def request_capture_start(self, req, product_id):
"""Signals that an observation is starting now
Publishes a message to the 'alerts' channel of the form:
capture-start:product_id
The product_id should match what what was sent in the ?configure request
This alert should notify all backend processes (such as beamformer)
that they need to be collecting data now
"""
msg = "capture-start:{}".format(product_id)
success = publish_to_redis(self.redis_server, REDIS_CHANNELS.alerts,
msg)
if success:
return ("ok", )
else:
return ("fail", "Failed to publish to our local redis server")
@request(Str())
@return_reply()
def request_capture_stop(self, req, product_id):
"""Signals that an observation is has stopped
Publishes a message to the 'alerts' channel of the form:
capture-stop:product_id
The product_id should match what what was sent in the ?configure request
This alert should notify all backend processes (such as beamformer)
that they should stop collecting data now
"""
msg = "capture-stop:{}".format(product_id)
success = publish_to_redis(self.redis_server, REDIS_CHANNELS.alerts,
msg)
if success:
return ("ok", )
else:
return ("fail", "Failed to publish to our local redis server")
@request(Str())
@return_reply()
def request_capture_done(self, req, product_id):
"""Signals that an observation has finished
Publishes a message to the 'alerts' channel of the form:
capture-done:product_id
The product_id should match what what was sent in the ?configure request
This alert should notify all backend processes (such as beamformer)
that their data streams are ending
"""
msg = "capture-done:{}".format(product_id)
success = publish_to_redis(self.redis_server, REDIS_CHANNELS.alerts,
msg)
if success:
return ("ok", )
else:
return ("fail", "Failed to publish to our local redis server")
@request(Str())
@return_reply()
def request_deconfigure(self, req, product_id):
"""Signals that the current data product is done.
Deconfigure the BLUSE instance that was created by the call
to ?configure with the corresponding product_id. Note: CAM is
expected to have sent a ?capture-done request before deconfiguring,
in order to ensure that all data has been written. If BLUSE uses an
instance of katportalclient to get information from CAM for this
BLUSE instance, then it should disconnect at this time.
Publishes a message to the 'alerts' channel of the form:
deconfigure:product_id
The product_id should match what what was sent in the ?configure request
This alert should notify all backend processes (such as beamformer)
that their data streams are ending
"""
msg = "deconfigure:{}".format(product_id)
success = publish_to_redis(self.redis_server, REDIS_CHANNELS.alerts,
msg)
if success:
return ("ok", )
else:
return ("fail", "Failed to publish to our local redis server")
def setup_sensors(self):
"""
@brief Set up monitoring sensors.
@note The following sensors are made available on top of defaul sensors
implemented in AsynDeviceServer and its base classes.
device-status: Reports the health status of the FBFUSE and associated devices:
Among other things report HW failure, SW failure and observation failure.
"""
self._device_status = Sensor.discrete(
"device-status",
description="Health status of BLUSE",
params=self.DEVICE_STATUSES,
default="ok",
initial_status=Sensor.NOMINAL)
self.add_sensor(self._device_status)
self._local_time_synced = Sensor.boolean(
"local-time-synced",
description="Indicates BLUSE is NTP syncronised.",
default=True, # TODO: implement actual NTP synchronization request
initial_status=Sensor.NOMINAL)
self.add_sensor(self._local_time_synced)
self._version = Sensor.string(
"version",
description="Reports the current BLUSE version",
default=str(self.VERSION_INFO[1:]).strip('()').replace(
' ', '').replace(",", '.'), # e.g. '1.0'
initial_status=Sensor.NOMINAL)
self.add_sensor(self._version)
def request_halt(self, req, msg):
"""Halts the server, logs to syslog and slack, and exits the program
Returns
-------
success : {'ok', 'fail'}
Whether scheduling the halt succeeded.
Examples
--------
::
?halt
!halt ok
TODO:
- Call halt method on superclass to avoid copy paste
Doing this caused an issue:
File "/Users/Eric/Berkeley/seti/packages/meerkat/lib/python2.7/site-packages/katcp/server.py", line 1102, in handle_request
assert (reply.mtype == Message.REPLY)
AttributeError: 'NoneType' object has no attribute 'mtype'
"""
f = Future()
@gen.coroutine
def _halt():
req.reply("ok")
yield gen.moment
self.stop(timeout=None)
raise AsyncReply
self.ioloop.add_callback(lambda: chain_future(_halt(), f))
log.critical("HALTING SERVER!!!")
# TODO: uncomment when you deploy
# notify_slack("KATCP server at MeerKAT has halted. Might want to check that!")
sys.exit(0)
@request()
@return_reply(Str())
def request_find_alien(self, req):
"""Finds an alien.
"""
return ("ok", R"""
. . . . . . . . . + .
. . : . .. :. .___---------___.
. . . . :.:. _".^ .^ ^. '.. :"-_. .
. : . . .:../: . .^ :.:\.
. . :: +. :.:/: . . . . . .:\
. : . . _ :::/: . ^ . . .:\
.. . . . - : :.:./. . .:\
. . . :..|: . . ^. .:|
. . : : ..|| . . . !:|
. . . . ::. ::\( . :)/
. . : . : .:.|. ###### .#######::|
:.. . :- : .: ::|.####### ..########:|
. . . .. . .. :\ ######## :######## :/
. .+ :: : -.:\ ######## . ########.:/
. .+ . . . . :.:\. ####### #######..:/
:: . . . . ::.:..:.\ . . ..:/
. . . .. : -::::.\. | | . .:/
. : . . .-:.":.::.\ ..:/
. -. . . . .: .:::.:.\. .:/
. . . : : ....::_:..:\ ___. :/
. . . .:. .. . .: :.:.:\ :/
+ . . : . ::. :.:. .:.|\ .:/|
. + . . ...:: ..| --.:|
. . . . . . . ... :..:.."( ..)"
. . . : . .: ::/ . .::\
""")
class ExampleProtocol(DeviceProtocol):
@request(include_msg=True)
@return_reply(Int(min=0))
def request_req(self, msg):
return "ok", 3
def test_pack_types_more_types_than_args(self):
expected = [b'one', b'2', b'1', b'four']
self.check_packing(
[Str(), Int(),
Bool(default=True),
Str(default='four')], ['one', 2], expected)
def test_request_multi(self):
with self.assertRaises(TypeError) as ex:
request(Bool(multiple=True), Int())
self.assertEqual(
str(ex.exception),
'Only the last parameter type can accept multiple arguments.')
class BLBackendInterface(AsyncDeviceServer):
"""Breakthrough Listen's KATCP Server Backend Interface
This server responds to requests sent from CAM, most notably:
@ configue
@ capture-init
@ capture-start
@ capture-stop
@ capture-done
@ deconfigure
But because it inherits from AsyncDeviceServer, also responds to:
* halt
* help
* log-level
* restart [#restartf1]_
* client-list
* sensor-list
* sensor-sampling
* sensor-value
* watchdog
* version-list (only standard in KATCP v5 or later)
* request-timeout-hint (pre-standard only if protocol flags indicates
timeout hints, supported for KATCP v5.1 or later)
* sensor-sampling-clear (non-standard)
"""
VERSION = "2020-06-19"
DEVICE_STATUSES = ["ok", "fail", "degraded"]
def __init__(self, server_host, server_port):
self.port = server_port
self.redis_server = redis.StrictRedis()
super(BLBackendInterface, self).__init__(server_host, server_port)
def start(self):
"""Start the server
Based on the passed configuration object this is
where the clients for subordinate nodes will be
set up.
"""
super(BLBackendInterface, self).start()
if (sys.stdout.isatty()):
print(R"""
,'''''-._
; ,. <> `-._
; \' _,--'"
; (
; , ` \
;, , \
; | | MeerKAT BL Backend Interface:
; |, | |\ KATCP Server
; | | | \ Version: {}
|.-\ ,\ |\ : Port: {}
|.| `. `-. | ||
:.| `-. \ ';;
.- , \;;|
; , | ,\ https://github.com/danielczech/meerkat-backend-interface
; , ; \ https://github.com/ejmichaud/meerkat-backend-interface
; , /`. , )
__,;, ,' \ ,|
_,--''__,| / \ :
,'_,-'' | ,/ | :
/ / | ; ; |
| | __,-| |--..__,--| |---.--....___
___,-| |----'' / | `._`-. `----
\ \ `''' ''' --
`.`. --'
`.`-._ _, ,- __,-
`-.`.
--' `;
""".format(self.VERSION, self.port))
@request(Str(), Str(), Int(), Str(), Str())
@return_reply()
def request_configure(self, req, product_id, antennas_csv, n_channels,
streams_json, proxy_name):
"""Receive metadata for upcoming observation.
This command is used to configure a BLUSE instance when a
new subarray is activated.
Args:
product_id (str): This is the name of the current subarray,
which is used when requesting sensor data specific to
components which belong to the current subarray.
Eg "array_1".
This name is unique across all current subarrays (no
two concurrently active subarrays will have the same name).
However, it is not a globally unique identifier for all
time. An identical name may be provided for later
activations of other subarrays.
antennas_csv (str): A comma separated list of physical antenna
names used in the current subarray.
n_channels (int): The integer number of frequency channels provided
by the CBF.
streams_json (str) is a JSON struct containing config keys and
values describing the streams. For example:
{'stream_type1': {
'stream_name1': 'stream_address1',
'stream_name2': 'stream_address2',
...},
'stream_type2': {
'stream_name1': 'stream_address1',
'stream_name2': 'stream_address2',
...},
...}
The steam type keys indicate the source of the data and the
type, e.g. a cam.http. stream_address will be a URL.
For SPEAD streams, the format will be spead://<ip>[+<count>]:<port>,
representing SPEAD stream multicast groups.
The count parameter indicates the number of additional consecutively
numbered multicast group IP addresses (sharing the same UDP port
number).
stream_name is the name used to identify the stream in CAM.
A Python example is shown below, for five streams:
One CAM stream, with type cam.http.
The CAM stream provides the connection string for katportalclient
(for the specific subarray that this BLUSE instance is being
configured for).
One F-engine stream, with type: cbf.antenna_channelised_voltage.
One X-engine stream, with type: cbf.baseline_correlation_products.
Two beam streams, with type: cbf.tied_array_channelised_voltage.
The stream names ending in x are horizontally polarised, and those
ending in y are vertically polarised.
proxy_name (str): The CAM name for the instance of the BLUSE data
proxy that is being configured. For example, "bluse_3".
There will only be a single BLUSE instance of the proxy per subarray.
Returns:
None (responds with "ok" or "fail" and logs either info or an error).
Writes:
- [product_id]:timestamp" -> "1534657577373.23423" (Redis string)
- [product_id]:antennas" -> [1,2,3,4] (Redis list)
- [product_id]:n_channels" -> "4096" (Redis string)
- [product_id]:proxy_name "-> "bluse_N" (Redis string)
- [product_id]:streams" -> {....} (Redis string)
- current:obs:id -> [product_id]
- [product_id]:cbf_prefix -> (Redis string)
Publishes:
redis-channel: 'alerts' <-- "configure"
Examples:
> ?configure array_1_bc856M4k a1,a2,a3,a4 128000 {
"cam.http":{"camdata":"http://monctl.devnmk.camlab.kat.ac.za/api/client/2"},
"stream_type2":{"stream_name1":"stream_address1","stream_name2":"stream_address2"}}
BLUSE_3
"""
try:
antennas_list = antennas_csv.split(",")
json_dict = unpack_dict(streams_json)
cam_url = json_dict['cam.http']['camdata']
except Exception as e:
log.error(e)
return ("fail", e)
# Ascertain the CBF sensor prefix (and F-engine output type).
# Default to 'wide'; if 'wide' is not available, try 'narrow1'.
# If neither are available, take the first available F-engine
# output type.
try:
stream_type = 'cbf.antenna_channelised_voltage'
if ('wide.antenna-channelised-voltage' in json_dict[stream_type]):
cbf_prefix = 'wide'
log.info('CBF prefix extracted: wide')
elif ('narrow1.antenna-channelised-voltage'
in json_dict[stream_type]):
cbf_prefix = 'narrow1'
log.info('CBF prefix extracted: narrow1')
else:
cbf_prefix = next(iter(json_dict[stream_type])).split('.')[0]
log.info('CBF prefix extracted: {}'.format(cb_prefix))
except Exception as e:
cbf_prefix = 'wide'
log.error('Could not extract CBF prefix; defaulting to \'wide\'')
statuses = []
statuses.append(
write_pair_redis(self.redis_server,
"{}:timestamp".format(product_id), time.time()))
statuses.append(
write_list_redis(self.redis_server,
"{}:antennas".format(product_id), antennas_list))
statuses.append(
write_pair_redis(self.redis_server,
"{}:n_channels".format(product_id), n_channels))
statuses.append(
write_pair_redis(self.redis_server,
"{}:proxy_name".format(product_id), proxy_name))
statuses.append(
write_pair_redis(self.redis_server,
"{}:streams".format(product_id),
json.dumps(json_dict)))
statuses.append(
write_pair_redis(self.redis_server,
"{}:cam:url".format(product_id), cam_url))
statuses.append(
write_pair_redis(self.redis_server, "current:obs:id", product_id))
statuses.append(
write_pair_redis(self.redis_server,
"{}:cbf_prefix".format(product_id), cbf_prefix))
msg = "configure:{}".format(product_id)
statuses.append(
publish_to_redis(self.redis_server, REDIS_CHANNELS.alerts, msg))
if all(statuses):
return ("ok", )
else:
return ("fail", "Failed to publish to our local redis server")
@request(Str())
@return_reply()
def request_capture_init(self, req, product_id):
"""Signals that an observation will start soon.
Publishes a message to the 'alerts' channel of the form:
capture-init:product_id
The product_id matches that which was sent in the
?configure request.
"""
msg = "capture-init:{}".format(product_id)
success = publish_to_redis(self.redis_server, REDIS_CHANNELS.alerts,
msg)
if success:
return ("ok", )
else:
return ("fail", "Failed to publish to our local redis server")
@request(Str())
@return_reply()
def request_capture_start(self, req, product_id):
"""Signals that an observation is starting now.
Publishes a message to the 'alerts' channel of the form:
capture-start:product_id
The product_id matches that which was sent in the
?configure request.
"""
msg = "capture-start:{}".format(product_id)
success = publish_to_redis(self.redis_server, REDIS_CHANNELS.alerts,
msg)
if success:
return ("ok", )
else:
return ("fail", "Failed to publish to our local redis server")
@request(Str())
@return_reply()
def request_capture_stop(self, req, product_id):
"""Signals that an observation is has stopped.
Publishes a message to the 'alerts' channel of the form:
capture-stop:product_id
The product_id matches that which was sent in the
?configure request.
"""
msg = "capture-stop:{}".format(product_id)
success = publish_to_redis(self.redis_server, REDIS_CHANNELS.alerts,
msg)
if success:
return ("ok", )
else:
return ("fail", "Failed to publish to our local redis server")
@request(Str())
@return_reply()
def request_capture_done(self, req, product_id):
"""Signals that an observation has finished.
Publishes a message to the 'alerts' channel of the form:
capture-done:product_id
The product_id matches that which was sent in the
?configure request.
"""
msg = "capture-done:{}".format(product_id)
success = publish_to_redis(self.redis_server, REDIS_CHANNELS.alerts,
msg)
if success:
return ("ok", )
else:
return ("fail", "Failed to publish to our local redis server")
@request(Str())
@return_reply()
def request_deconfigure(self, req, product_id):
"""Signals that the current subarray has been deconfigured (dismantled)
and its associated components released for use in another subarray.
This is the signal to deconfigure the BLUSE instance associated with
the current subarray (and which was created by the call to ?configure
with the corresponding product_id).
Note: CAM is expected to have sent a ?capture-done request before
deconfiguring. If BLUSE uses an instance of katportalclient to get
information from CAM for the current subarray, it should disconnect.
Publishes a message to the 'alerts' channel of the form:
deconfigure:product_id
The product_id matches that which was sent in the ?configure request.
Other backend processes (such as beamformer) should be notified
that their data streams are ending.
"""
msg = "deconfigure:{}".format(product_id)
success = publish_to_redis(self.redis_server, REDIS_CHANNELS.alerts,
msg)
if success:
return ("ok", )
else:
return ("fail", "Failed to publish to our local redis server")
def setup_sensors(self):
# TODO: Need to re-look at this function.
"""
@brief Set up monitoring sensors.
@note The following sensors are made available on top of default sensors
implemented in AsynDeviceServer and its base classes.
device-status: Reports the health status of the proxy and associated devices:
Among other things report HW failure, SW failure and observation failure.
"""
self._device_status = Sensor.discrete(
"device-status",
description="Health status of BLUSE",
params=self.DEVICE_STATUSES,
default="ok",
initial_status=Sensor.NOMINAL)
self.add_sensor(self._device_status)
self._local_time_synced = Sensor.boolean(
"local-time-synced",
description="Indicates BLUSE is NTP syncronised.",
default=True, # TODO: implement actual NTP synchronization request
initial_status=Sensor.NOMINAL)
self.add_sensor(self._local_time_synced)
self._version = Sensor.string(
"version",
description="Reports the current BLUSE version",
default=str(self.VERSION_INFO[1:]).strip('()').replace(
' ', '').replace(",", '.'), # e.g. '1.0'
initial_status=Sensor.NOMINAL)
self.add_sensor(self._version)
def request_halt(self, req, msg):
"""Halts the server, writes to the log, and exits the program.
Returns:
success : {'ok', 'fail'} (whether scheduling the halt succeeded).
Examples:
?halt
!halt ok
TODO:
- Call halt method on superclass to avoid copy paste
Doing this caused an issue:
File "/Users/Eric/Berkeley/seti/packages/meerkat/lib/python2.7/site-packages/katcp/server.py", line 1102, in handle_request
assert (reply.mtype == Message.REPLY)
AttributeError: 'NoneType' object has no attribute 'mtype'
"""
f = Future()
@gen.coroutine
def _halt():
req.reply("ok")
yield gen.moment
self.stop(timeout=None)
raise AsyncReply
self.ioloop.add_callback(lambda: chain_future(_halt(), f))
log.critical("HALTING SERVER!!!")
# notify_slack("KATCP server at MeerKAT has halted. Might want to check that!")
sys.exit(0)
@request()
@return_reply(Str())
def request_find_alien(self, req):
"""Finds an alien.
"""
return ("ok", R"""
. . . . . . . . . + .
. . : . .. :. .___---------___.
. . . . :.:. _".^ .^ ^. '.. :"-_. .
. : . . .:../: . .^ :.:\.
. . :: +. :.:/: . . . . . .:\
. : . . _ :::/: . ^ . . .:\
.. . . . - : :.:./. . .:\
. . . :..|: . . ^. .:|
. . : : ..|| . . . !:|
. . . . ::. ::\( . :)/
. . : . : .:.|. ###### .#######::|
:.. . :- : .: ::|.####### ..########:|
. . . .. . .. :\ ######## :######## :/
. .+ :: : -.:\ ######## . ########.:/
. .+ . . . . :.:\. ####### #######..:/
:: . . . . ::.:..:.\ . . ..:/
. . . .. : -::::.\. | | . .:/
. : . . .-:.":.::.\ ..:/
. -. . . . .: .:::.:.\. .:/
. . . : : ....::_:..:\ ___. :/
. . . .:. .. . .: :.:.:\ :/
+ . . : . ::. :.:. .:.|\ .:/|
. + . . ...:: ..| --.:|
. . . . . . . ... :..:.."( ..)"
. . . : . .: ::/ . .::\
""")
class DeviceExampleServer(katcp.DeviceServer):
## Interface version information.
VERSION_INFO = ("Python CASPER packetised correlator server", 0, 1)
## Device server build / instance information.
BUILD_INFO = ("corr", 0, 1, "rc2")
#pylint: disable-msg=R0904
def setup_sensors(self):
pass
def __init__(self, *args, **kwargs):
super(DeviceExampleServer, self).__init__(*args, **kwargs)
self.c = None # correlator object
self.b = None # beamformer object
@request(Int(default=-1))
@return_reply(Int(), Int(), Int())
def request_nb_set_cf(self, sock, freq):
"""Sets the center frequency for narrowband."""
if self.c is None:
return ("fail", "... you haven't connected yet!")
try:
rva, rvb, rvc = corr.corr_nb.channel_select(c=self.c, freq_hz=freq)
return ("ok", rva, rvb, rvc)
except:
return ("fail", "Something broke spectacularly. Check the log.")
@request(Str(default='/etc/corr/default'),
Int(default=100),
include_msg=True)
@return_reply()
def request_connect(self, sock, orgmsg, config_file, log_len):
"""Connect to all the ROACH boards. Please specify the config file and the log length. Clears any existing log. Call this again if you make external changes to the config file to reload it."""
self.lh = corr.log_handlers.DebugLogHandler(log_len)
try:
self.c = corr.corr_functions.Correlator(config_file=config_file,
log_handler=self.lh,
log_level=logging.INFO)
except Exception as err_msg:
return ("fail", err_msg)
try:
self.b = corr.bf_functions.fbf(host_correlator=self.c,
optimisations=True)
except:
self.reply_inform(
sock,
katcp.Message.inform(orgmsg.name, "Beamformer not available"),
orgmsg)
pass
return ("ok", )
@request(include_msg=True)
@return_reply(Int(min=0))
def request_get_rcs(self, sock, orgmsg):
"""Get the revision control information for the system."""
if self.c is None:
return katcp.Message.reply("fail",
"... you haven't connected yet!")
rcs = self.c.get_rcs()
ret_line = []
for e, r in rcs.iteritems():
for k, s in r.iteritems():
ret_line.append('%s:%s:%s' % (e, k, s))
self.reply_inform(
sock, katcp.Message.inform(orgmsg.name, ret_line[-1]),
orgmsg)
return ("ok", len(ret_line))
@request(Int(default=100))
@return_reply()
def request_initialise(self, sock, n_retries):
"""Initialise the correlator. This programs the FPGAs, configures network interfaces etc. Includes error checks. Beamformer mode will update and add relevant functionality and SPEAD metadata. Consult the log in event of errors."""
# First initiate the correlator
if self.c is None:
return ("fail", "... you haven't connected yet!")
try:
self.c.initialise(n_retries)
except:
return ("fail", "Correlator could not initialise. Check the log.")
# Next, if beamformer object is instantiated, initiate the beamformer
if self.b is not None:
try:
time.sleep(1) # allow little time for corr init to close
self.b.initialise()
except:
return ("fail",
"Beamformer could not initialise. Check the log.")
return ("ok", )
@request(include_msg=True)
@return_reply(Int(min=0))
def request_get_log(self, sock, orgmsg):
"""Fetch the log."""
if self.c is None:
return ("fail", "... you haven't connected yet!")
print "\nlog:"
self.lh.printMessages()
for logent in self.c.log_handler._records:
if logent.exc_info:
print '%s: %s Exception: ' % (
logent.name, logent.msg), logent.exc_info[0:-1]
self.reply_inform(
sock,
katcp.Message.inform(
"log",
'%s: %s Exception: ' % (logent.name, logent.msg),
logent.exc_info[0:-1]), orgmsg)
else:
#log error 1234567 basic "the error string"
self.reply_inform(
sock,
katcp.Message.inform(
"log",
logent.levelname.lower()
if logent.levelname.lower() != 'warning' else 'warn',
'%i' % (logent.created * 1000), logent.name,
logent.msg), orgmsg)
#print 'Sending this message:',logent.msg
return ("ok", len(self.c.log_handler._records))
@request()
@return_reply()
def request_clr_log(self, sock):
"""Clears the log."""
if self.c is None:
return ("fail", "... you haven't connected yet!")
self.c.log_handler.clear()
return ("ok", )
@request(Int(), Str())
@return_reply(Str())
def request_label_input(self, sock, input_n, ant_str):
"""Label the inputs. First argument is integer specifying the physical connection. Ordering: first input of first feng, second input of first feng, ... , first input of second feng, second input of second feng, ... , second-last input of last feng, last input of last feng."""
if self.c is None:
return ("fail", "... you haven't connected yet!")
if (input_n < self.c.config['n_inputs']):
self.c.label_input(input_n, ant_str)
# currently the correlator has no concept of the beamformer
# when the correlator label_input has been updated and a beamformer is known, the SPEAD metadata for the beamformer should be re-issued as well
# default is to issue to all beams
if self.b is not None: self.b.spead_labelling_issue()
return ("ok", "Input %i relabelled to %s." % (input_n, ant_str))
else:
#return("fail","it broke.")
return (
"fail",
"Sorry, your input number is invalid. Valid range: 0 to %i." %
(self.c.config['n_inputs'] - 1))
@return_reply(Str(), Str())
def request_bf_destination(self, sock, orgmsg):
"""Set destination for a given stream to a new IP address. The first argument should be the stream name, the second meta/data, the third the IP address in dotted-quad notation. An optional fourth parameters is the port."""
if self.c is None:
return ("fail", "... you haven't connected yet!")
if self.b is None:
return ("fail", "... no beamformer available!")
if len(orgmsg.arguments) < 3:
return ("fail", "... usage: <stream> <meta/data> <ip> [port]")
stream = orgmsg.arguments[0]
identifier = orgmsg.arguments[1]
ip = orgmsg.arguments[2]
if not stream in self.b.get_beams():
return ("fail", "... name %s not a known beam!" % (stream))
if len(ip.split('.')) != 4:
return ("fail", "Not an expected ip address format")
if len(orgmsg.arguments) > 3:
try:
port = int(orgmsg.arguments[3])
except Exception as e:
return ("fail", "... Exception %s" % e)
else:
port = None
if string.lower(identifier) == "meta":
self.b.config_meta_output(beams=stream,
dest_ip_str=ip,
dest_port=port,
issue_spead=False)
if string.lower(identifier) == "data":
self.b.config_udp_output(beams=stream,
dest_ip_str=ip,
dest_port=port,
issue_spead=False)
time.sleep(3)
return ("ok",
"data %s:%i" % (self.b.config['bf_rx_udp_ip_str_beam%d' %
self.b.beam2index(stream)[0]],
self.b.config['bf_rx_udp_port_beam%d' %
self.b.beam2index(stream)[0]]),
"meta %s:%i" % (self.b.config['bf_rx_meta_ip_str_beam%d' %
self.b.beam2index(stream)[0]],
self.b.config['bf_rx_udp_port_beam%d' %
self.b.beam2index(stream)[0]]))
@return_reply(Str(), Str())
def request_tx_start(self, sock, orgmsg):
"""Start transmission to the given IP address and port, or use the defaults from the config file if not specified. The first argument should be the IP address in dotted-quad notation. The second is the port."""
if self.c is None:
return ("fail", "... you haven't connected yet!")
# default assumption will be correlator output
beam = None
dest_ip_str = None
dest_port = None
if len(orgmsg.arguments) > 2:
# beamformer port
try:
dest_port = int(orgmsg.arguments[2])
except Exception as e:
return ("fail", "... %s" % e)
if len(orgmsg.arguments) > 1:
# second argument can be either port of ip
if (len(orgmsg.arguments[1].split('.'))
== 4) and (self.b is not None): # ip address
dest_ip_str = orgmsg.arguments[1]
else:
try:
dest_port = int(orgmsg.arguments[1])
except Exception as e:
return ("fail", "... %s" % e)
if len(orgmsg.arguments) > 0:
# first argument can be either stream name or ip
if len(orgmsg.arguments[0].split('.')) == 4: # ip address
dest_ip_str = orgmsg.arguments[0]
elif (self.b is None):
return ("fail", "... no beamformer available!")
elif (self.b is not None): # stream name = beam name
beam = orgmsg.arguments[0]
if not beam in self.b.get_beams():
self.reply_inform(
sock,
katcp.Message.inform(
orgmsg.name,
"Name %s not a known beam, assuming correlator output"
% (beam)), orgmsg)
beam = None
try:
if (self.b is not None) and (beam is not None):
self.b.config_meta_output(beams=beam,
dest_ip_str=dest_ip_str,
dest_port=dest_port,
issue_spead=False)
self.b.config_udp_output(beams=beam,
dest_ip_str=dest_ip_str,
dest_port=dest_port,
issue_spead=False)
self.b.spead_issue_all(beams=beam, from_fpga=False)
time.sleep(
1) # allow little time for meta data issue to finish
self.b.tx_start(beams=beam)
return ("ok", "data %s:%i" %
(self.b.config['bf_rx_udp_ip_str_beam%d' %
self.b.beam2index(beam)[0]],
self.b.config['bf_rx_udp_port_beam%d' %
self.b.beam2index(beam)[0]]),
"meta %s:%i" %
(self.b.config['bf_rx_meta_ip_str_beam%d' %
self.b.beam2index(beam)[0]],
self.b.config['bf_rx_udp_port_beam%d' %
self.b.beam2index(beam)[0]]))
else:
self.c.config_udp_output(dest_ip_str=dest_ip_str,
dest_port=dest_port)
self.c.spead_issue_all()
self.c.tx_start()
return ("ok", "data %s:%i" % (self.c.config['rx_udp_ip_str'],
self.c.config['rx_udp_port']),
"meta %s:%i" % (self.c.config['rx_meta_ip_str'],
self.c.config['rx_udp_port']))
except corr.bf_functions.fbfException as be:
return ("fail", "... %s" % be.errmsg)
except Exception as e:
return ("fail", "... %s" % e)
# @request()
@return_reply(Str())
def request_spead_issue(self, sock, orgmsg):
"""Issue the SPEAD metadata so that the receiver can interpret the data stream. If a beam name is specified the SPEAD metadata is issued to the requested beam, else it is issued to the correlator stream"""
if self.c is None:
return ("fail", "... you haven't connected yet!")
if len(orgmsg.arguments) > 0:
# issue beamformer spead metadata
beam = orgmsg.arguments[0]
if self.b is None:
return ("fail", "... no beams available!")
try:
self.b.spead_issue_all(beam)
return ("ok", "metadata sent to %s:%i" %
(self.b.config['bf_rx_meta_ip_str_beam%d' %
self.b.beam2index(beam)[0]],
self.b.config['bf_rx_udp_port_beam%d' %
self.b.beam2index(beam)[0]]))
except corr.bf_functions.fbfException as be:
return ("fail", "... %s" % be.errmsg)
except Exception as e:
return (
"fail",
"Couldn't complete the request. Something broke. Check the log."
)
else:
# issue correlator spead metadata
try:
self.c.spead_issue_all()
return ("ok", "metadata sent to %s:%i" %
(self.c.config['rx_meta_ip_str'],
self.c.config['rx_udp_port']))
except:
return ("fail", "Something broke. Check the log.")
@return_reply()
def request_tx_stop(self, sock, orgmsg):
"""Stop transmission to the IP given in the config file."""
if self.c is None:
return ("fail", "... you haven't connected yet!")
if len(orgmsg.arguments) > 0:
if self.b is None:
return (
'fail',
'... stream specification not available in correlator mode'
)
else:
# beamformer tx-stop
stream = orgmsg.arguments[0]
try:
if stream in self.b.get_beams(): # stop beamformer output
self.reply_inform(
sock,
katcp.Message.inform(
orgmsg.name,
"Stop beamformer stream %s" % stream), orgmsg)
self.b.tx_stop(beams=stream)
return ("ok", )
else: # default -- correlator status in beamformer mode
self.reply_inform(
sock,
katcp.Message.inform(
orgmsg.name,
"Stop correlator stream %s" % stream), orgmsg)
self.c.tx_stop()
return ("ok", )
except corr.bf_functions.fbfException as be:
return ("fail", "... %s" % be.errmsg)
except Exception as e:
return ("fail", "... %s" % e)
else:
# correlator tx-stop
self.reply_inform(
sock, katcp.Message.inform(orgmsg.name, "Stop correlator"),
orgmsg)
try:
self.c.tx_stop()
return ("ok", )
except:
return ("fail", "Something broke. Check the log.")
@return_reply(Str())
def request_tx_status(self, sock, orgmsg):
"""Check the TX status. Returns enabled or disabled."""
if self.c is None:
return ("fail", "... you haven't connected yet!")
if len(orgmsg.arguments) > 0:
if self.b is None:
return (
'fail',
'... stream specification not available in correlator mode'
)
else:
# beamformer tx-status
stream = orgmsg.arguments[0]
try:
if stream in self.b.get_beams(): # beamformer status
self.reply_inform(
sock,
katcp.Message.inform(
orgmsg.name,
"Beamformer status for stream %s" % stream),
orgmsg)
if self.b.tx_status_get(stream):
return ("ok", "enabled")
else:
return ("ok", "disabled")
else: # default -- correlator status in beamformer mode
self.reply_inform(
sock,
katcp.Message.inform(
orgmsg.name,
"Correlator status for stream %s" % stream),
orgmsg)
if self.c.tx_status_get(): return ("ok", "enabled")
else: return ("ok", "disabled")
except corr.bf_functions.fbfException as be:
return ("fail", "... %s" % be.errmsg)
except Exception as e:
return (
"fail",
"Couldn't complete the request. Something broke. Check the log."
)
else:
# correlator tx-status
self.reply_inform(
sock, katcp.Message.inform(orgmsg.name, "Correlator status"),
orgmsg)
try:
if self.c.tx_status_get(): return ("ok", "enabled")
else: return ("ok", "disabled")
except:
return (
"fail",
"Couldn't complete the request. Something broke. Check the log."
)
@request(include_msg=True)
def request_check_sys(self, sock, orgmsg):
"""Checks system health. Returns health tree informs for each engine in the system."""
if self.c is None:
return katcp.Message.reply(orgmsg.name, "fail",
"... you haven't connected yet!")
try:
stat = self.c.check_all(details=True)
for l, v in stat.iteritems():
ret_line = []
for k, s in v.iteritems():
ret_line.append('%s:%s' % (k, s))
self.reply_inform(
sock, katcp.Message.inform(orgmsg.name, str(l), *ret_line),
orgmsg)
return katcp.Message.reply(orgmsg.name, "ok", len(stat))
except:
return katcp.Message.reply(
orgmsg.name, "fail",
"Something broke spectacularly and the check didn't complete. Scrutinise the log."
)
@request()
@return_reply(Int(min=0))
def request_resync(self, sock):
"""Rearms the system. Returns the time at which the system was synch'd in ms since unix epoch."""
if self.c is None:
return ("fail", "... you haven't connected yet!")
try:
time = self.c.arm()
# currently the correlator has no concept of the beamformer
# when the correlator label_input has been updated and a beamformer is known, the SPEAD metadata for the beamformer should be re-issued as well
# default is to issue to all beams
if self.b is not None: self.b.spead_time_meta_issue()
return ("ok", (time * 1000))
except:
return ("fail", -1)
def request_get_adc_snapshots(self, sock, orgmsg):
"""Grabs a snapshot of data from the specified antennas.
\n@Param integer Sync to 1PPS (ie align all snapshots). Note that this could cost a bit of time as we wait for the next 1PPS.
\n@Param integer Wait for ADC level of trigger_level to capture transients.
\n@Params list of antenna strings.
\n@reply str antenna name.
\n@reply int timestamp (unix seconds since epoch) of first sample.
\n@reply int n_samples captured since trigger."""
if self.c is None:
return katcp.Message.reply(orgmsg.name, "fail",
"... you haven't connected yet!")
if len(orgmsg.arguments) < 3:
return katcp.Message.reply(
orgmsg.name, "fail",
"... you didn't specify enough arguments.")
try:
sync_to_pps = bool(int(orgmsg.arguments[0]))
trig_level = int(orgmsg.arguments[1])
ant_strs = orgmsg.arguments[2:]
for ant_str in ant_strs:
if not ant_str in self.c.config._get_ant_mapping_list():
return katcp.Message.reply(
orgmsg.name, "fail",
"Antenna not found. Valid entries are %s." %
str(self.c.config._get_ant_mapping_list()))
snap_data = self.c.get_adc_snapshots(ant_strs,
trig_level=trig_level,
sync_to_pps=sync_to_pps)
for ant_str, data in snap_data.iteritems():
self.reply_inform(
sock,
katcp.Message.inform(orgmsg.name, ant_str,
str(data['timestamp'] * 1000),
str(data['offset']), *data['data']),
orgmsg)
return katcp.Message.reply(orgmsg.name, 'ok', str(len(snap_data)))
except:
return katcp.Message.reply(orgmsg.name, "fail",
"something broke. sorry.")
@request(Str(), include_msg=True)
def request_get_adc_snapshot(self, sock, orgmsg, ant_str):
"""Grabs a snapshot of data from the antenna specified."""
if self.c is None:
return katcp.Message.reply(orgmsg.name, "fail",
"... you haven't connected yet!")
try:
if not ant_str in self.c.config._get_ant_mapping_list():
return katcp.Message.reply(
orgmsg.name, "fail",
"Antenna not found. Valid entries are %s." %
str(self.c.config._get_ant_mapping_list()))
unpackedBytes = self.c.get_adc_snapshots([ant_str
])[ant_str]['data']
return katcp.Message.reply(orgmsg.name, 'ok', *unpackedBytes)
#return katcp.Message.reply(orgmsg.name,'ok','Awaiting rewrite!')
except:
return katcp.Message.reply(orgmsg.name, 'fail',
"something broke. oops.")
@request(Str(), Int(default=1), include_msg=True)
def request_get_quant_snapshot(self, sock, orgmsg, ant_str, n_spectra):
"""Grabs a snapshot of data from the quantiser for antenna specified. Optional: number of spectra to grab (default 1)."""
#print 'Trying to get %i spectra.'%n_spectra
if self.c is None:
return katcp.Message.reply(orgmsg.name, "fail",
"... you haven't connected yet!")
try:
if not (ant_str in self.c.config._get_ant_mapping_list()):
return katcp.Message.reply(
orgmsg.name, "fail",
"Antenna not found. Valid entries are: %s" %
str(self.c.config._get_ant_mapping_list()))
unpackedBytes = self.c.get_quant_snapshot(ant_str, n_spectra)
print 'N spectra: %i.' % n_spectra
print unpackedBytes
if n_spectra == 1:
self.reply_inform(
sock,
katcp.Message.inform(
orgmsg.name,
*([
'%i+%ij' % (val.real, val.imag)
for val in unpackedBytes[0]
])), orgmsg)
elif n_spectra > 1:
for s_n, spectrum in enumerate(unpackedBytes):
#print 'Sending inform %i:'%s,unpackedBytes[s]
print 'trying to send the array:', [
'%i+%ij' % (val.real, val.imag)
for val in unpackedBytes[0][s_n]
]
self.reply_inform(
sock,
katcp.Message.inform(
orgmsg.name,
*([
'%i+%ij' % (val.real, val.imag)
for val in unpackedBytes[0][s_n]
])), orgmsg)
else:
raise RuntimeError(
"Please specify the number of spectra to be greater than zero!"
)
return katcp.Message.reply(orgmsg.name, 'ok', str(n_spectra))
except:
return katcp.Message.reply(orgmsg.name, 'fail',
"something broke. darn.")
@request(Float(min=0))
@return_reply(Float())
def request_acc_time(self, sock, acc_time):
"""Set the accumulation time in seconds (float)."""
if self.c is None:
return ("fail", "... you haven't connected yet!")
try:
running = self.c.tx_status_get()
if running:
self.c.tx_stop()
act_period = self.c.acc_time_set(acc_time)
print 'Set act time to %f' % act_period
if running:
self.c.tx_start()
return ("ok", act_period)
except:
return (
"fail",
"Something broke spectacularly and the request didn't complete. Scrutinise the log."
)
@request(include_msg=True)
@return_reply(Int())
def request_get_input_levs(self, sock, orgmsg):
"""Get the current RF input levels to the DBE in dBm."""
if self.c is None:
return ("fail", "... you haven't connected yet!")
amps = self.c.adc_amplitudes_get()
for ant_str, ampl in amps.iteritems():
# rf_level=amps[i]['rms_dbm'] - self.c.rf_status_get(i)[1]
if self.c.feng_status_get(ant_str)['adc_disabled'] == True:
stat = 'disabled'
elif ampl['low_level_warn'] == True:
stat = 'low'
elif ampl['high_level_warn'] == True:
stat = 'high'
else:
stat = 'ok'
self.reply_inform(
sock,
katcp.Message.inform(orgmsg.name, ant_str,
"%2.2f" % ampl['input_rms_dbm'], stat),
orgmsg)
return ("ok", len(amps))
@request(include_msg=True)
@return_reply(Int())
def request_get_ant_status(self, sock, orgmsg):
"""Decode and report the status of all connected F engines. This will automatically clear the registers after the readback."""
if self.c is None:
return ("fail", "... you haven't connected yet!")
fstat = self.c.feng_status_get_all()
self.c.rst_fstatus()
for i in fstat:
out_str = []
for ent in fstat[i]:
out_str.append(str(ent))
out_str.append(str(fstat[i][ent]))
self.reply_inform(sock,
katcp.Message.inform(orgmsg.name, i, *out_str),
orgmsg)
return ("ok", len(fstat))
@request(Str(), include_msg=True)
def request_eq_get(self, sock, orgmsg, ant_str):
"""Get the current EQ configuration."""
if self.c is None:
return katcp.Message.reply(orgmsg.name, "fail",
"... you haven't connected yet!")
if not ant_str in self.c.config._get_ant_mapping_list():
return katcp.Message.reply(
orgmsg.name, "fail",
"Antenna not found. Valid entries are %s." %
str(self.c.config._get_ant_mapping_list()))
eq = self.c.eq_spectrum_get(ant_str)
return katcp.Message.reply(orgmsg.name, 'ok', *eq)
def request_eq_set(self, sock, orgmsg):
"""Set the current EQ configuration for a given antenna. ?eq-set 0x 1123+456j 555+666j 987+765j..."""
if self.c is None:
return katcp.Message.reply(orgmsg.name, "fail",
"... you haven't connected yet!")
ant_str = orgmsg.arguments[0]
if not ant_str in self.c.config._get_ant_mapping_list():
return katcp.Message.reply(
orgmsg.name, "fail",
"Antenna not found. Valid entries are %s." %
str(self.c.config._get_ant_mapping_list()))
eq_coeffs = []
if len(
orgmsg.arguments
) == 2: #+1 to account for antenna label, assume single number across entire band
self.c.eq_spectrum_set(ant_str,
init_poly=[eval(orgmsg.arguments[1])])
return katcp.Message.reply(
orgmsg.name, 'ok',
"Set all coefficients to %i." % eval(orgmsg.arguments[1]))
elif len(orgmsg.arguments) != (self.c.config['n_chans'] +
1): #+1 to account for antenna label
return katcp.Message.reply(
orgmsg.name, "fail",
"Sorry, you didn't specify the right number of coefficients (expecting %i, got %i)."
% (self.c.config['n_chans'], len(orgmsg.arguments) - 1))
else:
for arg in orgmsg.arguments[1:]:
eq_coeffs.append(eval(arg))
self.c.eq_spectrum_set(ant_str, init_coeffs=eq_coeffs)
return katcp.Message.reply(orgmsg.name, 'ok')
def request_fr_delay_set(self, sock, orgmsg):
"""Set the fringe rate and delay compensation config for a given antenna. Parms: antpol, fringe_offset (degrees), fr_rate (Hz), delay (seconds), delay rate, load_time (Unix seconds), <ignore check>. If there is a seventh argument, don't do any checks to see if things loaded properly. If the load time is negative, load asap."""
if self.c is None:
return katcp.Message.reply(orgmsg.name, "fail",
"... you haven't connected yet!")
ant_str = orgmsg.arguments[0]
if not ant_str in self.c.config._get_ant_mapping_list():
return katcp.Message.reply(
orgmsg.name, "fail",
"Antenna not found. Valid entries are %s." %
str(self.c.config._get_ant_mapping_list()))
fr_offset = float(orgmsg.arguments[1])
fr_rate = float(orgmsg.arguments[2])
delay = float(orgmsg.arguments[3])
del_rate = float(orgmsg.arguments[4])
ld_time = float(orgmsg.arguments[5])
if len(orgmsg.arguments) > 6:
ld_check = False
# print 'Ignoring load check.'
else:
ld_check = True
# print 'Check for correct load.'
stat = self.c.fr_delay_set(ant_str,
fringe_phase=fr_offset,
fringe_rate=fr_rate,
delay=delay,
delay_rate=del_rate,
ld_time=ld_time,
ld_check=ld_check)
out_str = []
for ent in stat:
out_str.append(str(ent))
out_str.append("%12.10e" % (stat[ent]))
return katcp.Message.reply(orgmsg.name, 'ok', *out_str)
@request(Str(), Float(default=-1), Float(default=-1), include_msg=True)
@return_reply(Float(), Float())
def request_beam_passband(self, sock, orgmsg, beam, bw, cf):
"""Setup of beamformer output passband. Please specify a beam name to return the current bandwidth and centre frequency in Hz. Alternatively, specify a beam name, bandwidth and centre frequency in Hz to set the beamformer output passband. The closest actual bandwidth and centre frequency achievable will be returned."""
if self.b is None:
return ("fail", "... beamformer functionality only!")
if bw >= 0 and cf >= 0:
try:
self.b.set_passband(beams=beam,
bandwidth=bw,
centre_frequency=cf)
except corr.bf_functions.fbfException as be:
return ("fail", "... %s" % be.errmsg)
except Exception as e:
return ("fail", "... %s" % e)
elif (bw * cf < 0):
return (
"fail",
"... require both bandwidth and center frequency to be specified"
)
try:
cf, bw = self.b.get_passband(beam=beam)
return ("ok", bw, cf)
except:
return ("fail", "... unknown beam name %s" % orgmsg.arguments[0])
@request(Str(), Str(), include_msg=True)
def request_weights_get(self, sock, orgmsg, beam, ant_str):
"""Get the current beamformer weights. Params: beam, ant_str"""
if self.c is None or self.b is None:
return katcp.Message.reply(orgmsg.name, "fail",
"... you haven't connected yet!")
if not ant_str in self.c.config._get_ant_mapping_list():
return katcp.Message.reply(
orgmsg.name, "fail",
"Antenna not found. Valid entries are %s." %
str(self.c.config._get_ant_mapping_list()))
if not beam in self.b.get_beams():
return katcp.Message.reply(
orgmsg.name, "fail",
"Unknown beam name. Valid entries are %s." %
str(self.b.get_beams()))
try:
weights = self.b.cal_spectrum_get(beam=beam, ant_str=ant_str)
return katcp.Message.reply(orgmsg.name, 'ok', *weights)
except corr.bf_functions.fbfException as be:
return ("fail", "... %s" % be.errmsg)
except Exception as e:
return ("fail", "... %s" % e)
def request_weights_set(self, sock, orgmsg):
"""Set the weights for an input to a selected beam. ?weights-set bf0 0x 1123+456j 555+666j 987+765j..."""
if self.c is None or self.b is None:
return katcp.Message.reply(orgmsg.name, "fail",
"... you haven't connected yet!")
beam = orgmsg.arguments[0]
if not beam in self.b.get_beams():
return katcp.Message.reply(
orgmsg.name, "fail",
"Unknown beam name. Valid entries are %s." %
str(self.b.get_beams()))
ant_str = orgmsg.arguments[1]
if not ant_str in self.c.config._get_ant_mapping_list():
return katcp.Message.reply(
orgmsg.name, "fail",
"Antenna not found. Valid entries are %s." %
str(self.c.config._get_ant_mapping_list()))
try:
bw_coeffs = []
if len(
orgmsg.arguments
) == 3: #+2 to account for beam name and antenna label, assume single number across entire band
self.b.cal_spectrum_set(beam=beam,
ant_str=ant_str,
init_poly=[eval(orgmsg.arguments[2])])
return katcp.Message.reply(orgmsg.name, 'ok',
"Set all coefficients to",
eval(orgmsg.arguments[2]))
elif len(orgmsg.arguments) != (
self.c.config['n_chans'] +
2): #+2 to account for beam name and antenna label
return katcp.Message.reply(
orgmsg.name, "fail",
"Sorry, you didn't specify the right number of coefficients (expecting %i, got %i)."
% (self.c.config['n_chans'], len(orgmsg.arguments) - 2))
else:
for arg in orgmsg.arguments[2:]:
bw_coeffs.append(eval(arg))
self.b.cal_spectrum_set(beam=beam,
ant_str=ant_str,
init_coeffs=bw_coeffs)
return katcp.Message.reply(orgmsg.name, 'ok')
except corr.bf_functions.fbfException as be:
return ("fail", "... %s" % be.errmsg)
except Exception as e:
return ("fail", "... %s" % e)
class MasterController(AsyncDeviceServer):
"""This is the main KATCP interface for the FBFUSE
multi-beam beamformer on MeerKAT.
This interface satisfies the following ICDs:
CAM-FBFUSE: <link>
TUSE-FBFUSE: <link>
"""
VERSION_INFO = ("mpikat-api", 0, 1)
BUILD_INFO = ("mpikat-implementation", 0, 1, "rc1")
DEVICE_STATUSES = ["ok", "degraded", "fail"]
def __init__(self, ip, port, worker_pool):
"""
@brief Construct new MasterController instance
@params ip The IP address on which the server should listen
@params port The port that the server should bind to
"""
super(MasterController, self).__init__(ip, port)
self._products = {}
self._server_pool = worker_pool
def start(self):
"""
@brief Start the MasterController server
"""
super(MasterController, self).start()
def stop(self):
self._ntp_callback.stop()
super(MasterController, self).stop()
def add_sensor(self, sensor):
log.debug("Adding sensor: {}".format(sensor.name))
super(MasterController, self).add_sensor(sensor)
def remove_sensor(self, sensor):
log.debug("Removing sensor: {}".format(sensor.name))
super(MasterController, self).remove_sensor(sensor)
def setup_sensors(self):
"""
@brief Set up monitoring sensors.
@note The following sensors are made available on top of default sensors
implemented in AsynDeviceServer and its base classes.
device-status: Reports the health status of the controller and associated devices:
Among other things report HW failure, SW failure and observation failure.
local-time-synced: Indicates whether the local time of the servers
is synchronised to the master time reference (use NTP).
This sensor is aggregated from all nodes that are part
of FBF and will return "not sync'd" if any nodes are
unsyncronised.
products: The list of product_ids that controller is currently handling
"""
self._device_status = Sensor.discrete(
"device-status",
description="Health status of FBFUSE",
params=self.DEVICE_STATUSES,
default="ok",
initial_status=Sensor.NOMINAL)
self.add_sensor(self._device_status)
self._local_time_synced = Sensor.boolean(
"local-time-synced",
description="Indicates FBF is NTP syncronised.",
default=True,
initial_status=Sensor.UNKNOWN)
self.add_sensor(self._local_time_synced)
def ntp_callback():
log.debug("Checking NTP sync")
try:
synced = check_ntp_sync()
except Exception:
log.exception("Unable to check NTP sync")
self._local_time_synced.set_value(False)
else:
if not synced:
log.warning("Server is not NTP synced")
self._local_time_synced.set_value(synced)
ntp_callback()
self._ntp_callback = PeriodicCallback(ntp_callback,
NTP_CALLBACK_PERIOD)
self._ntp_callback.start()
self._products_sensor = Sensor.string(
"products",
description="The names of the currently configured products",
default="",
initial_status=Sensor.NOMINAL)
self.add_sensor(self._products_sensor)
def _update_products_sensor(self):
self._products_sensor.set_value(",".join(self._products.keys()))
def _get_product(self, product_id):
if product_id not in self._products:
raise ProductLookupError(
"No product configured with ID: {}".format(product_id))
else:
return self._products[product_id]
@request(Str(), Int())
@return_reply()
def request_register_worker_server(self, req, hostname, port):
"""
@brief Register an WorkerWrapper instance
@params hostname The hostname for the worker server
@params port The port number that the worker server serves on
@detail Register an WorkerWrapper instance that can be used for FBFUSE
computation. FBFUSE has no preference for the order in which control
servers are allocated to a subarray. An WorkerWrapper wraps an atomic
unit of compute comprised of one CPU, one GPU and one NIC (i.e. one NUMA
node on an FBFUSE compute server).
"""
log.debug("Received request to register worker server at {}:{}".format(
hostname, port))
self._server_pool.add(hostname, port)
return ("ok", )
@request(Str(), Int())
@return_reply()
def request_deregister_worker_server(self, req, hostname, port):
"""
@brief Deregister an WorkerWrapper instance
@params hostname The hostname for the worker server
@params port The port number that the worker server serves on
@detail The graceful way of removing a server from rotation. If the server is
currently actively processing an exception will be raised.
"""
log.debug(
"Received request to deregister worker server at {}:{}".format(
hostname, port))
try:
self._server_pool.remove(hostname, port)
except ServerDeallocationError as error:
log.error(
"Request to deregister worker server at {}:{} failed with error: {}"
.format(hostname, port, str(error)))
return ("fail", str(error))
else:
return ("ok", )
@request()
@return_reply(Int())
def request_worker_server_list(self, req):
"""
@brief List all control servers and provide minimal metadata
"""
for server in self._server_pool.used():
req.inform("{} allocated".format(server))
for server in self._server_pool.available():
req.inform("{} free".format(server))
return ("ok", len(self._server_pool.used()) +
len(self._server_pool.available()))
@request()
@return_reply(Int())
def request_product_list(self, req):
"""
@brief List all currently registered products and their states
@param req A katcp request object
@note The details of each product are provided via an #inform
as a JSON string containing information on the product state.
@return katcp reply object [[[ !product-list ok | (fail [error description]) <number of configured products> ]]],
"""
for product_id, product in self._products.items():
info = {}
info[product_id] = product.info()
as_json = json.dumps(info)
req.inform(as_json)
return ("ok", len(self._products))
def test_return_reply_multi(self):
with self.assertRaises(TypeError) as ex:
return_reply(Bool(multiple=True), Int())
self.assertEqual(
ex.exception.message,
'Only the last parameter type can accept multiple arguments.')
class FbfMasterController(MasterController):
"""This is the main KATCP interface for the FBFUSE
multi-beam beamformer on MeerKAT.
This interface satisfies the following ICDs:
CAM-FBFUSE: <link>
TUSE-FBFUSE: <link>
"""
VERSION_INFO = ("mpikat-fbf-api", 0, 1)
BUILD_INFO = ("mpikat-fbf-implementation", 0, 1, "rc1")
DEVICE_STATUSES = ["ok", "degraded", "fail"]
def __init__(self, ip, port, dummy=True,
ip_range = FBF_IP_RANGE):
"""
@brief Construct new FbfMasterController instance
@params ip The IP address on which the server should listen
@params port The port that the server should bind to
@params dummy Specifies if the instance is running in a dummy mode
@note In dummy mode, the controller will act as a mock interface only, sending no requests to nodes.
A valid node pool must still be provided to the instance, but this may point to non-existent nodes.
"""
self._ip_pool = IpRangeManager(ip_range_from_stream(ip_range))
super(FbfMasterController, self).__init__(ip, port, FbfWorkerPool())
self._dummy = dummy
if self._dummy:
for ii in range(64):
self._server_pool.add("127.0.0.1", 50000+ii)
def setup_sensors(self):
"""
@brief Set up monitoring sensors.
"""
super(FbfMasterController, self).setup_sensors()
self._ip_pool_sensor = Sensor.string(
"output-ip-range",
description="The multicast address allocation for coherent beams",
default=self._ip_pool.format_katcp(),
initial_status=Sensor.NOMINAL)
self.add_sensor(self._ip_pool_sensor)
@request(Str(), Str(), Int(), Str(), Str())
@return_reply()
def request_configure(self, req, product_id, antennas_csv, n_channels, streams_json, proxy_name):
"""
@brief Configure FBFUSE to receive and process data from a subarray
@detail REQUEST ?configure product_id antennas_csv n_channels streams_json proxy_name
Configure FBFUSE for the particular data products
@param req A katcp request object
@param product_id This is a name for the data product, which is a useful tag to include
in the data, but should not be analysed further. For example "array_1_bc856M4k".
@param antennas_csv A comma separated list of physical antenna names used in particular sub-array
to which the data products belongs (e.g. m007,m008,m009).
@param n_channels The integer number of frequency channels provided by the CBF.
@param streams_json a JSON struct containing config keys and values describing the streams.
For example:
@code
{'stream_type1': {
'stream_name1': 'stream_address1',
'stream_name2': 'stream_address2',
...},
'stream_type2': {
'stream_name1': 'stream_address1',
'stream_name2': 'stream_address2',
...},
...}
@endcode
The steam type keys indicate the source of the data and the type, e.g. cam.http.
stream_address will be a URI. For SPEAD streams, the format will be spead://<ip>[+<count>]:<port>,
representing SPEAD stream multicast groups. When a single logical stream requires too much bandwidth
to accommodate as a single multicast group, the count parameter indicates the number of additional
consecutively numbered multicast group ip addresses, and sharing the same UDP port number.
stream_name is the name used to identify the stream in CAM.
A Python example is shown below, for five streams:
One CAM stream, with type cam.http. The camdata stream provides the connection string for katportalclient
(for the subarray that this FBFUSE instance is being configured on).
One F-engine stream, with type: cbf.antenna_channelised_voltage.
One X-engine stream, with type: cbf.baseline_correlation_products.
Two beam streams, with type: cbf.tied_array_channelised_voltage. The stream names ending in x are
horizontally polarised, and those ending in y are vertically polarised.
@code
pprint(streams_dict)
{'cam.http':
{'camdata':'http://10.8.67.235/api/client/1'},
'cbf.antenna_channelised_voltage':
{'i0.antenna-channelised-voltage':'spead://239.2.1.150+15:7148'},
...}
@endcode
If using katportalclient to get information from CAM, then reconnect and re-subscribe to all sensors
of interest at this time.
@param proxy_name The CAM name for the instance of the FBFUSE data proxy that is being configured.
For example, "FBFUSE_3". This can be used to query sensors on the correct proxy,
in the event that there are multiple instances in the same subarray.
@note A configure call will result in the generation of a new subarray instance in FBFUSE that will be added to the clients list.
@return katcp reply object [[[ !configure ok | (fail [error description]) ]]]
"""
msg = ("Configuring new FBFUSE product",
"Product ID: {}".format(product_id),
"Antennas: {}".format(antennas_csv),
"Nchannels: {}".format(n_channels),
"Streams: {}".format(streams_json),
"Proxy name: {}".format(proxy_name))
log.info("\n".join(msg))
# Test if product_id already exists
if product_id in self._products:
return ("fail", "FBF already has a configured product with ID: {}".format(product_id))
# Determine number of nodes required based on number of antennas in subarray
# Note this is a poor way of handling this that may be updated later. In theory
# there is a throughput measure as a function of bandwidth, polarisations and number
# of antennas that allows one to determine the number of nodes to run. Currently we
# just assume one antennas worth of data per NIC on our servers, so two antennas per
# node.
try:
antennas = parse_csv_antennas(antennas_csv)
except AntennaValidationError as error:
return ("fail", str(error))
valid_n_channels = [1024, 4096, 32768]
if not n_channels in valid_n_channels:
return ("fail", "The provided number of channels ({}) is not valid. Valid options are {}".format(n_channels, valid_n_channels))
streams = json.loads(streams_json)
try:
streams['cam.http']['camdata']
# Need to check for endswith('.antenna-channelised-voltage') as the i0 is not
# guaranteed to stay the same.
# i0 = instrument name
# Need to keep this for future sensor lookups
streams['cbf.antenna_channelised_voltage']
except KeyError as error:
return ("fail", "JSON streams object does not contain required key: {}".format(str(error)))
for key, value in streams['cbf.antenna_channelised_voltage'].items():
if key.endswith('.antenna-channelised-voltage'):
instrument_name, _ = key.split('.')
feng_stream_name = key
feng_groups = value
log.debug("Parsed instrument name from streams: {}".format(instrument_name))
break
else:
return ("fail", "Could not determine instrument name (e.g. 'i0') from streams")
# TODO: change this request to @async_reply and make the whole thing a coroutine
@coroutine
def configure():
kpc = self._katportal_wrapper_type(streams['cam.http']['camdata'])
# Get all antenna observer strings
futures, observers = [],[]
for antenna in antennas:
log.debug("Fetching katpoint string for antenna {}".format(antenna))
futures.append(kpc.get_observer_string(antenna))
for ii,future in enumerate(futures):
try:
observer = yield future
except Exception as error:
log.error("Error on katportalclient call: {}".format(str(error)))
req.reply("fail", "Error retrieving katpoint string for antenna {}".format(antennas[ii]))
return
else:
log.debug("Fetched katpoint antenna: {}".format(observer))
observers.append(Antenna(observer))
# Get bandwidth, cfreq, sideband, f-eng mapping
#TODO: Also get sync-epoch
log.debug("Fetching F-engine and subarray configuration information")
bandwidth_future = kpc.get_bandwidth(feng_stream_name)
cfreq_future = kpc.get_cfreq(feng_stream_name)
sideband_future = kpc.get_sideband(feng_stream_name)
feng_antenna_map_future = kpc.get_antenna_feng_id_map(instrument_name, antennas)
sync_epoch_future = kpc.get_sync_epoch()
bandwidth = yield bandwidth_future
cfreq = yield cfreq_future
sideband = yield sideband_future
feng_antenna_map = yield feng_antenna_map_future
sync_epoch = yield sync_epoch_future
feng_config = {
'bandwidth': bandwidth,
'centre-frequency': cfreq,
'sideband': sideband,
'feng-antenna-map': feng_antenna_map,
'sync-epoch': sync_epoch,
'nchans': n_channels
}
for key, value in feng_config.items():
log.debug("{}: {}".format(key, value))
product = FbfProductController(self, product_id, observers, n_channels,
feng_groups, proxy_name, feng_config)
self._products[product_id] = product
self._update_products_sensor()
req.reply("ok",)
log.debug("Configured FBFUSE instance with ID: {}".format(product_id))
self.ioloop.add_callback(configure)
raise AsyncReply
@request(Str())
@return_reply()
def request_deconfigure(self, req, product_id):
"""
@brief Deconfigure the FBFUSE instance.
@note Deconfigure the FBFUSE instance. If FBFUSE uses katportalclient to get information
from CAM, then it should disconnect at this time.
@param req A katcp request object
@param product_id This is a name for the data product, used to track which subarray is being deconfigured.
For example "array_1_bc856M4k".
@return katcp reply object [[[ !deconfigure ok | (fail [error description]) ]]]
"""
log.info("Deconfiguring FBFUSE instace with ID '{}'".format(product_id))
# Test if product exists
try:
product = self._get_product(product_id)
except ProductLookupError as error:
return ("fail", str(error))
try:
product.deconfigure()
except Exception as error:
return ("fail", str(error))
del self._products[product_id]
self._update_products_sensor()
return ("ok",)
@request(Str(), Str())
@return_reply()
@coroutine
def request_target_start(self, req, product_id, target):
"""
@brief Notify FBFUSE that a new target is being observed
@param product_id This is a name for the data product, used to track which subarray is being deconfigured.
For example "array_1_bc856M4k".
@param target A KATPOINT target string
@return katcp reply object [[[ !target-start ok | (fail [error description]) ]]]
"""
log.info("Received new target: {}".format(target))
try:
product = self._get_product(product_id)
except ProductLookupError as error:
raise Return(("fail", str(error)))
try:
target = Target(target)
except Exception as error:
raise Return(("fail", str(error)))
yield product.target_start(target)
raise Return(("ok",))
# DELETE this
@request(Str())
@return_reply()
@coroutine
def request_target_stop(self, req, product_id):
"""
@brief Notify FBFUSE that the telescope has stopped observing a target
@param product_id This is a name for the data product, used to track which subarray is being deconfigured.
For example "array_1_bc856M4k".
@return katcp reply object [[[ !target-start ok | (fail [error description]) ]]]
"""
try:
product = self._get_product(product_id)
except ProductLookupError as error:
raise Return(("fail", str(error)))
yield product.target_stop()
raise Return(("ok",))
@request(Str())
@return_reply()
def request_capture_init(self, req, product_id):
"""NOOP"""
return ("ok",)
@request(Str())
@return_reply()
def request_capture_done(self, req, product_id):
"""NOOP"""
return ("ok",)
@request(Str())
@return_reply()
def request_capture_start(self, req, product_id):
"""
@brief Request that FBFUSE start beams streaming
@detail Upon this call the provided coherent and incoherent beam configurations will be evaluated
to determine if they are physical and can be met with the existing hardware. If the configurations
are acceptable then servers allocated to this instance will be triggered to begin production of beams.
@param req A katcp request object
@param product_id This is a name for the data product, used to track which subarray is being deconfigured.
For example "array_1_bc856M4k".
@return katcp reply object [[[ !start-beams ok | (fail [error description]) ]]]
"""
try:
product = self._get_product(product_id)
except ProductLookupError as error:
return ("fail", str(error))
@coroutine
def start():
try:
product.capture_start()
except Exception as error:
req.reply("fail", str(error))
else:
req.reply("ok",)
self.ioloop.add_callback(start)
raise AsyncReply
@request(Str(), Str())
@return_reply()
def request_provision_beams(self, req, product_id, sb_id):
"""
@brief Request that FBFUSE asynchronously prepare to start beams streaming
@detail Upon this call the provided coherent and incoherent beam configurations will be evaluated
to determine if they are physical and can be met with the existing hardware. If the configurations
are acceptable then servers allocated to this instance will be triggered to prepare for the production of beams.
Unlike a call to ?capture-start, ?provision-beams will not trigger a connection to multicast groups and will not
wait for completion before returning, instead it will start the process of beamformer resource alloction and compilation.
To determine when the process is complete, the user must wait on the value of the product "state" sensor becoming "ready",
e.g.
@code
client.sensor['{}-state'.format(proxy_name)].wait(
lambda reading: reading.value == 'ready')
@endcode
@param req A katcp request object
@param product_id This is a name for the data product, used to track which subarray is being deconfigured.
For example "array_1_bc856M4k".
@param sb_id Schedule block ID for the commencing schedule block
@return katcp reply object [[[ !start-beams ok | (fail [error description]) ]]]
"""
# Note: the state of the product won't be updated until the start call hits the top of the
# event loop. It may be preferable to keep a self.starting_future object and yield on it
# in capture-start if it exists. The current implementation may or may not be a bug...
try:
product = self._get_product(product_id)
except ProductLookupError as error:
return ("fail", str(error))
# This check needs to happen here as this call
# should return immediately
if not product.idle:
return ("fail", "Can only provision beams on an idle FBF product")
self.ioloop.add_callback(lambda : product.prepare(sb_id))
return ("ok",)
@request(Str())
@return_reply()
def request_capture_stop(self, req, product_id):
"""
@brief Stop FBFUSE streaming
@param product_id This is a name for the data product, used to track which subarray is being deconfigured.
For example "array_1_bc856M4k".
"""
try:
product = self._get_product(product_id)
except ProductLookupError as error:
return ("fail", str(error))
@coroutine
def stop():
product.capture_stop()
req.reply("ok",)
self.ioloop.add_callback(stop)
raise AsyncReply
@request(Str(), Str(), Int())
@return_reply()
def request_set_configuration_authority(self, req, product_id, hostname, port):
"""
@brief Set the configuration authority for an FBF product
@detail The parameters passed here specify the address of a server that
can be triggered to provide FBFUSE with configuration information
at schedule block and target boundaries. The configuration authority
must be a valid KATCP server.
"""
try:
product = self._get_product(product_id)
except ProductLookupError as error:
return ("fail", str(error))
product.set_configuration_authority(hostname, port)
return ("ok",)
@request(Str())
@return_reply()
def request_reset_beams(self, req, product_id):
"""
@brief Reset the positions of all allocated beams
@note This call may only be made AFTER a successful call to start-beams. Before this point no beams are
allocated to the instance. If all beams are currently allocated an exception will be raised.
@param req A katcp request object
@param product_id This is a name for the data product, used to track which subarray is being deconfigured.
For example "array_1_bc856M4k".
@return katcp reply object [[[ !reset-beams m ok | (fail [error description]) ]]]
"""
try:
product = self._get_product(product_id)
except ProductLookupError as error:
return ("fail", str(error))
else:
beam = product.reset_beams()
return ("ok", )
@request(Str(), Str())
@return_reply(Str())
def request_add_beam(self, req, product_id, target):
"""
@brief Configure the parameters of one beam
@note This call may only be made AFTER a successful call to start-beams. Before this point no beams are
allocated to the instance. If all beams are currently allocated an exception will be raised.
@param req A katcp request object
@param product_id This is a name for the data product, used to track which subarray is being deconfigured.
For example "array_1_bc856M4k".
@param target A KATPOINT target string
@return katcp reply object [[[ !add-beam ok | (fail [error description]) ]]]
"""
try:
product = self._get_product(product_id)
except ProductLookupError as error:
return ("fail", str(error))
try:
target = Target(target)
except Exception as error:
return ("fail", str(error))
beam = product.add_beam(target)
return ("ok", beam.idx)
@request(Str(), Str(), Int(), Float(), Float(), Float())
@return_reply(Str())
def request_add_tiling(self, req, product_id, target, nbeams, reference_frequency, overlap, epoch):
"""
@brief Configure the parameters of a static beam tiling
@note This call may only be made AFTER a successful call to start-beams. Before this point no beams are
allocated to the instance. If there are not enough free beams to satisfy the request an
exception will be raised.
@note Beam shapes calculated for tiling are always assumed to be 2D elliptical Gaussians.
@param req A katcp request object
@param product_id This is a name for the data product, used to track which subarray is being deconfigured.
For example "array_1_bc856M4k".
@param target A KATPOINT target string
@param nbeams The number of beams in this tiling pattern.
@param reference_frequency The reference frequency at which to calculate the synthesised beam shape,
and thus the tiling pattern. Typically this would be chosen to be the
centre frequency of the current observation.
@param overlap The desired overlap point between beams in the pattern. The overlap defines
at what power point neighbouring beams in the tiling pattern will meet. For
example an overlap point of 0.1 corresponds to beams overlapping only at their
10%-power points. Similarly a overlap of 0.5 corresponds to beams overlapping
at their half-power points. [Note: This is currently a tricky parameter to use
when values are close to zero. In future this may be define in sigma units or
in multiples of the FWHM of the beam.]
@param epoch The desired epoch for the tiling pattern as a unix time. A typical usage would
be to set the epoch to half way into the coming observation in order to minimise
the effect of parallactic angle and array projection changes altering the shape
and position of the beams and thus changing the efficiency of the tiling pattern.
@return katcp reply object [[[ !add-tiling ok | (fail [error description]) ]]]
"""
try:
product = self._get_product(product_id)
except ProductLookupError as error:
return ("fail", str(error))
try:
target = Target(target)
except Exception as error:
return ("fail", str(error))
tiling = product.add_tiling(target, nbeams, reference_frequency, overlap, epoch)
return ("ok", tiling.idxs())
@request()
@return_reply(Int())
def request_product_list(self, req):
"""
@brief List all currently registered products and their states
@param req A katcp request object
@note The details of each product are provided via an #inform
as a JSON string containing information on the product state.
@return katcp reply object [[[ !product-list ok | (fail [error description]) <number of configured products> ]]],
"""
for product_id,product in self._products.items():
info = {}
info[product_id] = product.info()
as_json = json.dumps(info)
req.inform(as_json)
return ("ok",len(self._products))
@request(Str(), Str())
@return_reply()
def request_set_default_target_configuration(self, req, product_id, target):
"""
@brief Set the configuration of FBFUSE from the FBFUSE configuration server
@param product_id This is a name for the data product, used to track which subarray is being deconfigured.
For example "array_1_bc856M4k".
@param target A KATPOINT target string
"""
try:
product = self._get_product(product_id)
except ProductLookupError as error:
return ("fail", str(error))
try:
target = Target(target)
except Exception as error:
return ("fail", str(error))
if not product.capturing:
return ("fail","Product must be capturing before a target confiugration can be set.")
product.reset_beams()
# TBD: Here we connect to some database and request the default configurations
# For example this may return secondary target in the FoV
#
# As a default the current system will put one beam directly on target and
# the rest of the beams in a static tiling pattern around this target
now = time.time()
nbeams = product._beam_manager.nbeams
product.add_tiling(target, nbeams-1, 1.4e9, 0.5, now)
product.add_beam(target)
return ("ok",)
@request(Str(), Str())
@return_reply()
def request_set_default_sb_configuration(self, req, product_id, sb_id):
"""
@brief Set the configuration of FBFUSE from the FBFUSE configuration server
@param product_id This is a name for the data product, used to track which subarray is being deconfigured.
For example "array_1_bc856M4k".
@param sb_id The schedule block ID. Decisions of the configuarion of FBFUSE will be made dependent on
the configuration of the current subarray, the primary and secondary science projects
active and the targets expected to be visted during the execution of the schedule block.
"""
try:
product = self._get_product(product_id)
except ProductLookupError as error:
return ("fail", str(error))
if product.capturing:
return ("fail", "Cannot reconfigure a currently capturing instance.")
product.configure_coherent_beams(400, product._katpoint_antennas, 1, 16)
product.configure_incoherent_beam(product._katpoint_antennas, 1, 16)
now = time.time()
nbeams = product._beam_manager.nbeams
product.add_tiling(target, nbeams-1, 1.4e9, 0.5, now)
product.add_beam(target)
return ("ok",)
def test_unpack_types_many_without_multiple(self):
expected = ['one', 2]
self.check_unpacking([Str(), Int()], [b'one', b'2'], expected)
class MasterController(AsyncDeviceServer):
"""This is the main KATCP interface for the FBFUSE
multi-beam beamformer on MeerKAT.
This interface satisfies the following ICDs:
CAM-FBFUSE: <link>
TUSE-FBFUSE: <link>
"""
VERSION_INFO = ("mpikat-api", 0, 1)
BUILD_INFO = ("mpikat-implementation", 0, 1, "rc1")
DEVICE_STATUSES = ["ok", "degraded", "fail"]
def __init__(self, ip, port, worker_pool):
"""
@brief Construct new MasterController instance
@params ip The IP address on which the server should listen
@params port The port that the server should bind to
"""
super(MasterController, self).__init__(ip, port)
self._products = {}
self._katportal_wrapper_type = KatportalClientWrapper
self._server_pool = worker_pool
def start(self):
"""
@brief Start the MasterController server
"""
super(MasterController, self).start()
def add_sensor(self, sensor):
log.debug("Adding sensor: {}".format(sensor.name))
super(MasterController, self).add_sensor(sensor)
def remove_sensor(self, sensor):
log.debug("Removing sensor: {}".format(sensor.name))
super(MasterController, self).remove_sensor(sensor)
def setup_sensors(self):
"""
@brief Set up monitoring sensors.
@note The following sensors are made available on top of default sensors
implemented in AsynDeviceServer and its base classes.
device-status: Reports the health status of the controller and associated devices:
Among other things report HW failure, SW failure and observation failure.
local-time-synced: Indicates whether the local time of the servers
is synchronised to the master time reference (use NTP).
This sensor is aggregated from all nodes that are part
of FBF and will return "not sync'd" if any nodes are
unsyncronised.
products: The list of product_ids that controller is currently handling
"""
self._device_status = Sensor.discrete(
"device-status",
description="Health status of FBFUSE",
params=self.DEVICE_STATUSES,
default="ok",
initial_status=Sensor.NOMINAL)
self.add_sensor(self._device_status)
self._local_time_synced = Sensor.boolean(
"local-time-synced",
description="Indicates FBF is NTP syncronised.",
default=True,
initial_status=Sensor.UNKNOWN)
self.add_sensor(self._local_time_synced)
def ntp_callback():
try:
synced = check_ntp_sync()
except Exception as error:
log.exception("Unable to check NTP sync")
self._local_time_synced.set_value(False)
else:
if not synced:
log.warning("Server is not NTP synced")
self._local_time_synced.set_value(synced)
self._ntp_callback = PeriodicCallback(ntp_callback,
NTP_CALLBACK_PERIOD)
self._ntp_callback.start()
self._products_sensor = Sensor.string(
"products",
description="The names of the currently configured products",
default="",
initial_status=Sensor.UNKNOWN)
self.add_sensor(self._products_sensor)
def _update_products_sensor(self):
self._products_sensor.set_value(",".join(self._products.keys()))
def _get_product(self, product_id):
if product_id not in self._products:
raise ProductLookupError(
"No product configured with ID: {}".format(product_id))
else:
return self._products[product_id]
@request(Str(), Int())
@return_reply()
def request_register_worker_server(self, req, hostname, port):
"""
@brief Register an WorkerWrapper instance
@params hostname The hostname for the worker server
@params port The port number that the worker server serves on
@detail Register an WorkerWrapper instance that can be used for FBFUSE
computation. FBFUSE has no preference for the order in which control
servers are allocated to a subarray. An WorkerWrapper wraps an atomic
unit of compute comprised of one CPU, one GPU and one NIC (i.e. one NUMA
node on an FBFUSE compute server).
"""
log.debug("Received request to register worker server at {}:{}".format(
hostname, port))
self._server_pool.add(hostname, port)
return ("ok", )
@request(Str(), Int())
@return_reply()
def request_deregister_worker_server(self, req, hostname, port):
"""
@brief Deregister an WorkerWrapper instance
@params hostname The hostname for the worker server
@params port The port number that the worker server serves on
@detail The graceful way of removing a server from rotation. If the server is
currently actively processing an exception will be raised.
"""
log.debug(
"Received request to deregister worker server at {}:{}".format(
hostname, port))
try:
self._server_pool.remove(hostname, port)
except ServerDeallocationError as error:
log.error(
"Request to deregister worker server at {}:{} failed with error: {}"
.format(hostname, port, str(error)))
return ("fail", str(error))
else:
return ("ok", )
@request()
@return_reply(Int())
def request_worker_server_list(self, req):
"""
@brief List all control servers and provide minimal metadata
"""
for server in self._server_pool.used():
req.inform("{} allocated".format(server))
for server in self._server_pool.available():
req.inform("{} free".format(server))
return ("ok", len(self._server_pool.used()) +
len(self._server_pool.available()))
@request()
@return_reply(Int())
def request_product_list(self, req):
"""
@brief List all currently registered products and their states
@param req A katcp request object
@note The details of each product are provided via an #inform
as a JSON string containing information on the product state.
@return katcp reply object [[[ !product-list ok | (fail [error description]) <number of configured products> ]]],
"""
for product_id, product in self._products.items():
info = {}
info[product_id] = product.info()
as_json = json.dumps(info)
req.inform(as_json)
return ("ok", len(self._products))
@request(Str(), Str())
@return_reply()
def request_set_default_target_configuration(self, req, product_id,
target):
"""
@brief Set the configuration of FBFUSE from the FBFUSE configuration server
@param product_id This is a name for the data product, used to track which subarray is being deconfigured.
For example "array_1_bc856M4k".
@param target A KATPOINT target string
"""
try:
product = self._get_product(product_id)
except ProductLookupError as error:
return ("fail", str(error))
try:
target = Target(target)
except Exception as error:
return ("fail", str(error))
if not product.capturing:
return (
"fail",
"Product must be capturing before a target confiugration can be set."
)
product.reset_beams()
# TBD: Here we connect to some database and request the default configurations
# For example this may return secondary target in the FoV
#
# As a default the current system will put one beam directly on target and
# the rest of the beams in a static tiling pattern around this target
now = time.time()
nbeams = product._beam_manager.nbeams
product.add_tiling(target, nbeams - 1, 1.4e9, 0.5, now)
product.add_beam(target)
return ("ok", )
@request(Str(), Str())
@return_reply()
def request_set_default_sb_configuration(self, req, product_id, sb_id):
"""
@brief Set the configuration of FBFUSE from the FBFUSE configuration server
@param product_id This is a name for the data product, used to track which subarray is being deconfigured.
For example "array_1_bc856M4k".
@param sb_id The schedule block ID. Decisions of the configuarion of FBFUSE will be made dependent on
the configuration of the current subarray, the primary and secondary science projects
active and the targets expected to be visted during the execution of the schedule block.
"""
try:
product = self._get_product(product_id)
except ProductLookupError as error:
return ("fail", str(error))
if product.capturing:
return ("fail",
"Cannot reconfigure a currently capturing instance.")
product.configure_coherent_beams(400, product._katpoint_antennas, 1,
16)
product.configure_incoherent_beam(product._katpoint_antennas, 1, 16)
now = time.time()
nbeams = product._beam_manager.nbeams
product.add_tiling(target, nbeams - 1, 1.4e9, 0.5, now)
product.add_beam(target)
return ("ok", )
def test_unpack_types_many_with_multiple(self):
expected = ['one', 2, 3]
self.check_unpacking([Str(), Int(multiple=True)], [b'one', b'2', b'3'],
expected)
class KATCPServer (DeviceServer):
VERSION_INFO = ("ptuse-api", 1, 0)
BUILD_INFO = ("ptuse-implementation", 0, 1, "")
# Optionally set the KATCP protocol version and features. Defaults to
# the latest implemented version of KATCP, with all supported optional
# features
PROTOCOL_INFO = ProtocolFlags(5, 0, set([
ProtocolFlags.MULTI_CLIENT,
ProtocolFlags.MESSAGE_IDS,
]))
def __init__ (self, server_host, server_port, script):
self.script = script
self._host_sensors = {}
self._beam_sensors = {}
self._data_product = {}
self._data_product["id"] = "None"
self.data_product_res = []
self.data_product_res.append(re.compile ("^[a-zA-Z]+_1"))
self.data_product_res.append(re.compile ("^[a-zA-Z]+_2"))
self.data_product_res.append(re.compile ("^[a-zA-Z]+_3"))
self.data_product_res.append(re.compile ("^[a-zA-Z]+_4"))
self.script.log(2, "KATCPServer::__init__ starting DeviceServer on " + server_host + ":" + str(server_port))
DeviceServer.__init__(self, server_host, server_port)
DEVICE_STATUSES = ["ok", "degraded", "fail"]
def setup_sensors(self):
"""Setup server sensors."""
self.script.log(2, "KATCPServer::setup_sensors()")
self._device_status = Sensor.discrete("device-status",
description="Status of entire system",
params=self.DEVICE_STATUSES,
default="ok")
self.add_sensor(self._device_status)
self._beam_name = Sensor.string("beam-name",
description="name of configured beam",
unit="",
default="")
self.add_sensor(self._beam_name)
# setup host based sensors
self._host_name = Sensor.string("host-name",
description="hostname of this server",
unit="",
default="")
self.add_sensor(self._host_name)
self.script.log(2, "KATCPServer::setup_sensors lmc="+str(self.script.lmc))
(host, port) = self.script.lmc.split(":")
self.setup_sensors_host (host, port)
self.script.log(2, "KATCPServer::setup_sensors beams="+str(self.script.beam))
self.setup_sensors_beam (self.script.beam_name)
# add sensors based on the reply from the specified host
def setup_sensors_host (self, host, port):
self.script.log(2, "KATCPServer::setup_sensors_host ("+host+","+port+")")
sock = sockets.openSocket (DL, host, int(port), 1)
if sock:
self.script.log(2, "KATCPServer::setup_sensors_host sock.send(" + self.script.lmc_cmd + ")")
sock.send (self.script.lmc_cmd + "\r\n")
lmc_reply = sock.recv (65536)
sock.close()
xml = xmltodict.parse(lmc_reply)
self._host_sensors = {}
# Disk sensors
self.script.log(2, "KATCPServer::setup_sensors_host configuring disk sensors")
disk_prefix = host+".disk"
self._host_sensors["disk_size"] = Sensor.float(disk_prefix+".size",
description=host+": disk size",
unit="MB",
params=[8192,1e9],
default=0)
self._host_sensors["disk_available"] = Sensor.float(disk_prefix+".available",
description=host+": disk available space",
unit="MB",
params=[1024,1e9],
default=0)
self.add_sensor(self._host_sensors["disk_size"])
self.add_sensor(self._host_sensors["disk_available"])
# Server Load sensors
self.script.log(2, "KATCPServer::setup_sensors_host configuring load sensors")
self._host_sensors["num_cores"] = Sensor.integer (host+".num_cores",
description=host+": disk available space",
unit="MB",
params=[1,64],
default=0)
self._host_sensors["load1"] = Sensor.float(host+".load.1min",
description=host+": 1 minute load ",
unit="",
default=0)
self._host_sensors["load5"] = Sensor.float(host+".load.5min",
description=host+": 5 minute load ",
unit="",
default=0)
self._host_sensors["load15"] = Sensor.float(host+".load.15min",
description=host+": 15 minute load ",
unit="",
default=0)
self.add_sensor(self._host_sensors["num_cores"])
self.add_sensor(self._host_sensors["load1"])
self.add_sensor(self._host_sensors["load5"])
self.add_sensor(self._host_sensors["load15"])
cpu_temp_pattern = re.compile("cpu[0-9]+_temp")
fan_speed_pattern = re.compile("fan[0-9,a-z]+")
power_supply_pattern = re.compile("ps[0-9]+_status")
self.script.log(2, "KATCPServer::setup_sensors_host configuring other metrics")
if not xml["lmc_reply"]["sensors"] == None:
for sensor in xml["lmc_reply"]["sensors"]["metric"]:
name = sensor["@name"]
if name == "system_temp":
self._host_sensors[name] = Sensor.float((host+".system_temp"),
description=host+": system temperature",
unit="C",
params=[-20,150],
default=0)
self.add_sensor(self._host_sensors[name])
if cpu_temp_pattern.match(name):
(cpu, junk) = name.split("_")
self._host_sensors[name] = Sensor.float((host+"." + name),
description=host+": "+ cpu +" temperature",
unit="C",
params=[-20,150],
default=0)
self.add_sensor(self._host_sensors[name])
if fan_speed_pattern.match(name):
self._host_sensors[name] = Sensor.float((host+"." + name),
description=host+": "+name+" speed",
unit="RPM",
params=[0,20000],
default=0)
self.add_sensor(self._host_sensors[name])
if power_supply_pattern.match(name):
self._host_sensors[name] = Sensor.boolean((host+"." + name),
description=host+": "+name,
unit="",
default=0)
self.add_sensor(self._host_sensors[name])
# TODO consider adding power supply sensors: e.g.
# device-status-kronos1-powersupply1
# device-status-kronos1-powersupply2
# device-status-kronos2-powersupply1
# device-status-kronos2-powersupply2
# TODO consider adding raid/disk sensors: e.g.
# device-status-<host>-raid
# device-status-<host>-raid-disk1
# device-status-<host>-raid-disk2
self.script.log(2, "KATCPServer::setup_sensors_host done!")
else:
self.script.log(2, "KATCPServer::setup_sensors_host no sensors found")
else:
self.script.log(-2, "KATCPServer::setup_sensors_host: could not connect to LMC")
# setup sensors for each beam
def setup_sensors_beam (self, beam):
b = str(beam)
self._beam_sensors = {}
self.script.log(2, "KATCPServer::setup_sensors_beam ="+b)
self._beam_sensors["observing"] = Sensor.boolean("observing",
description="Beam " + b + " is observing",
unit="",
default=0)
self.add_sensor(self._beam_sensors["observing"])
self._beam_sensors["snr"] = Sensor.float("snr",
description="SNR of Beam "+b,
unit="",
params=[0,1e9],
default=0)
self.add_sensor(self._beam_sensors["snr"])
self._beam_sensors["power"] = Sensor.float("power",
description="Power Level of Beam "+b,
unit="",
default=0)
self.add_sensor(self._beam_sensors["power"])
self._beam_sensors["integrated"] = Sensor.float("integrated",
description="Length of integration for Beam "+b,
unit="",
default=0)
self.add_sensor(self._beam_sensors["integrated"])
@request()
@return_reply(Str())
def request_beam(self, req):
"""Return the configure beam name."""
return ("ok", self._beam_name.value())
@request()
@return_reply(Str())
def request_host_name(self, req):
"""Return the name of this server."""
return ("ok", self._host_name.value())
@request()
@return_reply(Float())
def request_snr(self, req):
"""Return the SNR for this beam."""
return ("ok", self._beam_sensors["snr"].value())
@request()
@return_reply(Float())
def request_power(self, req):
"""Return the standard deviation of the 8-bit power level."""
return ("ok", self._beam_sensors["power"].value())
@request(Str(), Float())
@return_reply(Str())
def request_sync_time (self, req, data_product_id, adc_sync_time):
"""Set the ADC_SYNC_TIME for beam of the specified data product."""
if not data_product_id == self._data_product["id"]:
return ("fail", "data product " + str (data_product_id) + " was not configured")
self.script.beam_config["lock"].acquire()
self.script.beam_config["ADC_SYNC_TIME"] = str(adc_sync_time)
self.script.beam_config["lock"].release()
return ("ok", "")
@request(Str(), Str())
@return_reply(Str())
def request_target_start (self, req, data_product_id, target_name):
"""Commence data processing on specific data product and beam using target."""
self.script.log (1, "request_target_start(" + data_product_id + "," + target_name+")")
self.script.beam_config["lock"].acquire()
self.script.beam_config["ADC_SYNC_TIME"] = self.script.cam_config["ADC_SYNC_TIME"]
self.script.beam_config["OBSERVER"] = self.script.cam_config["OBSERVER"]
self.script.beam_config["ANTENNAE"] = self.script.cam_config["ANTENNAE"]
self.script.beam_config["SCHEDULE_BLOCK_ID"] = self.script.cam_config["SCHEDULE_BLOCK_ID"]
self.script.beam_config["EXPERIMENT_ID"] = self.script.cam_config["EXPERIMENT_ID"]
self.script.beam_config["DESCRIPTION"] = self.script.cam_config["DESCRIPTION"]
self.script.beam_config["lock"].release()
# check the pulsar specified is listed in the catalog
(result, message) = self.test_pulsar_valid (target_name)
if result != "ok":
return (result, message)
# check the ADC_SYNC_TIME is valid for this beam
if self.script.beam_config["ADC_SYNC_TIME"] == "0":
return ("fail", "ADC Synchronisation Time was not valid")
# set the pulsar name, this should include a check if the pulsar is in the catalog
self.script.beam_config["lock"].acquire()
if self.script.beam_config["MODE"] == "CAL":
target_name = target_name + "_R"
self.script.beam_config["SOURCE"] = target_name
self.script.beam_config["lock"].release()
host = self.script.tcs_host
port = self.script.tcs_port
self.script.log (2, "request_target_start: opening socket for beam " + beam_id + " to " + host + ":" + str(port))
sock = sockets.openSocket (DL, host, int(port), 1)
if sock:
xml = self.script.get_xml_config()
sock.send(xml + "\r\n")
reply = sock.recv (65536)
xml = self.script.get_xml_start_cmd()
sock.send(xml + "\r\n")
reply = sock.recv (65536)
sock.close()
return ("ok", "")
else:
return ("fail", "could not connect to TCS")
@request(Str())
@return_reply(Str())
def request_target_stop (self, req, data_product_id):
"""Cease data processing with target_name."""
self.script.log (1, "request_target_stop(" + data_product_id+")")
self.script.beam_config["lock"].acquire()
self.script.beam_config["SOURCE"] = ""
self.script.beam_config["lock"].release()
host = self.script.tcs_host
port = self.script.tcs_port
sock = sockets.openSocket (DL, host, int(port), 1)
if sock:
xml = self.script.get_xml_stop_cmd ()
sock.send(xml + "\r\n")
reply = sock.recv (65536)
sock.close()
return ("ok", "")
else:
return ("fail", "could not connect to tcs[beam]")
@request(Str())
@return_reply(Str())
def request_capture_init (self, req, data_product_id):
"""Prepare the ingest process for data capture."""
self.script.log (1, "request_capture_init: " + str(data_product_id) )
if not data_product_id == self._data_product["id"]:
return ("fail", "data product " + str (data_product_id) + " was not configured")
return ("ok", "")
@request(Str())
@return_reply(Str())
def request_capture_done(self, req, data_product_id):
"""Terminte the ingest process for the specified data_product_id."""
self.script.log (1, "request_capture_done: " + str(data_product_id))
if not data_product_id == self._data_product["id"]:
return ("fail", "data product " + str (data_product_id) + " was not configured")
return ("ok", "")
@return_reply(Str())
def request_configure(self, req, msg):
"""Prepare and configure for the reception of the data_product_id."""
self.script.log (1, "request_configure: nargs= " + str(len(msg.arguments)) + " msg=" + str(msg))
if len(msg.arguments) == 0:
self.script.log (-1, "request_configure: no arguments provided")
return ("ok", "configured data products: TBD")
# the sub-array identifier
data_product_id = msg.arguments[0]
if len(msg.arguments) == 1:
self.script.log (1, "request_configure: request for configuration of " + str(data_product_id))
if data_product_id == self._data_product["id"]:
configuration = str(data_product_id) + " " + \
str(self._data_product['antennas']) + " " + \
str(self._data_product['n_channels']) + " " + \
str(self._data_product['cbf_source'])
self.script.log (1, "request_configure: configuration of " + str(data_product_id) + "=" + configuration)
return ("ok", configuration)
else:
self.script.log (-1, "request_configure: no configuration existed for " + str(data_product_id))
return ("fail", "no configuration existed for " + str(data_product_id))
if len(msg.arguments) == 4:
# if the configuration for the specified data product matches extactly the
# previous specification for that data product, then no action is required
self.script.log (1, "configure: configuring " + str(data_product_id))
if data_product_id == self._data_product["id"] and \
self._data_product['antennas'] == msg.arguments[1] and \
self._data_product['n_channels'] == msg.arguments[2] and \
self._data_product['cbf_source'] == msg.arguments[3]:
response = "configuration for " + str(data_product_id) + " matched previous"
self.script.log (1, "configure: " + response)
return ("ok", response)
# the data product requires configuration
else:
self.script.log (1, "configure: new data product " + data_product_id)
# determine which sub-array we are matched against
the_sub_array = -1
for i in range(4):
self.script.log (1, "configure: testing self.data_product_res[" + str(i) +"].match(" + data_product_id +")")
if self.data_product_res[i].match (data_product_id):
the_sub_array = i + 1
if the_sub_array == -1:
self.script.log (1, "configure: could not match subarray from " + data_product_id)
return ("fail", "could not data product to sub array")
self.script.log (1, "configure: restarting pubsub for subarray " + str(the_sub_array))
self.script.pubsub.set_sub_array (the_sub_array, self.script.beam_name)
self.script.pubsub.restart()
antennas = msg.arguments[1]
n_channels = msg.arguments[2]
cbf_source = msg.arguments[3]
# check if the number of existing + new beams > available
(cfreq, bwd, nchan) = self.script.cfg["SUBBAND_CONFIG_0"].split(":")
if nchan != n_channels:
self._data_product.pop(data_product_id, None)
response = "PTUSE configured for " + nchan + " channels"
self.script.log (-1, "configure: " + response)
return ("fail", response)
self._data_product['id'] = data_product_id
self._data_product['antennas'] = antennas
self._data_product['n_channels'] = n_channels
self._data_product['cbf_source'] = cbf_source
# parse the CBF_SOURCE to determine multicast groups
(addr, port) = cbf_source.split(":")
(mcast, count) = addr.split("+")
self.script.log (2, "configure: parsed " + mcast + "+" + count + ":" + port)
if not count == "1":
response = "CBF source did not match ip_address+1:port"
self.script.log (-1, "configure: " + response)
return ("fail", response)
mcasts = ["",""]
ports = [0, 0]
quartets = mcast.split(".")
mcasts[0] = ".".join(quartets)
quartets[3] = str(int(quartets[3])+1)
mcasts[1] = ".".join(quartets)
ports[0] = int(port)
ports[1] = int(port)
self.script.log (1, "configure: connecting to RECV instance to update configuration")
for istream in range(int(self.script.cfg["NUM_STREAM"])):
(host, beam_idx, subband) = self.script.cfg["STREAM_" + str(istream)].split(":")
beam = self.script.cfg["BEAM_" + beam_idx]
if beam == self.script.beam_name:
# reset ADC_SYNC_TIME on the beam
self.script.beam_config["lock"].acquire()
self.script.beam_config["ADC_SYNC_TIME"] = "0";
self.script.beam_config["lock"].release()
port = int(self.script.cfg["STREAM_RECV_PORT"]) + istream
self.script.log (3, "configure: connecting to " + host + ":" + str(port))
sock = sockets.openSocket (DL, host, port, 1)
if sock:
req = "<?req version='1.0' encoding='ISO-8859-1'?>"
req += "<recv_cmd>"
req += "<command>configure</command>"
req += "<params>"
req += "<param key='DATA_MCAST_0'>" + mcasts[0] + "</param>"
req += "<param key='DATA_MCAST_1'>" + mcasts[1] + "</param>"
req += "<param key='DATA_PORT_0'>" + str(ports[0]) + "</param>"
req += "<param key='DATA_PORT_1'>" + str(ports[1]) + "</param>"
req += "<param key='META_MCAST_0'>" + mcasts[0] + "</param>"
req += "<param key='META_MCAST_1'>" + mcasts[1] + "</param>"
req += "<param key='META_PORT_0'>" + str(ports[0]) + "</param>"
req += "<param key='META_PORT_1'>" + str(ports[1]) + "</param>"
req += "</params>"
req += "</recv_cmd>"
self.script.log (1, "configure: sending XML req")
sock.send(req)
recv_reply = sock.recv (65536)
self.script.log (1, "configure: received " + recv_reply)
sock.close()
return ("ok", "data product " + str (data_product_id) + " configured")
else:
response = "expected 0, 1 or 4 arguments"
self.script.log (-1, "configure: " + response)
return ("fail", response)
@return_reply(Str())
def request_deconfigure(self, req, msg):
"""Deconfigure for the data_product."""
if len(msg.arguments) == 0:
self.script.log (-1, "request_configure: no arguments provided")
return ("fail", "expected 1 argument")
# the sub-array identifier
data_product_id = msg.arguments[0]
self.script.log (1, "configure: deconfiguring " + str(data_product_id))
# check if the data product was previously configured
if not data_product_id == self._data_product["id"]:
response = str(data_product_id) + " did not match configured data product [" + self._data_product["id"] + "]"
self.script.log (-1, "configure: " + response)
return ("fail", response)
for istream in range(int(self.script.cfg["NUM_STREAM"])):
(host, beam_idx, subband) = self.script.cfg["STREAM_" + str(istream)].split(":")
if self.script.beam_name == self.script.cfg["BEAM_" + beam_idx]:
# reset ADC_SYNC_TIME on the beam
self.script.beam_config["lock"].acquire()
self.script.beam_config["ADC_SYNC_TIME"] = "0";
self.script.beam_config["lock"].release()
port = int(self.script.cfg["STREAM_RECV_PORT"]) + istream
self.script.log (3, "configure: connecting to " + host + ":" + str(port))
sock = sockets.openSocket (DL, host, port, 1)
if sock:
req = "<?req version='1.0' encoding='ISO-8859-1'?>"
req += "<recv_cmd>"
req += "<command>deconfigure</command>"
req += "</recv_cmd>"
sock.send(req)
recv_reply = sock.recv (65536)
sock.close()
# remove the data product
self._data_product["id"] = "None"
response = "data product " + str(data_product_id) + " deconfigured"
self.script.log (1, "configure: " + response)
return ("ok", response)
@request(Int())
@return_reply(Str())
def request_output_channels (self, req, nchannels):
"""Set the number of output channels."""
if not self.test_power_of_two (nchannels):
return ("fail", "number of channels not a power of two")
if nchannels < 64 or nchannels > 4096:
return ("fail", "number of channels not within range 64 - 2048")
self.script.beam_config["OUTNCHAN"] = str(nchannels)
return ("ok", "")
@request(Int())
@return_reply(Str())
def request_output_bins(self, req, nbin):
"""Set the number of output phase bins."""
if not self.test_power_of_two(nbin):
return ("fail", "nbin not a power of two")
if nbin < 64 or nbin > 2048:
return ("fail", "nbin not within range 64 - 2048")
self.script.beam_config["OUTNBIN"] = str(nbin)
return ("ok", "")
@request(Int())
@return_reply(Str())
def request_output_tsubint (self, req, tsubint):
"""Set the length of output sub-integrations."""
if tsubint < 10 or tsubint > 60:
return ("fail", "length of output subints must be between 10 and 600 seconds")
self.script.beam_config["OUTTSUBINT"] = str(tsubint)
return ("ok", "")
@request(Float())
@return_reply(Str())
def request_dm(self, req, dm):
"""Set the value of dispersion measure to be removed"""
if dm < 0 or dm > 2000:
return ("fail", "dm not within range 0 - 2000")
self.script.beam_config["DM"] = str(dm)
return ("ok", "")
@request(Float())
@return_reply(Str())
def request_cal_freq(self, req, cal_freq):
"""Set the value of noise diode firing frequecny in Hz."""
if cal_freq < 0 or cal_freq > 1000:
return ("fail", "CAL freq not within range 0 - 1000")
self.script.beam_config["CALFREQ"] = str(cal_freq)
if cal_freq == 0:
self.script.beam_config["MODE"] = "PSR"
else:
self.script.beam_config["MODE"] = "CAL"
return ("ok", "")
# test if a number is a power of two
def test_power_of_two (self, num):
return num > 0 and not (num & (num - 1))
# test whether the specified target exists in the pulsar catalog
def test_pulsar_valid (self, target):
self.script.log (2, "test_pulsar_valid: get_psrcat_param (" + target + ", jname)")
(reply, message) = self.get_psrcat_param (target, "jname")
if reply != "ok":
return (reply, message)
self.script.log (2, "test_pulsar_valid: get_psrcat_param () reply=" + reply + " message=" + message)
if message == target:
return ("ok", "")
else:
return ("fail", "pulsar " + target + " did not exist in catalog")
def get_psrcat_param (self, target, param):
cmd = "psrcat -all " + target + " -c " + param + " -nohead -o short"
rval, lines = self.script.system (cmd, 3)
if rval != 0 or len(lines) <= 0:
return ("fail", "could not use psrcat")
if lines[0].startswith("WARNING"):
return ("fail", "pulsar " + target_name + " did not exist in catalog")
parts = lines[0].split()
if len(parts) == 2 and parts[0] == "1":
return ("ok", parts[1])
def test_unpack_types_more_types_than_args(self):
expected = ['one', 2, True, None]
self.check_unpacking(
[Str(), Int(),
Bool(default=True),
Str(optional=True)], [b'one', b'2'], expected)
class JsonStatusServer(AsyncDeviceServer):
VERSION_INFO = ("reynard-eff-jsonstatusserver-api", 0, 1)
BUILD_INFO = ("reynard-eff-jsonstatusserver-implementation", 0, 1, "rc1")
def __init__(self,
server_host,
server_port,
mcast_group=JSON_STATUS_MCAST_GROUP,
mcast_port=JSON_STATUS_PORT,
parser=EFF_JSON_CONFIG,
dummy=False):
self._mcast_group = mcast_group
self._mcast_port = mcast_port
self._parser = parser
self._dummy = dummy
if not dummy:
self._catcher_thread = StatusCatcherThread()
else:
self._catcher_thread = None
self._monitor = None
self._updaters = {}
self._controlled = set()
super(JsonStatusServer, self).__init__(server_host, server_port)
@coroutine
def _update_sensors(self):
log.debug("Updating sensor values")
data = self._catcher_thread.data
if data is None:
log.warning("Catcher thread has not received any data yet")
return
for name, params in self._parser.items():
if name in self._controlled:
continue
if "updater" in params:
self._sensors[name].set_value(params["updater"](data))
def start(self):
"""start the server"""
super(JsonStatusServer, self).start()
if not self._dummy:
self._catcher_thread.start()
self._monitor = PeriodicCallback(self._update_sensors,
1000,
io_loop=self.ioloop)
self._monitor.start()
def stop(self):
"""stop the server"""
if not self._dummy:
if self._monitor:
self._monitor.stop()
self._catcher_thread.stop()
return super(JsonStatusServer, self).stop()
@request()
@return_reply(Str())
def request_xml(self, req):
"""request an XML version of the status message"""
def make_elem(parent, name, text):
child = etree.Element(name)
child.text = text
parent.append(child)
@coroutine
def convert():
try:
root = etree.Element("TelescopeStatus",
attrib={"timestamp": str(time.time())})
for name, sensor in self._sensors.items():
child = etree.Element("TelStat")
make_elem(child, "Name", name)
make_elem(child, "Value", str(sensor.value()))
make_elem(child, "Status", str(sensor.status()))
make_elem(child, "Type", self._parser[name]["type"])
if "units" in self._parser[name]:
make_elem(child, "Units", self._parser[name]["units"])
root.append(child)
except Exception as error:
req.reply("ok", str(error))
else:
req.reply("ok", etree.tostring(root))
self.ioloop.add_callback(convert)
raise AsyncReply
@request()
@return_reply(Str())
def request_json(self, req):
"""request an JSON version of the status message"""
return ("ok", self.as_json())
def as_json(self):
"""Convert status sensors to JSON object"""
out = {}
for name, sensor in self._sensors.items():
out[name] = str(sensor.value())
return json.dumps(out)
@request(Str())
@return_reply(Str())
def request_sensor_control(self, req, name):
"""take control of a given sensor value"""
if name not in self._sensors:
return ("fail", "No sensor named '{0}'".format(name))
else:
self._controlled.add(name)
return ("ok", "{0} under user control".format(name))
@request()
@return_reply(Str())
def request_sensor_control_all(self, req):
"""take control of all sensors value"""
for name, sensor in self._sensors.items():
self._controlled.add(name)
return ("ok",
"{0} sensors under user control".format(len(self._controlled)))
@request()
@return_reply(Int())
def request_sensor_list_controlled(self, req):
"""List all controlled sensors"""
count = len(self._controlled)
for name in list(self._controlled):
req.inform("{0} -- {1}".format(name, self._sensors[name].value()))
return ("ok", count)
@request(Str())
@return_reply(Str())
def request_sensor_release(self, req, name):
"""release a sensor from user control"""
if name not in self._sensors:
return ("fail", "No sensor named '{0}'".format(name))
else:
self._controlled.remove(name)
return ("ok", "{0} released from user control".format(name))
@request()
@return_reply(Str())
def request_sensor_release_all(self, req):
"""take control of all sensors value"""
self._controlled = set()
return ("ok", "All sensors released")
@request(Str(), Str())
@return_reply(Str())
def request_sensor_set(self, req, name, value):
"""Set the value of a sensor"""
if name not in self._sensors:
return ("fail", "No sensor named '{0}'".format(name))
if name not in self._controlled:
return ("fail", "Sensor '{0}' not under user control".format(name))
try:
param = self._parser[name]
value = TYPE_CONVERTER[param["type"]](value)
self._sensors[name].set_value(value)
except Exception as error:
return ("fail", str(error))
else:
return ("ok", "{0} set to {1}".format(name,
self._sensors[name].value()))
def setup_sensors(self):
"""Set up basic monitoring sensors.
"""
for name, params in self._parser.items():
if params["type"] == "float":
sensor = Sensor.float(name,
description=params["description"],
unit=params.get("units", None),
default=params.get("default", 0.0),
initial_status=Sensor.UNKNOWN)
elif params["type"] == "string":
sensor = Sensor.string(name,
description=params["description"],
default=params.get("default", ""),
initial_status=Sensor.UNKNOWN)
elif params["type"] == "int":
sensor = Sensor.integer(name,
description=params["description"],
default=params.get("default", 0),
unit=params.get("units", None),
initial_status=Sensor.UNKNOWN)
elif params["type"] == "bool":
sensor = Sensor.boolean(name,
description=params["description"],
default=params.get("default", False),
initial_status=Sensor.UNKNOWN)
else:
raise Exception("Unknown sensor type '{0}' requested".format(
params["type"]))
self.add_sensor(sensor)
class FbfWorkerServer(AsyncDeviceServer):
VERSION_INFO = ("fbf-control-server-api", 0, 1)
BUILD_INFO = ("fbf-control-server-implementation", 0, 1, "rc1")
DEVICE_STATUSES = ["ok", "degraded", "fail"]
STATES = ["idle", "preparing", "ready", "starting", "capturing", "stopping", "error"]
IDLE, PREPARING, READY, STARTING, CAPTURING, STOPPING, ERROR = STATES
def __init__(self, ip, port, dummy=False):
"""
@brief Construct new FbfWorkerServer instance
@params ip The interface address on which the server should listen
@params port The port that the server should bind to
@params de_ip The IP address of the delay engine server
@params de_port The port number for the delay engine server
"""
self._dc_ip = None
self._dc_port = None
self._delay_client = None
self._delay_client = None
self._delays = None
self._dummy = dummy
self._dada_input_key = 0xdada
self._dada_coh_output_key = 0xcaca
self._dada_incoh_output_key = 0xbaba
super(FbfWorkerServer, self).__init__(ip,port)
@coroutine
def start(self):
"""Start FbfWorkerServer server"""
super(FbfWorkerServer,self).start()
@coroutine
def stop(self):
yield self.deregister()
yield super(FbfWorkerServer,self).stop()
def setup_sensors(self):
"""
@brief Set up monitoring sensors.
Sensor list:
- device-status
- local-time-synced
- fbf0-status
- fbf1-status
@note The following sensors are made available on top of default sensors
implemented in AsynDeviceServer and its base classes.
device-status: Reports the health status of the FBFUSE and associated devices:
Among other things report HW failure, SW failure and observation failure.
"""
self._device_status_sensor = Sensor.discrete(
"device-status",
description = "Health status of FbfWorkerServer instance",
params = self.DEVICE_STATUSES,
default = "ok",
initial_status = Sensor.NOMINAL)
self.add_sensor(self._device_status_sensor)
self._state_sensor = LoggingSensor.discrete(
"state",
params = self.STATES,
description = "The current state of this worker instance",
default = self.IDLE,
initial_status = Sensor.NOMINAL)
self._state_sensor.set_logger(log)
self.add_sensor(self._state_sensor)
self._delay_client_sensor = Sensor.string(
"delay-engine-server",
description = "The address of the currently set delay engine",
default = "",
initial_status = Sensor.UNKNOWN)
self.add_sensor(self._delay_client_sensor)
self._antenna_capture_order_sensor = Sensor.string(
"antenna-capture-order",
description = "The order in which the worker will capture antennas internally",
default = "",
initial_status = Sensor.UNKNOWN)
self.add_sensor(self._antenna_capture_order_sensor)
self._mkrecv_header_sensor = Sensor.string(
"mkrecv-header",
description = "The MKRECV/DADA header used for configuring capture with MKRECV",
default = "",
initial_status = Sensor.UNKNOWN)
self.add_sensor(self._mkrecv_header_sensor)
@property
def capturing(self):
return self.state == self.CAPTURING
@property
def idle(self):
return self.state == self.IDLE
@property
def starting(self):
return self.state == self.STARTING
@property
def stopping(self):
return self.state == self.STOPPING
@property
def ready(self):
return self.state == self.READY
@property
def preparing(self):
return self.state == self.PREPARING
@property
def error(self):
return self.state == self.ERROR
@property
def state(self):
return self._state_sensor.value()
def _system_call_wrapper(self, cmd):
log.debug("System call: '{}'".format(" ".join(cmd)))
if self._dummy:
log.debug("Server is running in dummy mode, system call will be ignored")
else:
check_call(cmd)
def _determine_feng_capture_order(self, antenna_to_feng_id_map, coherent_beam_config, incoherent_beam_config):
# Need to sort the f-engine IDs into 4 states
# 1. Incoherent but not coherent
# 2. Incoherent and coherent
# 3. Coherent but not incoherent
# 4. Neither coherent nor incoherent
#
# We must catch all antennas as even in case 4 the data is required for the
# transient buffer.
#
# To make this split, we first create the three sets, coherent, incoherent and all.
mapping = antenna_to_feng_id_map
all_feng_ids = set(mapping.values())
coherent_feng_ids = set(mapping[antenna] for antenna in parse_csv_antennas(coherent_beam_config['antennas']))
incoherent_feng_ids = set(mapping[antenna] for antenna in parse_csv_antennas(incoherent_beam_config['antennas']))
incoh_not_coh = incoherent_feng_ids.difference(coherent_feng_ids)
incoh_and_coh = incoherent_feng_ids.intersection(coherent_feng_ids)
coh_not_incoh = coherent_feng_ids.difference(incoherent_feng_ids)
used_fengs = incoh_not_coh.union(incoh_and_coh).union(coh_not_incoh)
unused_fengs = all_feng_ids.difference(used_fengs)
# Output final order
final_order = list(incoh_not_coh) + list(incoh_and_coh) + list(coh_not_incoh) + list(unused_fengs)
start_of_incoherent_fengs = 0
end_of_incoherent_fengs = len(incoh_not_coh) + len(incoh_and_coh)
start_of_coherent_fengs = len(incoh_not_coh)
end_of_coherent_fengs = len(incoh_not_coh) + len(incoh_and_coh) + len(coh_not_incoh)
start_of_unused_fengs = end_of_coherent_fengs
end_of_unused_fengs = len(all_feng_ids)
info = {
"order": final_order,
"incoherent_span":(start_of_incoherent_fengs, end_of_incoherent_fengs),
"coherent_span":(start_of_coherent_fengs, end_of_coherent_fengs),
"unused_span":(start_of_unused_fengs, end_of_unused_fengs)
}
return info
@request(Str(), Int(), Int(), Float(), Float(), Str(), Str(), Str(), Str(), Str(), Int())
@return_reply()
def request_prepare(self, req, feng_groups, nchans_per_group, chan0_idx, chan0_freq,
chan_bw, mcast_to_beam_map, feng_config, coherent_beam_config,
incoherent_beam_config, dc_ip, dc_port):
"""
@brief Prepare FBFUSE to receive and process data from a subarray
@detail REQUEST ?configure feng_groups, nchans_per_group, chan0_idx, chan0_freq,
chan_bw, mcast_to_beam_map, antenna_to_feng_id_map, coherent_beam_config,
incoherent_beam_config
Configure FBFUSE for the particular data products
@param req A katcp request object
@param feng_groups The contiguous range of multicast groups to capture F-engine data from,
the parameter is formatted in stream notation, e.g.: spead://239.11.1.150+3:7147
@param nchans_per_group The number of frequency channels per multicast group
@param chan0_idx The index of the first channel in the set of multicast groups
@param chan0_freq The frequency in Hz of the first channel in the set of multicast groups
@param chan_bw The channel bandwidth in Hz
@param mcast_to_beam_map A JSON mapping between output multicast addresses and beam IDs. This is the sole
authority for the number of beams that will be produced and their indexes. The map
is in the form:
@code
{
"spead://239.11.2.150:7147":"cfbf00001,cfbf00002,cfbf00003,cfbf00004",
"spead://239.11.2.151:7147":"ifbf00001"
}
@param feng_config JSON dictionary containing general F-engine parameters.
@code
{
'bandwidth': 856e6,
'centre-frequency': 1200e6,
'sideband': 'upper',
'feng-antenna-map': {...},
'sync-epoch': 12353524243.0,
'nchans': 4096
}
@param coherent_beam_config A JSON object specifying the coherent beam configuration in the form:
@code
{
'tscrunch':16,
'fscrunch':1,
'antennas':'m007,m008,m009'
}
@endcode
@param incoherent_beam_config A JSON object specifying the incoherent beam configuration in the form:
@code
{
'tscrunch':16,
'fscrunch':1,
'antennas':'m007,m008,m009'
}
@endcode
@return katcp reply object [[[ !configure ok | (fail [error description]) ]]]
"""
if not self.idle:
return ("fail", "FBF worker not in IDLE state")
log.info("Preparing worker server instance")
try:
feng_config = json.loads(feng_config)
except Exception as error:
return ("fail", "Unable to parse F-eng config with error: {}".format(str(error)))
try:
mcast_to_beam_map = json.loads(mcast_to_beam_map)
except Exception as error:
return ("fail", "Unable to parse multicast beam mapping with error: {}".format(str(error)))
try:
coherent_beam_config = json.loads(coherent_beam_config)
except Exception as error:
return ("fail", "Unable to parse coherent beam config with error: {}".format(str(error)))
try:
incoherent_beam_config = json.loads(incoherent_beam_config)
except Exception as error:
return ("fail", "Unable to parse incoherent beam config with error: {}".format(str(error)))
@coroutine
def configure():
self._state_sensor.set_value(self.PREPARING)
log.debug("Starting delay configuration server client")
self._delay_client = KATCPClientResource(dict(
name="delay-configuration-client",
address=(dc_ip, dc_port),
controlled=True))
self._delay_client.start()
log.debug("Determining F-engine capture order")
feng_capture_order_info = self._determine_feng_capture_order(feng_config['feng-antenna-map'], coherent_beam_config,
incoherent_beam_config)
log.debug("Capture order info: {}".format(feng_capture_order_info))
feng_to_antenna_map = {value:key for key,value in feng_config['feng-antenna-map'].items()}
antenna_capture_order_csv = ",".join([feng_to_antenna_map[feng_id] for feng_id in feng_capture_order_info['order']])
self._antenna_capture_order_sensor.set_value(antenna_capture_order_csv)
log.debug("Parsing F-engines to capture: {}".format(feng_groups))
capture_range = ip_range_from_stream(feng_groups)
ngroups = capture_range.count
partition_nchans = nchans_per_group * ngroups
partition_bandwidth = partition_nchans * chan_bw
npol = 2
ndim = 2
nbits = 8
tsamp = 1.0 / (feng_config['bandwidth'] / feng_config['nchans'])
sample_clock = feng_config['bandwidth'] * 2
timestamp_step = feng_config['nchans'] * 2 * 256 # WARNING: This is only valid in 4k mode
frequency_ids = [chan0_idx+nchans_per_group*ii for ii in range(ngroups)] #WARNING: Assumes contigous groups
mkrecv_config = {
'frequency_mhz': (chan0_freq + feng_config['nchans']/2.0 * chan_bw) / 1e6,
'bandwidth': partition_bandwidth,
'tsamp_us': tsamp * 1e6,
'bytes_per_second': partition_bandwidth * npol * ndim * nbits,
'nchan': partition_nchans,
'dada_key': self._dada_input_key,
'nantennas': len(feng_capture_order_info['order']),
'antennas_csv': antenna_capture_order_csv,
'sync_epoch': feng_config['sync-epoch'],
'sample_clock': sample_clock,
'mcast_sources': ",".join([str(group) for group in capture_range]),
'mcast_port': capture_range.port,
'interface': "192.168.0.1",
'timestamp_step': timestamp_step,
'ordered_feng_ids_csv': ",".join(map(str, feng_capture_order_info['order'])),
'frequency_partition_ids_csv': ",".join(map(str,frequency_ids))
}
mkrecv_header = make_mkrecv_header(mkrecv_config)
self._mkrecv_header_sensor.set_value(mkrecv_header)
log.info("Determined MKRECV configuration:\n{}".format(mkrecv_header))
log.debug("Parsing beam to multicast mapping")
incoherent_beam = None
incoherent_beam_group = None
coherent_beam_to_group_map = {}
for group, beams in mcast_to_beam_map.items():
for beam in beams.split(","):
if beam.startswith("cfbf"):
coherent_beam_to_group_map[beam] = group
if beam.startswith("ifbf"):
incoherent_beam = beam
incoherent_beam_group = group
log.debug("Determined coherent beam to multicast mapping: {}".format(coherent_beam_to_group_map))
if incoherent_beam:
log.debug("Incoherent beam will be sent to: {}".format(incoherent_beam_group))
else:
log.debug("No incoherent beam specified")
"""
Tasks:
- compile kernels
- create shared memory banks
"""
# Compile beamformer
# TBD
# Need to come up with a good way to allocate keys for dada buffers
# Create input DADA buffer
log.debug("Creating dada buffer for input with key '{}'".format("%x"%self._dada_input_key))
#self._system_call_wrapper(["dada_db","-k",self._dada_input_key,"-n","64","-l","-p"])
# Create coherent beam output DADA buffer
log.debug("Creating dada buffer for coherent beam output with key '{}'".format("%x"%self._dada_coh_output_key))
#self._system_call_wrapper(["dada_db","-k",self._dada_coh_output_key,"-n","64","-l","-p"])
# Create incoherent beam output DADA buffer
log.debug("Creating dada buffer for incoherent beam output with key '{}'".format("%x"%self._dada_incoh_output_key))
#self._system_call_wrapper(["dada_db","-k",self._dada_incoh_output_key,"-n","64","-l","-p"])
# Create SPEAD transmitter for coherent beams
# Call to MKSEND
# Create SPEAD transmitter for incoherent beam
# Call to MKSEND
# Need to pass the delay buffer controller the F-engine capture order but only for the coherent beams
cstart, cend = feng_capture_order_info['coherent_span']
coherent_beam_feng_capture_order = feng_capture_order_info['order'][cstart:cend]
coherent_beam_antenna_capture_order = [feng_to_antenna_map[idx] for idx in coherent_beam_feng_capture_order]
# Start DelayBufferController instance
# Here we are going to make the assumption that the server and processing all run in
# one docker container that will be preallocated with the right CPU set, GPUs, memory
# etc. This means that the configurations need to be unique by NUMA node... [Note: no
# they don't, we can use the container IPC channel which isolates the IPC namespaces.]
if not self._dummy:
n_coherent_beams = len(coherent_beam_to_group_map)
coherent_beam_antennas = parse_csv_antennas(coherent_beam_config['antennas'])
self._delay_buffer_controller = DelayBufferController(self._delay_client,
coherent_beam_to_group_map.keys(),
coherent_beam_antenna_capture_order, 1)
yield self._delay_buffer_controller.start()
# Start beamformer instance
# TBD
# Define MKRECV configuration file
# SPEAD receiver does not get started until a capture init call
self._state_sensor.set_value(self.READY)
req.reply("ok",)
self.ioloop.add_callback(configure)
raise AsyncReply
@request(Str())
@return_reply()
def request_deconfigure(self, req):
"""
@brief Deconfigure the FBFUSE instance.
@note Deconfigure the FBFUSE instance. If FBFUSE uses katportalclient to get information
from CAM, then it should disconnect at this time.
@param req A katcp request object
@return katcp reply object [[[ !deconfigure ok | (fail [error description]) ]]]
"""
# Need to make sure everything is stopped
# Call self.stop?
# Need to delete all allocated DADA buffers:
@coroutine
def deconfigure():
log.info("Destroying dada buffer for input with key '{}'".format(self._dada_input_key))
self._system_call_wrapper(["dada_db","-k",self._dada_input_key,"-d"])
log.info("Destroying dada buffer for coherent beam output with key '{}'".format(self._dada_coh_output_key))
self._system_call_wrapper(["dada_db","-k",self._dada_coh_output_key,"-n","64","-l","-p"])
log.info("Destroying dada buffer for incoherent beam output with key '{}'".format(self._dada_incoh_output_key))
self._system_call_wrapper(["dada_db","-k",self._dada_coh_output_key,"-n","64","-l","-p"])
log.info("Destroying delay buffer controller")
del self._delay_buffer_controller
self._delay_buffer_controller = None
req.reply("ok",)
self.ioloop.add_callback(deconfigure)
raise AsyncReply
@request(Str())
@return_reply()
def request_capture_start(self, req):
"""
@brief Prepare FBFUSE ingest process for data capture.
@note A successful return value indicates that FBFUSE is ready for data capture and
has sufficient resources available. An error will indicate that FBFUSE is not
in a position to accept data
@param req A katcp request object
@return katcp reply object [[[ !capture-init ok | (fail [error description]) ]]]
"""
if not self.ready:
return ("fail", "FBF worker not in READY state")
# Here we start MKRECV running into the input dada buffer
self._mkrecv_ingest_proc = Popen(["mkrecv","--config",self._mkrecv_config_filename], stdout=PIPE, stderr=PIPE)
return ("ok",)
@request(Str())
@return_reply()
def request_capture_stop(self, req):
"""
@brief Terminate the FBFUSE ingest process for the particular FBFUSE instance
@note This writes out any remaining metadata, closes all files, terminates any remaining processes and
frees resources for the next data capture.
@param req A katcp request object
@param product_id This is a name for the data product, used to track which subarray is being told to stop capture.
For example "array_1_bc856M4k".
@return katcp reply object [[[ !capture-done ok | (fail [error description]) ]]]
"""
if not self.capturing and not self.error:
return ("ok",)
@coroutine
def stop_mkrecv_capture():
#send SIGTERM to MKRECV
log.info("Sending SIGTERM to MKRECV process")
self._mkrecv_ingest_proc.terminate()
self._mkrecv_timeout = 10.0
log.info("Waiting {} seconds for MKRECV to terminate...".format(self._mkrecv_timeout))
now = time.time()
while time.time()-now < self._mkrecv_timeout:
retval = self._mkrecv_ingest_proc.poll()
if retval is not None:
log.info("MKRECV returned a return value of {}".format(retval))
break
else:
yield sleep(0.5)
else:
log.warning("MKRECV failed to terminate in alloted time")
log.info("Killing MKRECV process")
self._mkrecv_ingest_proc.kill()
req.reply("ok",)
self.ioloop.add_callback(self.stop_mkrecv_capture)
raise AsyncReply
class TestDevice(object):
def __init__(self):
self.sent_messages = []
@request(Int(min=1, max=10), Discrete(("on", "off")), Bool())
@return_reply(Int(min=1, max=10), Discrete(("on", "off")), Bool())
def request_one(self, sock, i, d, b):
if i == 3:
return ("fail", "I failed!")
if i == 5:
return ("bananas", "This should never be sent")
if i == 6:
return ("ok", i, d, b, "extra parameter")
if i == 9:
# This actually gets put in the callback params automatically
orig_msg = Message.request("one", "foo", "bar")
self.finish_request_one(orig_msg, sock, i, d, b)
raise AsyncReply()
return ("ok", i, d, b)
@send_reply(Int(min=1, max=10), Discrete(("on", "off")), Bool())
def finish_request_one(self, msg, sock, i, d, b):
return (sock, msg, "ok", i, d, b)
def reply(self, sock, msg, orig_msg):
self.sent_messages.append([sock, msg])
@request(Int(min=1, max=3, default=2),
Discrete(("on", "off"), default="off"), Bool(default=True))
@return_reply(Int(min=1, max=3), Discrete(("on", "off")), Bool())
def request_two(self, sock, i, d, b):
return ("ok", i, d, b)
@return_reply(Int(min=1, max=3), Discrete(("on", "off")), Bool())
@request(Int(min=1, max=3), Discrete(("on", "off")), Bool())
def request_three(self, sock, i, d, b):
return ("ok", i, d, b)
@return_reply()
@request()
def request_four(self, sock):
return ["ok"]
@inform(Int(min=1, max=3), Discrete(("on", "off")), Bool())
def inform_one(self, sock, i, d, b):
pass
@request(Int(min=1, max=3), Discrete(("on", "off")), Bool())
@return_reply(Int(min=1, max=3), Discrete(("on", "off")), Bool())
def request_five(self, i, d, b):
return ("ok", i, d, b)
@return_reply(Int(min=1, max=3), Discrete(("on", "off")), Bool())
@request(Int(min=1, max=3), Discrete(("on", "off")), Bool())
def request_six(self, i, d, b):
return ("ok", i, d, b)
@return_reply(Int(), Str())
@request(Int(), include_msg=True)
def request_seven(self, msg, i):
return ("ok", i, msg.name)
@return_reply(Int(), Str())
@request(Int(), include_msg=True)
def request_eight(self, sock, msg, i):
return ("ok", i, msg.name)
class FitsInterfaceServer(AsyncDeviceServer):
"""
Class providing an interface between EDD processes and the
Effelsberg FITS writer
"""
VERSION_INFO = ("edd-fi-server-api", 1, 0)
BUILD_INFO = ("edd-fi-server-implementation", 0, 1, "")
DEVICE_STATUSES = ["ok", "degraded", "fail"]
PROTOCOL_INFO = ProtocolFlags(
5, 0, set([
ProtocolFlags.MULTI_CLIENT,
ProtocolFlags.MESSAGE_IDS,
]))
def __init__(self, interface, port, capture_interface, capture_port, fw_ip,
fw_port):
"""
@brief Initialization of the FitsInterfaceServer object
@param interface Interface address to serve on
@param port Port number to serve on
@param capture_interface Interface to capture data on
@param capture_port Port to capture data on from instruments
@param fw_ip IP address of the FITS writer
@param fw_port Port number to connected to on FITS writer
"""
self._configured = False
self._no_active_beams = None
self._nchannels = None
self._integ_time = None
self._blank_phase = None
self._capture_interface = capture_interface
self._capture_port = capture_port
self._fw_connection_manager = FitsWriterConnectionManager(
fw_ip, fw_port)
self._capture_thread = None
self._shutdown = False
super(FitsInterfaceServer, self).__init__(interface, port)
def start(self):
"""
@brief Start the server
"""
self._fw_connection_manager.start()
super(FitsInterfaceServer, self).start()
def stop(self):
"""
@brief Stop the server
"""
self._shutdown = True
self._stop_capture()
self._fw_connection_manager.stop()
super(FitsInterfaceServer, self).stop()
@property
def nbeams(self):
return self._active_beams_sensor.value()
@nbeams.setter
def nbeams(self, value):
self._active_beams_sensor.set_value(value)
@property
def nchannels(self):
return self._nchannels_sensor.value()
@nchannels.setter
def nchannels(self, value):
self._nchannels_sensor.set_value(value)
@property
def integration_time(self):
return self._integration_time_sensor.value()
@integration_time.setter
def integration_time(self, value):
self._integration_time_sensor.set_value(value)
@property
def nblank_phases(self):
return self._nblank_phases_sensor.value()
@nblank_phases.setter
def nblank_phases(self, value):
self._nblank_phases_sensor.set_value(value)
def setup_sensors(self):
"""
@brief Setup monitoring sensors
"""
self._device_status_sensor = Sensor.discrete(
"device-status",
description="Health status of FIServer",
params=self.DEVICE_STATUSES,
default="ok",
initial_status=Sensor.UNKNOWN)
self.add_sensor(self._device_status_sensor)
self._active_beams_sensor = Sensor.float(
"nbeams",
description="Number of beams that are currently active",
default=1,
initial_status=Sensor.UNKNOWN)
self.add_sensor(self._active_beams_sensor)
self._nchannels_sensor = Sensor.float(
"nchannels",
description="Number of channels in each beam",
default=1,
initial_status=Sensor.UNKNOWN)
self.add_sensor(self._nchannels_sensor)
self._integration_time_sensor = Sensor.float(
"integration-time",
description="The integration time for each beam",
default=1,
initial_status=Sensor.UNKNOWN)
self.add_sensor(self._integration_time_sensor)
self._nblank_phases_sensor = Sensor.integer(
"nblank-phases",
description="The number of blank phases",
default=1,
initial_status=Sensor.UNKNOWN)
self.add_sensor(self._nblank_phases_sensor)
def _stop_capture(self):
if self._capture_thread:
log.debug("Cleaning up capture thread")
self._capture_thread.stop()
self._capture_thread.join()
self._capture_thread = None
log.debug("Capture thread cleaned")
@request(Int(), Int(), Int(), Int())
@return_reply()
def request_configure(self, req, beams, channels, int_time, blank_phases):
"""
@brief Configure the FITS interface server
@param beams The number of beams expected
@param channels The number of channels expected
@param int_time The integration time (milliseconds int)
@param blank_phases The number of blank phases (1-4)
@return katcp reply object [[[ !configure ok | (fail [error description]) ]]]
"""
message = ("nbeams={}, nchannels={}, integration_time={},"
" nblank_phases={}").format(beams, channels, int_time,
blank_phases)
log.info("Configuring FITS interface server with params: {}".format(
message))
self.nbeams = beams
self.nchannels = channels
self.integration_time = int_time
self.nblank_phases = blank_phases
self._fw_connection_manager.drop_connection()
self._stop_capture()
self._configured = True
return ("ok", )
@request()
@return_reply()
def request_start(self, req):
"""
@brief Start the FITS interface server capturing data
@return katcp reply object [[[ !configure ok | (fail [error description]) ]]]
"""
log.info("Received start request")
if not self._configured:
msg = "FITS interface server is not configured"
log.error(msg)
return ("fail", msg)
try:
fw_socket = self._fw_connection_manager.get_transmit_socket()
except Exception as error:
log.exception(str(error))
return ("fail", str(error))
log.info("Starting FITS interface capture")
self._stop_capture()
buffer_size = 4 * (self.nchannels + 2)
handler = R2SpectrometerHandler(2, self.nchannels,
self.integration_time,
self.nblank_phases, fw_socket)
self._capture_thread = CaptureData(self._capture_interface,
self._capture_port, buffer_size,
handler)
self._capture_thread.start()
return ("ok", )
@request()
@return_reply()
def request_stop(self, req):
"""
@brief Stop the FITS interface server capturing data
@return katcp reply object [[[ !configure ok | (fail [error description]) ]]]
"""
log.info("Received stop request")
if not self._configured:
msg = "FITS interface server is not configured"
log.error(msg)
return ("fail", msg)
log.info("Stopping FITS interface capture")
self._stop_capture()
self._fw_connection_manager.drop_connection()
return ("ok", )
class ProxyProtocol(DeviceProtocol):
@request(include_msg=True)
@return_reply(Int(min=0))
def request_device_list(self, reqmsg):
"""Return a list of devices aggregated by the proxy.
Returns the list of devices a sequence of #device-list informs.
Inform Arguments
----------------
device : str
Name of a device.
Returns
-------
success : {'ok', 'fail'}
Whether sending the list of devices succeeded.
informs : int
Number of #device-list informs sent.
Examples
--------
?device-list
#device-list antenna
#device-list enviro
!device-list ok 2
"""
for name in sorted(self.factory.devices):
self.send_message(
Message.inform("device-list", name,
self.factory.devices[name].TYPE))
return "ok", len(self.factory.devices)
def request_sensor_list(self, msg):
"""Request the list of sensors.
The list of sensors is sent as a sequence of #sensor-list informs.
Parameters
----------
name : str or pattern, optional
If the name is not a pattern, list just the sensor with the given name.
A pattern starts and ends with a slash ('/') and uses the Python re
module's regular expression syntax. All sensors whose names contain the
pattern are listed. The default is to list all sensors.
Inform Arguments
----------------
name : str
The name of the sensor being described.
description : str
Description of the named sensor.
units : str
Units for the value of the named sensor.
type : str
Type of the named sensor.
params : list of str, optional
Additional sensor parameters (type dependent). For integer and float
sensors the additional parameters are the minimum and maximum sensor
value. For discrete sensors the additional parameters are the allowed
values. For all other types no additional parameters are sent.
Returns
-------
success : {'ok', 'fail'}
Whether sending the sensor list succeeded.
informs : int
Number of #sensor-list inform messages sent.
Examples
--------
?sensor-list
#sensor-list psu.voltage PSU\_voltage. V float 0.0 5.0
#sensor-list cpu.status CPU\_status. \@ discrete on off error
...
!sensor-list ok 5
?sensor-list /voltage/
#sensor-list psu.voltage PSU\_voltage. V float 0.0 5.0
#sensor-list cpu.voltage CPU\_voltage. V float 0.0 3.0
!sensor-list ok 2
?sensor-list cpu.power.on
#sensor-list cpu.power.on Whether\_CPU\_hase\_power. \@ boolean
!sensor-list ok 1
"""
# handle non-regex cases
if not msg.arguments or not (msg.arguments[0].startswith("/")
and msg.arguments[0].endswith("/")):
return DeviceProtocol.request_sensor_list(self, msg)
# handle regex
name_re = re.compile(msg.arguments[0][1:-1])
sensors = dict([(name, sensor)
for name, sensor in self.factory.sensors.iteritems()
if name_re.search(name)])
for name, sensor in sorted(sensors.items(), key=lambda x: x[0]):
self.send_message(
Message.inform("sensor-list", name, sensor.description,
sensor.units, sensor.stype,
*sensor.formatted_params))
return Message.reply(msg.name, "ok", len(sensors))
def _send_all_sensors(self, filter=None):
""" Sends all sensor values with given filter (None = all)
"""
counter = [0] # this has to be a list or an object, thanks to
# python lexical scoping rules (we could not write count += 1
# in a function)
def device_ok((informs, reply), device):
for inform in informs:
inform.arguments[2] = device.name + '.' + \
inform.arguments[2]
if filter is None or re.match(filter, inform.arguments[2]):
self.send_message(inform)
counter[0] += 1
def all_ok(_):
self.send_message(
Message.reply('sensor-value', 'ok', str(counter[0])))
wait_for = []
for device in self.factory.devices.itervalues():
if device.state == device.SYNCED:
d = device.send_request('sensor-value')
d.addCallback(device_ok, device)
wait_for.append(d)
# otherwise we don't have the list of sensors, so we don't
# send the message
DeferredList(wait_for).addCallback(all_ok)
for name, sensor in self.factory.sensors.iteritems():
if not isinstance(sensor, ProxiedSensor):
if filter is None or re.match(filter, name):
timestamp_ms, status, value = sensor.read_formatted()
counter[0] += 1
self.send_message(
Message.inform('sensor-value', timestamp_ms, "1", name,
status, value))
class KATCPServer(DeviceServer):
VERSION_INFO = ("ptuse-api", 2, 0)
BUILD_INFO = ("ptuse-implementation", 0, 1, "")
# Optionally set the KATCP protocol version and features. Defaults to
# the latest implemented version of KATCP, with all supported optional
# features
PROTOCOL_INFO = ProtocolFlags(
5, 0, set([
ProtocolFlags.MULTI_CLIENT,
ProtocolFlags.MESSAGE_IDS,
]))
def __init__(self, server_host, server_port, script):
self.script = script
self._host_sensors = {}
self._beam_sensors = {}
self._data_product = {}
self._data_product["id"] = "None"
self._data_product["state"] = "None"
self.data_product_res = []
self.data_product_res.append(re.compile("^[a-zA-Z]+_1"))
self.data_product_res.append(re.compile("^[a-zA-Z]+_2"))
self.data_product_res.append(re.compile("^[a-zA-Z]+_3"))
self.data_product_res.append(re.compile("^[a-zA-Z]+_4"))
self.script.log(
2, "KATCPServer::__init__ starting DeviceServer on " +
server_host + ":" + str(server_port))
DeviceServer.__init__(self, server_host, server_port)
DEVICE_STATUSES = ["ok", "degraded", "fail"]
def setup_sensors(self):
"""Setup server sensors."""
self.script.log(2, "KATCPServer::setup_sensors()")
self._device_status = Sensor.discrete(
"device-status",
description="Status of entire system",
params=self.DEVICE_STATUSES,
default="ok")
self.add_sensor(self._device_status)
self._beam_name = Sensor.string("beam-name",
description="name of configured beam",
unit="",
default="")
self.add_sensor(self._beam_name)
# setup host based sensors
self._host_name = Sensor.string("host-name",
description="hostname of this server",
unit="",
default="")
self.add_sensor(self._host_name)
# GUI URL TODO remove hardcoding
guis = [{
"title": "PTUSE Web Interface",
"description": "Live Pulsar timing monitoring plots",
"href": self.script.cfg["SPIP_ADDRESS"]
}]
encoded = json.dumps(guis)
self._gui_urls = Sensor.string("gui-urls",
description="PTUSE GUI URL",
unit="",
default=encoded)
self.add_sensor(self._gui_urls)
self._gui_urls.set_value(encoded)
# give LMC some time to prepare the socket
time.sleep(5)
self.script.log(
1, "KATCPServer::setup_sensors lmc=" + str(self.script.lmc))
(host, port) = self.script.lmc.split(":")
self.setup_sensors_host(host, port)
self.script.log(
2, "KATCPServer::setup_sensors beams=" + str(self.script.beam))
self.setup_sensors_beam(self.script.beam_name)
# add sensors based on the reply from the specified host
def setup_sensors_host(self, host, port):
self.script.log(
1, "KATCPServer::setup_sensors_host (" + host + "," + port + ")")
sock = sockets.openSocket(DL, host, int(port), 1)
if sock:
self.script.log(
2, "KATCPServer::setup_sensors_host sock.send(" +
self.script.lmc_cmd + ")")
sock.send(self.script.lmc_cmd + "\r\n")
lmc_reply = sock.recv(65536)
sock.close()
xml = xmltodict.parse(lmc_reply)
self.script.log(
2, "KATCPServer::setup_sensors_host sock.recv=" + str(xml))
self._host_sensors = {}
# Disk sensors
self.script.log(
2, "KATCPServer::setup_sensors_host configuring disk sensors")
disk_prefix = host + ".disk"
self._host_sensors["disk_size"] = Sensor.float(
disk_prefix + ".size",
description=host + ": disk size",
unit="MB",
params=[8192, 1e9],
default=0)
self._host_sensors["disk_available"] = Sensor.float(
disk_prefix + ".available",
description=host + ": disk available space",
unit="MB",
params=[1024, 1e9],
default=0)
self.add_sensor(self._host_sensors["disk_size"])
self.add_sensor(self._host_sensors["disk_available"])
# Server Load sensors
self.script.log(
2, "KATCPServer::setup_sensors_host configuring load sensors")
self._host_sensors["num_cores"] = Sensor.integer(
host + ".num_cores",
description=host + ": disk available space",
unit="MB",
params=[1, 64],
default=0)
self._host_sensors["load1"] = Sensor.float(host + ".load.1min",
description=host +
": 1 minute load ",
unit="",
default=0)
self._host_sensors["load5"] = Sensor.float(host + ".load.5min",
description=host +
": 5 minute load ",
unit="",
default=0)
self._host_sensors["load15"] = Sensor.float(host + ".load.15min",
description=host +
": 15 minute load ",
unit="",
default=0)
self._host_sensors["local_time_synced"] = Sensor.boolean(
"local_time_synced",
description=host + ": NTP server synchronisation",
unit="",
default=0)
self.add_sensor(self._host_sensors["num_cores"])
self.add_sensor(self._host_sensors["num_cores"])
self.add_sensor(self._host_sensors["load1"])
self.add_sensor(self._host_sensors["load5"])
self.add_sensor(self._host_sensors["load15"])
self.add_sensor(self._host_sensors["local_time_synced"])
cpu_temp_pattern = re.compile("cpu[0-9]+_temp")
fan_speed_pattern = re.compile("fan[0-9,a-z]+")
power_supply_pattern = re.compile("ps[0-9]+_status")
self.script.log(
2, "KATCPServer::setup_sensors_host configuring other metrics")
if not xml["lmc_reply"]["sensors"] == None:
for sensor in xml["lmc_reply"]["sensors"]["metric"]:
name = sensor["@name"]
if name == "system_temp":
self._host_sensors[name] = Sensor.float(
(host + ".system_temp"),
description=host + ": system temperature",
unit="C",
params=[-20, 150],
default=0)
self.add_sensor(self._host_sensors[name])
if cpu_temp_pattern.match(name):
(cpu, junk) = name.split("_")
self._host_sensors[name] = Sensor.float(
(host + "." + name),
description=host + ": " + cpu + " temperature",
unit="C",
params=[-20, 150],
default=0)
self.add_sensor(self._host_sensors[name])
if fan_speed_pattern.match(name):
self._host_sensors[name] = Sensor.float(
(host + "." + name),
description=host + ": " + name + " speed",
unit="RPM",
params=[0, 20000],
default=0)
self.add_sensor(self._host_sensors[name])
if power_supply_pattern.match(name):
self._host_sensors[name] = Sensor.boolean(
(host + "." + name),
description=host + ": " + name,
unit="",
default=0)
self.add_sensor(self._host_sensors[name])
# TODO consider adding power supply sensors: e.g.
# device-status-kronos1-powersupply1
# device-status-kronos1-powersupply2
# device-status-kronos2-powersupply1
# device-status-kronos2-powersupply2
# TODO consider adding raid/disk sensors: e.g.
# device-status-<host>-raid
# device-status-<host>-raid-disk1
# device-status-<host>-raid-disk2
self.script.log(2, "KATCPServer::setup_sensors_host done!")
else:
self.script.log(
2, "KATCPServer::setup_sensors_host no sensors found")
else:
self.script.log(
-2,
"KATCPServer::setup_sensors_host: could not connect to LMC")
# setup sensors for each beam
def setup_sensors_beam(self, beam):
b = str(beam)
self._beam_sensors = {}
self.script.log(2, "KATCPServer::setup_sensors_beam beam=" + b)
self._beam_sensors["observing"] = Sensor.boolean("observing",
description="Beam " +
b + " is observing",
unit="",
default=0)
self.add_sensor(self._beam_sensors["observing"])
self._beam_sensors["snr"] = Sensor.float("snr",
description="SNR of Beam " +
b,
unit="",
params=[0, 1e9],
default=0)
self.add_sensor(self._beam_sensors["snr"])
self._beam_sensors["beamformer_stddev_polh"] = Sensor.float(
"beamformer_stddev_polh",
description="Standard deviation of beam voltages for pol H",
unit="",
params=[0, 127],
default=0)
self.add_sensor(self._beam_sensors["beamformer_stddev_polh"])
self._beam_sensors["beamformer_stddev_polv"] = Sensor.float(
"beamformer_stddev_polv",
description="Standard deviation of beam voltages for pol V",
unit="",
params=[0, 127],
default=0)
self.add_sensor(self._beam_sensors["beamformer_stddev_polv"])
self._beam_sensors["integrated"] = Sensor.float(
"integrated",
description="Length of integration for Beam " + b,
unit="",
default=0)
self.add_sensor(self._beam_sensors["integrated"])
self._beam_sensors["input_channels"] = Sensor.integer(
"input_channels",
description="Number of configured input channels for Beam " + b,
unit="",
default=0)
self.add_sensor(self._beam_sensors["input_channels"])
@request()
@return_reply(Str())
def request_beam(self, req):
"""Return the configure beam name."""
return ("ok", self._beam_name.value())
@request()
@return_reply(Str())
def request_host_name(self, req):
"""Return the name of this server."""
return ("ok", self._host_name.value())
@request()
@return_reply(Float())
def request_snr(self, req):
"""Return the SNR for this beam."""
return ("ok", self._beam_sensors["snr"].value())
@request()
@return_reply(Float())
def request_beamformer_stddev_polh(self, req):
"""Return the standard deviation of the 8-bit power level of pol H."""
return ("ok", self._beam_sensors["beamformer_stddev_polh"].value())
@request()
@return_reply(Float())
def request_beamformer_stddev_polv(self, req):
"""Return the standard deviation of the 8-bit power level of pol V."""
return ("ok", self._beam_sensors["beamformer_stddev_polv"].value())
@request()
@return_reply(Float())
def request_local_time_synced(self, req):
"""Return the sychronisation with NTP time"""
return ("ok", self._beam_sensors["local_time_synced"].value())
@request(Float())
@return_reply(Str())
def request_sync_time(self, req, adc_sync_time):
"""Set the ADC_SYNC_TIME for beam of the data product."""
if self._data_product["id"] == "None":
return ("fail", "data product was not configured")
self.script.beam_config["lock"].acquire()
self.script.beam_config["ADC_SYNC_TIME"] = str(adc_sync_time)
self.script.beam_config["lock"].release()
return ("ok", "")
@request(Str())
@return_reply(Str())
def request_proposal_id(self, req, proposal_id):
"""Set the PROPOSAL_ID for the data product."""
if self._data_product["id"] == "None":
return ("fail", "data product was not configured")
self.script.beam_config["lock"].acquire()
self.script.beam_config["PROPOSAL_ID"] = proposal_id
self.script.beam_config["lock"].release()
return ("ok", "")
# ensure state changes work
def change_state(self, command):
state = self._data_product["state"]
reply = "ok"
message = ""
if command == "configure":
if state != "unconfigured":
message = "received " + command + " command when in " + state + " state"
else:
new_state = "configured"
elif command == "capture_init":
if state != "configured":
message = "received " + command + " command when in " + state + " state"
else:
new_state = "ready"
elif command == "target_start":
if state != "ready":
message = "received " + command + " command when in " + state + " state"
else:
new_state = "recording"
elif command == "target_stop":
if state != "recording":
message = "received " + command + " command when in " + state + " state"
else:
new_state = "ready"
elif command == "capture_done":
if state != "ready" and state != "configured":
message = "received " + command + " command when in " + state + " state"
else:
new_state = "configured"
elif command == "deconfigure":
if state != "configured":
message = "received " + command + " command when in " + state + " state"
else:
new_state = "unconfigured"
if message == "":
self.script.log(
1, "change_state: " + self._data_product["state"] + " -> " +
new_state)
self._data_product["state"] = new_state
else:
self.script.log(-1, "change_state: " + message)
reply = "fail"
return (reply, message)
@request(Str())
@return_reply(Str())
def request_target_start(self, req, target_name):
"""Commence data processing using target."""
self.script.log(1, "request_target_start(" + target_name + ")")
if self._data_product["id"] == "None":
return ("fail", "data product was not configured")
self.script.log(
1, "request_target_start ADC_SYNC_TIME=" +
self.script.cam_config["ADC_SYNC_TIME"])
self.script.beam_config["lock"].acquire()
self.script.beam_config["TARGET"] = self.script.cam_config["TARGET"]
if self.script.cam_config["ADC_SYNC_TIME"] != "0":
self.script.beam_config["ADC_SYNC_TIME"] = self.script.cam_config[
"ADC_SYNC_TIME"]
self.script.beam_config["NCHAN_PER_STREAM"] = self.script.cam_config[
"NCHAN_PER_STREAM"]
self.script.beam_config[
"PRECISETIME_FRACTION_POLV"] = self.script.cam_config[
"PRECISETIME_FRACTION_POLV"]
self.script.beam_config[
"PRECISETIME_FRACTION_POLH"] = self.script.cam_config[
"PRECISETIME_FRACTION_POLH"]
self.script.beam_config[
"PRECISETIME_UNCERTAINTY_POLV"] = self.script.cam_config[
"PRECISETIME_UNCERTAINTY_POLV"]
self.script.beam_config[
"PRECISETIME_UNCERTAINTY_POLH"] = self.script.cam_config[
"PRECISETIME_UNCERTAINTY_POLH"]
self.script.beam_config["TFR_KTT_GNSS"] = self.script.cam_config[
"TFR_KTT_GNSS"]
self.script.beam_config["ITRF"] = self.script.cam_config["ITRF"]
self.script.beam_config["OBSERVER"] = self.script.cam_config[
"OBSERVER"]
self.script.beam_config["ANTENNAE"] = self.script.cam_config[
"ANTENNAE"]
self.script.beam_config["SCHEDULE_BLOCK_ID"] = self.script.cam_config[
"SCHEDULE_BLOCK_ID"]
self.script.beam_config["PROPOSAL_ID"] = self.script.cam_config[
"PROPOSAL_ID"]
self.script.beam_config["EXPERIMENT_ID"] = self.script.cam_config[
"EXPERIMENT_ID"]
self.script.beam_config["DESCRIPTION"] = self.script.cam_config[
"DESCRIPTION"]
self.script.beam_config["lock"].release()
# check the pulsar specified is listed in the catalog
(result, message) = self.test_pulsar_valid(target_name)
if result != "ok":
return (result, message)
# check the ADC_SYNC_TIME is valid for this beam
if self.script.beam_config["ADC_SYNC_TIME"] == "0":
return ("fail", "ADC Synchronisation Time was not valid")
# change the state
(result, message) = self.change_state("target_start")
if result != "ok":
self.script.log(-1,
"target_start: change_state failed: " + message)
return (result, message)
# set the pulsar name, this should include a check if the pulsar is in the catalog
self.script.beam_config["lock"].acquire()
self.script.beam_config["SOURCE"] = target_name
self.script.beam_config["lock"].release()
host = self.script.tcs_host
port = self.script.tcs_port
self.script.log(
2, "request_target_start: opening socket to " + host + ":" +
str(port))
sock = sockets.openSocket(DL, host, int(port), 1)
if sock:
xml = self.script.get_xml_config()
self.script.log(2,
"request_target_start: get_xml_config=" + str(xml))
sock.send(xml + "\r\n")
reply = sock.recv(65536)
self.script.log(2, "request_target_start: reply=" + str(reply))
xml = self.script.get_xml_start_cmd()
self.script.log(
2, "request_target_start: get_xml_start_cmd=" + str(xml))
sock.send(xml + "\r\n")
reply = sock.recv(65536)
self.script.log(2, "request_target_start: reply=" + str(reply))
sock.close()
return ("ok", "")
else:
return ("fail", "could not connect to TCS")
@request()
@return_reply(Str())
def request_target_stop(self, req):
"""Cease data processing with target_name."""
self.script.log(1, "request_target_stop()")
return self.target_stop()
def target_stop(self):
if self._data_product["id"] == "None":
return ("fail", "data product was not configured")
# change the state
(result, message) = self.change_state("target_stop")
if result != "ok":
self.script.log(-1, "target_stop: change_state failed: " + message)
return (result, message)
self.script.reset_beam_config()
host = self.script.tcs_host
port = self.script.tcs_port
sock = sockets.openSocket(DL, host, int(port), 1)
if sock:
xml = self.script.get_xml_stop_cmd()
sock.send(xml + "\r\n")
reply = sock.recv(65536)
sock.close()
return ("ok", "")
else:
return ("fail", "could not connect to tcs[beam]")
@request()
@return_reply(Str())
def request_capture_init(self, req):
"""Prepare the ingest process for data capture."""
self.script.log(1, "request_capture_init()")
# change the state
(result, message) = self.change_state("capture_init")
if result != "ok":
self.script.log(-1,
"capture_init: change_state failed: " + message)
return (result, message)
return ("ok", "")
@request()
@return_reply(Str())
def request_capture_done(self, req):
"""Terminte the ingest process."""
self.script.log(1, "request_capture_done()")
return self.capture_done()
def capture_done(self):
# in case the observing was terminated early
if self._data_product["state"] == "recording":
(result, message) = self.target_stop()
# change the state
(result, message) = self.change_state("capture_done")
if result != "ok":
self.script.log(-1,
"capture_done: change_state failed: " + message)
return (result, message)
return ("ok", "")
@return_reply(Str())
def request_configure(self, req, msg):
"""Prepare and configure for the reception of the data_product_id."""
self.script.log(
1, "request_configure: nargs= " + str(len(msg.arguments)) +
" msg=" + str(msg))
if len(msg.arguments) == 0:
self.script.log(-1, "request_configure: no arguments provided")
return ("ok", "configured data products: TBD")
# the sub-array identifier
data_product_id = msg.arguments[0]
if len(msg.arguments) == 1:
self.script.log(
1, "request_configure: request for configuration of " +
str(data_product_id))
if data_product_id == self._data_product["id"]:
configuration = str(data_product_id) + " " + \
str(self._data_product['antennas']) + " " + \
str(self._data_product['n_channels']) + " " + \
str(self._data_product['cbf_source']) + " " + \
str(self._data_product['proxy_name'])
self.script.log(
1, "request_configure: configuration of " +
str(data_product_id) + "=" + configuration)
return ("ok", configuration)
else:
self.script.log(
-1, "request_configure: no configuration existed for " +
str(data_product_id))
return ("fail",
"no configuration existed for " + str(data_product_id))
if len(msg.arguments) == 5:
# if the configuration for the specified data product matches extactly the
# previous specification for that data product, then no action is required
self.script.log(1,
"configure: configuring " + str(data_product_id))
if data_product_id == self._data_product["id"] and \
self._data_product['antennas'] == msg.arguments[1] and \
self._data_product['n_channels'] == msg.arguments[2] and \
self._data_product['cbf_source'] == str(msg.arguments[3]) and \
self._data_product['proxy_name'] == str(msg.arguments[4]):
response = "configuration for " + str(
data_product_id) + " matched previous"
self.script.log(1, "configure: " + response)
return ("ok", response)
# the data product requires configuration
else:
self.script.log(
1, "configure: new data product " + data_product_id)
# TODO decide what to do regarding preconfigured params (e.g. FREQ, BW) vs CAM supplied values
# determine which sub-array we are matched against
the_sub_array = -1
for i in range(4):
self.script.log(
1, "configure: testing self.data_product_res[" +
str(i) + "].match(" + data_product_id + ")")
if self.data_product_res[i].match(data_product_id):
the_sub_array = i + 1
if the_sub_array == -1:
self.script.log(
1, "configure: could not match subarray from " +
data_product_id)
return ("fail", "could not data product to sub array")
antennas = msg.arguments[1]
n_channels = msg.arguments[2]
cbf_source = str(msg.arguments[3])
streams = json.loads(msg.arguments[3])
proxy_name = str(msg.arguments[4])
self.script.log(2, "configure: streams=" + str(streams))
# check if the number of existing + new beams > available
# (cfreq, bwd, nchan1) = self.script.cfg["SUBBAND_CONFIG_0"].split(":")
# (cfreq, bwd, nchan2) = self.script.cfg["SUBBAND_CONFIG_1"].split(":")
# nchan = int(nchan1) + int(nchan2)
#if nchan != int(n_channels):
# self._data_product.pop(data_product_id, None)
# response = "PTUSE configured for " + str(nchan) + " channels"
# self.script.log (-1, "configure: " + response)
# return ("fail", response)
self._data_product['id'] = data_product_id
self._data_product['antennas'] = antennas
self._data_product['n_channels'] = n_channels
self._data_product['cbf_source'] = cbf_source
self._data_product['streams'] = str(streams)
self._data_product['proxy_name'] = proxy_name
self._data_product['state'] = "unconfigured"
# change the state
(result, message) = self.change_state("configure")
if result != "ok":
self.script.log(
-1, "configure: change_state failed: " + message)
return (result, message)
# determine the CAM metadata server and update pubsub
cam_server = "None"
fengine_stream = "None"
polh_stream = "None"
polv_stream = "None"
self.script.log(
2, "configure: streams.keys()=" + str(streams.keys()))
self.script.log(
2, "configure: streams['cam.http'].keys()=" +
str(streams['cam.http'].keys()))
if 'cam.http' in streams.keys(
) and 'camdata' in streams['cam.http'].keys():
cam_server = streams['cam.http']['camdata']
self.script.log(2,
"configure: cam_server=" + str(cam_server))
if 'cbf.antenna_channelised_voltage' in streams.keys():
stream_name = streams[
'cbf.antenna_channelised_voltage'].keys()[0]
fengine_stream = stream_name.split(".")[0]
self.script.log(
2, "configure: fengine_stream=" + str(fengine_stream))
if 'cbf.tied_array_channelised_voltage' in streams.keys():
for s in streams[
'cbf.tied_array_channelised_voltage'].keys():
if s.endswith('y'):
polv_stream = s
if s.endswith('x'):
polh_stream = s
self.script.log(
2, "configure: polh_stream=" + str(polh_stream) +
" polv_stream=" + str(polv_stream))
if cam_server != "None" and fengine_stream != "None" and polh_stream != "None":
self.script.pubsub.update_cam(cam_server, fengine_stream,
polh_stream, polv_stream,
antennas)
else:
response = "Could not extract streams[cam.http][camdata]"
self.script.log(1, "configure: cam_server=" + cam_server)
self.script.log(
1, "configure: fengine_stream=" + fengine_stream)
self.script.log(1, "configure: polh_stream=" + polh_stream)
self.script.log(-1, "configure: " + response)
return ("fail", response)
# restart the pubsub service
self.script.log(
1, "configure: restarting pubsub for new meta-data")
self.script.pubsub.restart()
# determine the X and Y tied array channelised voltage streams
mcasts = {}
ports = {}
key = 'cbf.tied_array_channelised_voltage'
if key in streams.keys():
stream = 'i0.tied-array-channelised-voltage.0x'
if stream in streams[key].keys():
(mcast,
port) = self.parseStreamAddress(streams[key][stream])
mcasts['x'] = mcast
ports['x'] = int(port)
else:
response = "Could not extract streams[" + key + "][" + stream + "]"
self.script.log(-1, "configure: " + response)
return ("fail", response)
stream = 'i0.tied-array-channelised-voltage.0y'
if stream in streams[key].keys():
(mcast,
port) = self.parseStreamAddress(streams[key][stream])
mcasts['y'] = mcast
ports['y'] = int(port)
else:
response = "Could not extract streams[" + key + "][" + stream + "]"
self.script.log(-1, "configure: " + response)
return ("fail", response)
# if the backend nchan is < CAM nchan
self.script.log(1, "configure: n_channels=" + str(n_channels))
if int(n_channels) == 1024 and False:
nchan = 992
self.script.log(
1, "configure: reconfiguring MCAST groups from " +
mcasts['x'] + ", " + mcasts['y'])
(mcast_base_x, mcast_ngroups_x) = mcasts['x'].split("+")
(mcast_base_y, mcast_ngroups_y) = mcasts['y'].split("+")
nchan_per_group = int(n_channels) / int(mcast_ngroups_x)
new_ngroups = nchan / nchan_per_group
offset = (int(mcast_ngroups_x) - new_ngroups) / 2
self.script.log(
1, "configure: nchan_per_group=" +
str(nchan_per_group) + " new_ngroups=" +
str(new_ngroups) + " offset=" + str(offset))
parts_x = mcast_base_x.split(".")
parts_y = mcast_base_y.split(".")
parts_x[3] = str(int(parts_x[3]) + offset)
parts_y[3] = str(int(parts_y[3]) + offset)
mcasts['x'] = ".".join(parts_x) + "+" + str(new_ngroups)
mcasts['y'] = ".".join(parts_y) + "+" + str(new_ngroups)
self.script.log(
1, "configure: reconfigured MCAST groups to " +
mcasts['x'] + ", " + mcasts['y'])
self.script.log(
1,
"configure: connecting to RECV instance to update configuration"
)
for istream in range(int(self.script.cfg["NUM_STREAM"])):
(host, beam_idx,
subband) = self.script.cfg["STREAM_" +
str(istream)].split(":")
beam = self.script.cfg["BEAM_" + beam_idx]
self.script.log(
1, "configure: istream=" + str(istream) + " beam=" +
beam + " script.beam_name=" + self.script.beam_name)
if beam == self.script.beam_name:
# reset ADC_SYNC_TIME on the beam
self.script.beam_config["lock"].acquire()
self.script.beam_config["ADC_SYNC_TIME"] = "0"
self.script.beam_config["lock"].release()
port = int(
self.script.cfg["STREAM_RECV_PORT"]) + istream
self.script.log(
1, "configure: connecting to " + host + ":" +
str(port))
sock = sockets.openSocket(DL, host, port, 1)
if sock:
req = "<?req version='1.0' encoding='ISO-8859-1'?>"
req += "<recv_cmd>"
req += "<command>configure</command>"
req += "<params>"
req += "<param key='DATA_MCAST_0'>" + mcasts[
'x'] + "</param>"
req += "<param key='DATA_PORT_0'>" + str(
ports['x']) + "</param>"
req += "<param key='META_MCAST_0'>" + mcasts[
'x'] + "</param>"
req += "<param key='META_PORT_0'>" + str(
ports['x']) + "</param>"
req += "<param key='DATA_MCAST_1'>" + mcasts[
'y'] + "</param>"
req += "<param key='DATA_PORT_1'>" + str(
ports['y']) + "</param>"
req += "<param key='META_MCAST_1'>" + mcasts[
'y'] + "</param>"
req += "<param key='META_PORT_1'>" + str(
ports['y']) + "</param>"
req += "</params>"
req += "</recv_cmd>"
self.script.log(
1, "configure: sending XML req [" + req + "]")
sock.send(req)
self.script.log(
1, "configure: send XML, receiving reply")
recv_reply = sock.recv(65536)
self.script.log(
1, "configure: received " + recv_reply)
sock.close()
else:
response = "configure: could not connect to stream " + str(
istream) + " at " + host + ":" + str(port)
self.script.log(-1, "configure: " + response)
return ("fail", response)
return ("ok",
"data product " + str(data_product_id) + " configured")
else:
response = "expected 0, 1 or 5 arguments, received " + str(
len(msg.arguments))
self.script.log(-1, "configure: " + response)
return ("fail", response)
# parse address of from spead://AAA.BBB.CCC.DDD+NN:PORT into
def parseStreamAddress(self, stream):
self.script.log(2, "parseStreamAddress: parsing " + stream)
(prefix, spead_address) = stream.split("//")
(mcast, port) = spead_address.split(":")
self.script.log(2, "parseStreamAddress: parsed " + mcast + ":" + port)
return (mcast, port)
@return_reply(Str())
def request_deconfigure(self, req, msg):
"""Deconfigure for the data_product."""
self.script.log(1, "request_deconfigure()")
# in case the observing was terminated early
if self._data_product["state"] == "recording":
(result, message) = self.target_stop()
if self._data_product["state"] == "ready":
(result, message) = self.capture_done()
data_product_id = self._data_product["id"]
# check if the data product was previously configured
if not data_product_id == self._data_product["id"]:
response = str(
data_product_id
) + " did not match configured data product [" + self._data_product[
"id"] + "]"
self.script.log(-1, "configure: " + response)
return ("fail", response)
# change the state
(result, message) = self.change_state("deconfigure")
if result != "ok":
self.script.log(-1, "deconfigure: change_state failed: " + message)
return (result, message)
for istream in range(int(self.script.cfg["NUM_STREAM"])):
(host, beam_idx,
subband) = self.script.cfg["STREAM_" + str(istream)].split(":")
if self.script.beam_name == self.script.cfg["BEAM_" + beam_idx]:
# reset ADC_SYNC_TIME on the beam
self.script.beam_config["lock"].acquire()
self.script.beam_config["ADC_SYNC_TIME"] = "0"
self.script.beam_config["lock"].release()
port = int(self.script.cfg["STREAM_RECV_PORT"]) + istream
self.script.log(
3, "configure: connecting to " + host + ":" + str(port))
sock = sockets.openSocket(DL, host, port, 1)
if sock:
req = "<?req version='1.0' encoding='ISO-8859-1'?>"
req += "<recv_cmd>"
req += "<command>deconfigure</command>"
req += "</recv_cmd>"
sock.send(req)
recv_reply = sock.recv(65536)
sock.close()
# remove the data product
self._data_product["id"] = "None"
response = "data product " + str(data_product_id) + " deconfigured"
self.script.log(1, "configure: " + response)
return ("ok", response)
@request(Int())
@return_reply(Str())
def request_output_channels(self, req, nchannels):
"""Set the number of output channels."""
self.script.log(1,
"request_output_channels: nchannels=" + str(nchannels))
if not self.test_power_of_two(nchannels):
self.script.log(
-1, "request_output_channels: " + str(nchannels) +
" not a power of two")
return ("fail", "number of channels not a power of two")
if nchannels < 64 or nchannels > 4096:
self.script.log(
-1, "request_output_channels: " + str(nchannels) +
" not within range 64 - 4096")
return ("fail", "number of channels not within range 64 - 4096")
self.script.beam_config["lock"].acquire()
self.script.beam_config["OUTNCHAN"] = str(nchannels)
self.script.beam_config["lock"].release()
return ("ok", "")
@request(Int())
@return_reply(Str())
def request_output_bins(self, req, nbin):
"""Set the number of output phase bins."""
self.script.log(1, "request_output_bins: nbin=" + str(nbin))
if not self.test_power_of_two(nbin):
self.script.log(
-1,
"request_output_bins: " + str(nbin) + " not a power of two")
return ("fail", "nbin not a power of two")
if nbin < 64 or nbin > 2048:
self.script.log(
-1, "request_output_bins: " + str(nbin) +
" not within range 64 - 2048")
return ("fail", "nbin not within range 64 - 2048")
self.script.beam_config["lock"].acquire()
self.script.beam_config["OUTNBIN"] = str(nbin)
self.script.beam_config["lock"].release()
return ("ok", "")
@request(Int())
@return_reply(Str())
def request_output_tsubint(self, req, tsubint):
"""Set the length of output sub-integrations."""
self.script.log(1, "request_output_tsubint: tsubint=" + str(tsubint))
if tsubint < 10 or tsubint > 60:
self.script.log(
-1, "request_output_tsubint: " + str(tsubint) +
" not within range 10 - 60")
return (
"fail",
"length of output subints must be between 10 and 60 seconds")
self.script.beam_config["lock"].acquire()
self.script.beam_config["OUTTSUBINT"] = str(tsubint)
self.script.beam_config["lock"].release()
return ("ok", "")
@request(Float())
@return_reply(Str())
def request_dispersion_measure(self, req, dm):
"""Set the value of dispersion measure to be removed"""
self.script.log(1, "request_dispersion_measure: dm=" + str(dm))
if dm > 2000:
self.script.log(
-1, "request_dispersion_measure: " + str(dm) + " > 2000")
return ("fail", "dm greater than limit of 2000")
self.script.beam_config["lock"].acquire()
self.script.beam_config["DM"] = str(dm)
self.script.beam_config["lock"].release()
return ("ok", "")
@request(Float())
@return_reply(Str())
def request_calibration_freq(self, req, cal_freq):
"""Set the value of noise diode firing frequecny in Hz."""
self.script.log(1,
"request_calibration_freq: cal_freq=" + str(cal_freq))
if cal_freq < 0 or cal_freq > 1000:
return ("fail", "CAL freq not within range 0 - 1000")
self.script.beam_config["lock"].acquire()
self.script.beam_config["CALFREQ"] = str(cal_freq)
if cal_freq == 0:
self.script.beam_config["MODE"] = "PSR"
else:
self.script.beam_config["MODE"] = "CAL"
self.script.beam_config["lock"].release()
return ("ok", "")
@request(Int())
@return_reply(Str())
def request_output_npol(self, req, outnpol):
"""Set the number of output pol parameters."""
self.script.log(1, "request_output_npol: outnpol=" + str(outnpol))
if outnpol != 1 and outnpol != 2 and outnpol != 3 and outnpol != 4:
self.script.log(
-1, "request_output_npol: " + str(outnpol) + " not 1, 2 or 4")
return ("fail", "output npol must be between 1, 2 or 4")
self.script.beam_config["lock"].acquire()
self.script.beam_config["OUTNPOL"] = str(outnpol)
self.script.beam_config["lock"].release()
return ("ok", "")
@request(Int())
@return_reply(Str())
def request_output_nbit(self, req, outnbit):
"""Set the number of bits per output sample."""
self.script.log(1, "request_output_nbit: outnbit=" + str(outnbit))
if outnbit != 1 and outnbit != 2 and outnbit != 4 and outnbit != 8:
self.script.log(
-1,
"request_output_nbit: " + str(outnbit) + " not 1, 2, 4 or 8")
return ("fail", "output nbit must be between 1, 2, 4 or 8")
self.script.beam_config["lock"].acquire()
self.script.beam_config["OUTNBIT"] = str(outnbit)
self.script.beam_config["lock"].release()
return ("ok", "")
@request(Int())
@return_reply(Str())
def request_output_tdec(self, req, outtdec):
"""Set the number of input samples integrated into 1 output sample."""
self.script.log(1, "request_output_tdec: outtdec=" + str(outtdec))
if outtdec < 16 or outtdec > 131072:
self.script.log(
-1, "request_output_tdec: " + str(outtdec) +
" not in range [16..131072]")
return ("fail", "output tdec must be between 16 and 131072")
self.script.beam_config["lock"].acquire()
self.script.beam_config["OUTTDEC"] = str(outtdec)
self.script.beam_config["lock"].release()
return ("ok", "")
@request()
@return_reply(Str())
def request_fold_mode(self, req):
"""Set the processing mode to produce folded archives."""
self.script.beam_config["lock"].acquire()
self.script.beam_config["PERFORM_FOLD"] = "1"
self.script.beam_config["PERFORM_SEARCH"] = "0"
self.script.log(1, "request_search_mode: PERFORM_FOLD=1")
self.script.beam_config["lock"].release()
return ("ok", "")
@request()
@return_reply(Str())
def request_search_mode(self, req):
"""Set the processing mode to produce filterbank data."""
self.script.beam_config["lock"].acquire()
self.script.beam_config["PERFORM_FOLD"] = "0"
self.script.beam_config["PERFORM_SEARCH"] = "1"
self.script.log(1, "request_search_mode: PERFORM_SEARCH=1")
self.script.beam_config["lock"].release()
return ("ok", "")
@request()
@return_reply(Str())
def request_disable_zeroed_buffers(self, req):
"""Disable zeroing of ring buffers, enabling stats mode."""
self.script.beam_config["lock"].acquire()
self.script.log(1, "request_disable_zeroed_buffers: ZERO_COPY=0")
self.script.beam_config["ZERO_COPY"] = "0"
self.script.beam_config["lock"].release()
return ("ok", "")
@request()
@return_reply(Str())
def request_enable_zeroed_buffers(self, req):
"""Enable zeroing of ring buffers, disabling stats mode."""
self.script.beam_config["lock"].acquire()
self.script.log(1, "request_enable_zeroed_buffers: ZERO_COPY=1")
self.script.beam_config["ZERO_COPY"] = "1"
self.script.beam_config["lock"].release()
return ("ok", "")
# test if a number is a power of two
def test_power_of_two(self, num):
return num > 0 and not (num & (num - 1))
# test whether the specified target exists in the pulsar catalog
def test_pulsar_valid(self, target):
self.script.log(2, "test_pulsar_valid: target='[" + target + "]")
# remove the _R suffix
if target.endswith('_R'):
target = target[:-2]
# check if the target matches the fluxcal.on file
cmd = "grep " + target + " " + self.script.cfg[
"CONFIG_DIR"] + "/fluxcal.on | wc -l"
rval, lines = self.script.system(cmd, 3)
if rval == 0 and len(lines) == 1 and int(lines[0]) > 0:
return ("ok", "")
# check if the target matches the fluxcal.off file
cmd = "grep " + target + " " + self.script.cfg[
"CONFIG_DIR"] + "/fluxcal.off | wc -l"
rval, lines = self.script.system(cmd, 3)
if rval == 0 and len(lines) == 1 and int(lines[0]) > 0:
return ("ok", "")
self.script.log(
2, "test_pulsar_valid: get_psrcat_param (" + target + ", jname)")
(reply, message) = self.get_psrcat_param(target, "jname")
if reply != "ok":
return (reply, message)
self.script.log(
2, "test_pulsar_valid: get_psrcat_param () reply=" + reply +
" message=" + message)
if message == target:
return ("ok", "")
else:
return ("fail", "pulsar " + target + " did not exist in catalog")
def get_psrcat_param(self, target, param):
# remove the _R suffix
if target.endswith('_R'):
target = target[:-2]
cmd = "psrcat -all " + target + " -c " + param + " -nohead -o short"
rval, lines = self.script.system(cmd, 3)
if rval != 0 or len(lines) <= 0:
return ("fail", "could not use psrcat")
if lines[0].startswith("WARNING"):
return ("fail",
"pulsar " + target_name + " did not exist in catalog")
parts = lines[0].split()
if len(parts) == 2 and parts[0] == "1":
return ("ok", parts[1])