def get_target(self, product_id, target_key, retries, retry_duration):
"""
Try to fetch the most recent target name by comparing its timestamp
to that of the most recent capture-start message.
"""
for i in range(retries):
last_target = float(
self.red.get("{}:last-target".format(product_id)))
last_start = float(
self.red.get("{}:last-capture-start".format(product_id)))
if ((last_target - last_start) <
0): # Check if new target available
log.warning("No new target name, retrying.")
time.sleep(retry_duration)
continue
else:
break
if (i == (retries - 1)):
log.error(
"No new target name after {} retries; defaulting to UNKNOWN".
format(retries))
target = 'UNKNOWN'
else:
target = self.red.get(target_key)
return target
def datadir(self, product_id):
"""Determine DATADIR according to the current schedule block ID. This
entails retrieving the list of schedule blocks, extracting the ID of
the current one and formatting it as a file path.
Schedule block IDs follow the format: YYYYMMDD-XXXX where XXXX is the
schedule block number (in order of assignment). To create the correct
DATADIR, we format it (per schedule block) as follows:
DATADIR=YYYYMMDD/XXXX
If the schedule block ID is not known, we set it to:
DATADIR=Unknown_SB
"""
current_sb_id = 'Unknown_SB' # Default
try:
sb_key = '{}:sched_observation_schedule_1'.format(product_id)
sb_ids = self.red.get(sb_key)
# First ID in list is the current SB (as per CAM GUI)
current_sb_id = sb_ids.split(',')[0]
# Format for file path
current_sb_id = current_sb_id.replace('-', '/')
except:
log.error("Schedule block IDs not available")
log.warning("Setting DATADIR=/buf0/Unknown_SB")
datadir = '/buf0/{}'.format(current_sb_id)
return datadir
def get_start_idx(self, host_list, idx_margin, log):
"""Calculate the packet index at which recording should begin
(synchronously) for all processing nodes.
Args:
red_server: Redis server.
host_list (List): List of host/processing node names (incuding
instance number).
idx_margin (int): The safety margin (number of extra packets
before beginning to record) to ensure a synchronous start across
processing nodes.
log: Logger.
Returns:
start_pkt (int): The packet index at which to begin recording
data.
"""
pkt_idxs = []
for host in host_list:
host_key = '{}://{}/status'.format(HPGDOMAIN, host)
pkt_idx = self.get_pkt_idx(host_key)
if (pkt_idx is not None):
pkt_idxs = pkt_idxs + [pkt_idx]
if (len(pkt_idxs) > 0):
start_pkt = self.select_pkt_start(pkt_idxs, log, idx_margin)
return start_pkt
else:
log.warning('No active processing nodes. Cannot set PKTIDX')
return None
def select_pkt_start(self, pkt_idxs, log, idx_margin):
"""Calculates the index of the first packet from which to record
for each processing node.
Employs rudimentary statistics on packet indices to determine
a sensible starting packet for all processing nodes.
Args:
pkt_idxs (list): List of the packet indices from each active host.
log: Logger.
idx_margin (int): The safety margin (number of extra packets
before beginning to record) to ensure a synchronous start across
processing nodes.
Returns:
start_pkt (int): The packet index at which to begin recording
data.
"""
pkt_idxs = np.asarray(pkt_idxs, dtype=np.int64)
median = np.median(pkt_idxs)
margin_diff = np.abs(pkt_idxs - median)
# Using idx_margin as safety margin
margin = idx_margin
outliers = np.where(margin_diff > margin)[0]
n_outliers = len(outliers)
if (n_outliers > 0):
log.warning('{} PKTIDX value(s) exceed margin.'.format(n_outliers))
if (n_outliers > len(pkt_idxs) / 2):
log.warning('Large PKTIDX spread. Check PKTSTART value.')
# Find largest value less than margin
margin_diff[outliers] = 0
max_idx = np.argmax(margin_diff)
start_pkt = pkt_idxs[max_idx] + idx_margin
return start_pkt
def main(port):
FORMAT = "[ %(levelname)s - %(asctime)s - %(filename)s:%(lineno)s] %(message)s"
# logger = logging.getLogger('reynard')
logging.basicConfig(format=FORMAT)
log.setLevel(logging.DEBUG)
log.info("Starting distributor")
red = redis.StrictRedis(port=port)
ps = red.pubsub(ignore_subscribe_messages=True)
ps.subscribe(CHANNEL)
try:
for message in ps.listen():
msg_parts = message['data'].split(':')
if len(msg_parts) != 2:
log.info("Not processing this message --> {}".format(message))
continue
msg_type = msg_parts[0]
product_id = msg_parts[1]
if msg_type == 'configure':
all_streams = json.loads(json_str_formatter(red.get("{}:streams".format(product_id))))
streams = all_streams[STREAM_TYPE]
addr_list, port = parse_spead_addresses(streams.values()[0])
nstreams = len(addr_list)
if nstreams > NCHANNELS:
log.warning("More than {} ({}) stream addresses found".format(NCHANNELS, nstreams))
for i in range(min(nstreams, NCHANNELS)):
msg = "{}:configure:stream:{}".format(product_id, addr_list[i])
red.publish(CHANNELS[i], msg)
except KeyboardInterrupt:
log.info("Stopping distributor")
sys.exit(0)
except Exception as e:
log.error(e)
sys.exit(1)
def create_addr_list_filled(self, addr0, n_groups, n_addrs,
streams_per_instance, offset):
"""Creates list of IP multicast subscription address groups.
Fills the list for each available processing instance
sequentially untill all streams have been assigned.
Args:
addr0 (str): IP address of the first stream.
n_groups (int): number of available processing instances.
n_addrs (int): total number of streams to process.
streams_per_instance (int): number of streams to be processed
by each instance.
offset (int): number of streams to skip before apportioning
IPs.
Returns:
addr_list (list): list of IP address groups for subscription.
"""
prefix, suffix0 = addr0.rsplit('.', 1)
suffix0 = int(suffix0) + offset
n_addrs = n_addrs - offset
addr_list = []
if (n_addrs > streams_per_instance * n_groups):
log.warning('Too many streams: {} will not be processed.'.format(
n_addrs - streams_per_instance * n_groups))
for i in range(0, n_groups):
addr_list.append(
prefix +
'.{}+{}'.format(suffix0, streams_per_instance - 1))
suffix0 = suffix0 + streams_per_instance
else:
n_instances_req = int(
np.ceil(n_addrs / float(streams_per_instance)))
for i in range(1, n_instances_req):
addr_list.append(
prefix +
'.{}+{}'.format(suffix0, streams_per_instance - 1))
suffix0 = suffix0 + streams_per_instance
addr_list.append(prefix + '.{}+{}'.format(
suffix0, n_addrs - 1 - i * streams_per_instance))
return addr_list
def get_pkt_idx(self, host_key):
"""Get PKTIDX for a host (if active).
Args:
red_server: Redis server.
host_key (str): Key for Redis hash of status buffer for a
particular active host.
Returns:
pkt_idx (str): Current packet index (PKTIDX) for a particular
active host. Returns None if host is not active.
"""
pkt_idx = None
host_status = self.red.hgetall(host_key)
if (len(host_status) > 0):
if ('NETSTAT' in host_status):
if (host_status['NETSTAT'] != 'idle'):
if ('PKTIDX' in host_status):
pkt_idx = host_status['PKTIDX']
else:
log.warning(
'PKTIDX is missing for {}'.format(host_key))
else:
log.warning('NETSTAT is missing for {}'.format(host_key))
else:
log.warning('Cannot acquire {}'.format(host_key))
return pkt_idx
def get_dwell_time(self, host_key):
"""Get the current dwell time from the status buffer
stored in Redis for a particular host.
Args:
host_key (str): Key for Redis hash of status buffer
for a particular host.
Returns:
dwell_time (int): Dwell time (recording length) in
seconds.
"""
dwell_time = 0
host_status = self.red.hgetall(host_key)
if (len(host_status) > 0):
if ('DWELL' in host_status):
dwell_time = host_status['DWELL']
else:
log.warning('DWELL is missing for {}'.format(host_key))
else:
log.warning('Cannot acquire {}'.format(host_key))
return dwell_time
def start(self):
"""Start the coordinator as follows:
- The list of available Hashpipe instances and the number of streams per
instance are retrieved from the main configuration .yml file.
- The number of available instances/hosts is read from the appropriate Redis key.
- Subscriptions are made to the three Redis channels.
- Incoming messages trigger the appropriate function for the stage of the
observation. The contents of the messages (if any) are sent to the
appropriate function.
"""
# Configure coordinator
try:
self.hashpipe_instances, self.streams_per_instance = self.config(
self.cfg_file)
log.info('Configured from {}'.format(self.cfg_file))
except:
log.warning(
'Configuration not updated; old configuration might be present.'
)
# Attempt to read list of available hosts. If key does not exist, recreate from
# config file
free_hosts = self.red.lrange('coordinator:free_hosts', 0,
self.red.llen('coordinator:free_hosts'))
if (len(free_hosts) == 0):
redis_tools.write_list_redis(self.red, 'coordinator:free_hosts',
self.hashpipe_instances)
log.info(
'First configuration - no list of available hosts. Retrieving from config file.'
)
# Subscribe to the required Redis channels.
ps = self.red.pubsub(ignore_subscribe_messages=True)
ps.subscribe(ALERTS_CHANNEL)
ps.subscribe(SENSOR_CHANNEL)
ps.subscribe(TRIGGER_CHANNEL)
# Process incoming Redis messages:
try:
for msg in ps.listen():
msg_type, description, value = self.parse_redis_msg(msg)
# If trigger mode is changed on the fly:
if ((msg_type == 'coordinator') &
(description == 'trigger_mode')):
self.triggermode = value
self.red.set('coordinator:trigger_mode', value)
log.info('Trigger mode set to \'{}\''.format(value))
# If all the sensor values required on configure have been
# successfully fetched by the katportalserver
elif (msg_type == 'conf_complete'):
self.conf_complete(description)
# If the current subarray is deconfigured, instruct processing nodes
# to unsubscribe from their respective streams.
# Only instruct processing nodes in the current subarray to unsubscribe.
# Likewise, release hosts only for the current subarray.
elif (msg_type == 'deconfigure'):
self.deconfigure(description)
# Handle the full data-suspect bitmask, one bit per polarisation
# per F-engine.
elif (msg_type == 'data-suspect'):
self.data_suspect(description, value)
# If the current subarray has transitioned to 'track' - that is,
# the antennas are on source and trackign successfully.
elif (msg_type == 'tracking'):
# Note that the description field is equivalent to product_id
# here:
self.tracking_start(description)
# If the current subarray transitions out of the tracking state:
elif (msg_type == 'not-tracking'):
self.tracking_stop(description)
# If pointing updates are received during tracking
elif ('pos_request_base' in description):
self.pointing_update(msg_type, description, value)
except KeyboardInterrupt:
log.info("Stopping coordinator")
sys.exit(0)
except Exception as e:
log.error(e)
sys.exit(1)
def conf_complete(self, description):
"""This function is run when a new subarray is configured and the
katportal_server has retrieved all the associated metadata required
for the processing nodes to ingest and record data from the F-engines.
The required metadata is published to the Hashpipe-Redis gateway in
the key-value pair format described in Appendix B of:
https://arxiv.org/pdf/1906.07391.pdf
Notably, the DESTIP value is set for each processing node - the IP
address of the multicast group it is to join.
Args:
description (str): the second field of the Redis message, which
in this case is the name of the current subarray.
"""
# This is the identifier for the subarray that has completed configuration.
product_id = description
tracking = 0 # Initialise tracking state to 0
log.info('New subarray built: {}'.format(product_id))
# Get IP address offset (if there is one) for ingesting only a specific
# portion of the full band.
offset = self.ip_offset(product_id)
# Initialise trigger mode (idle, armed or auto)
self.red.set('coordinator:trigger_mode:{}'.format(description),
self.triggermode)
log.info('Trigger mode for {} on startup: {}'.format(
description, self.triggermode))
# Generate list of stream IP addresses and publish appropriate messages to
# processing nodes:
addr_list, port, n_addrs, n_red_chans = self.ip_addresses(
product_id, offset)
# Allocate hosts:
free_hosts = self.red.lrange('coordinator:free_hosts', 0,
self.red.llen('coordinator:free_hosts'))
# Allocate hosts for the current subarray:
if (len(free_hosts) == 0):
log.warning(
"No free resources, cannot process data from {}".format(
product_id))
else:
allocated_hosts = free_hosts[0:n_red_chans]
redis_tools.write_list_redis(
self.red, 'coordinator:allocated_hosts:{}'.format(product_id),
allocated_hosts)
# Remove allocated hosts from list of available hosts
# NOTE: in future, append/pop with Redis commands instead of write_list_redis
if (len(free_hosts) < n_red_chans):
log.warning(
"Insufficient resources to process full band for {}".
format(product_id))
free_hosts = [] # Empty
else:
free_hosts = free_hosts[n_red_chans:]
redis_tools.write_list_redis(self.red, 'coordinator:free_hosts',
free_hosts)
log.info('Allocated {} hosts to {}'.format(n_red_chans,
product_id))
# Build list of Hashpipe-Redis Gateway channels to publish to:
chan_list = self.host_list(HPGDOMAIN, allocated_hosts)
# Apply to processing nodes
# NOTE: can we address multiple processing nodes more easily?
for i in range(len(chan_list)):
# Port (BINDPORT)
self.pub_gateway_msg(self.red, chan_list[i], 'BINDPORT', port,
log, True)
# Total number of streams (FENSTRM)
self.pub_gateway_msg(self.red, chan_list[i], 'FENSTRM',
n_addrs, log, True)
# Sync time (UNIX, seconds)
t_sync = self.sync_time(product_id)
self.pub_gateway_msg(self.red, chan_list[i], 'SYNCTIME',
t_sync, log, True)
# Centre frequency (FECENTER)
fecenter = self.centre_freq(product_id)
self.pub_gateway_msg(self.red, chan_list[i], 'FECENTER',
fecenter, log, True)
# Total number of frequency channels (FENCHAN)
n_freq_chans = self.red.get('{}:n_channels'.format(product_id))
self.pub_gateway_msg(self.red, chan_list[i], 'FENCHAN',
n_freq_chans, log, True)
# Coarse channel bandwidth (from F engines)
# Note: no sign information!
# (CHAN_BW)
chan_bw = self.coarse_chan_bw(product_id, n_freq_chans)
self.pub_gateway_msg(self.red, chan_list[i], 'CHAN_BW',
chan_bw, log, True)
# Number of channels per substream (HNCHAN)
hnchan = self.chan_per_substream(product_id)
self.pub_gateway_msg(self.red, chan_list[i], 'HNCHAN', hnchan,
log, True)
# Number of spectra per heap (HNTIME)
hntime = self.spectra_per_heap(product_id)
self.pub_gateway_msg(self.red, chan_list[i], 'HNTIME', hntime,
log, True)
# Number of ADC samples per heap (HCLOCKS)
adc_per_heap = self.samples_per_heap(product_id, hntime)
self.pub_gateway_msg(self.red, chan_list[i], 'HCLOCKS',
adc_per_heap, log, True)
# Number of antennas (NANTS)
n_ants = self.antennas(product_id)
self.pub_gateway_msg(self.red, chan_list[i], 'NANTS', n_ants,
log, True)
# Set PKTSTART to 0 on configure
self.pub_gateway_msg(self.red, chan_list[i], 'PKTSTART', 0,
log, True)
# Number of streams for instance i (NSTRM)
n_streams_per_instance = int(addr_list[i][-1]) + 1
self.pub_gateway_msg(self.red, chan_list[i], 'NSTRM',
n_streams_per_instance, log, True)
# Absolute starting channel for instance i (SCHAN)
s_chan = offset * int(
hnchan) + i * n_streams_per_instance * int(hnchan)
self.pub_gateway_msg(self.red, chan_list[i], 'SCHAN', s_chan,
log, True)
# Destination IP addresses for instance i (DESTIP)
self.pub_gateway_msg(self.red, chan_list[i], 'DESTIP',
addr_list[i], log, True)