def deconfigure(self, description):
"""Deconfigure the current subarray. Release hosts for future use with
other subarrays.
"""
# Fetch hosts allocated to this subarray:
# Note description equivalent to product_id here
array_key = 'coordinator:allocated_hosts:{}'.format(description)
allocated_hosts = self.red.lrange(array_key, 0,
self.red.llen(array_key))
# Build list of Hashpipe-Redis Gateway channels to publish to:
chan_list = self.host_list(HPGDOMAIN, allocated_hosts)
# Send deconfigure message to these specific hosts:
for i in range(len(chan_list)):
self.pub_gateway_msg(self.red, chan_list[i], 'DESTIP', '0.0.0.0',
log, False)
log.info('Subarray {} deconfigured'.format(description))
# Release hosts:
# NOTE: in future, get rid of write_list_redis function and append or pop.
# This will simplify this step.
# Get list of currently available hosts:
free_hosts = self.red.lrange('coordinator:free_hosts', 0,
self.red.llen('coordinator:free_hosts'))
# Append released hosts and write
free_hosts = free_hosts + allocated_hosts
redis_tools.write_list_redis(self.red, 'coordinator:free_hosts',
free_hosts)
# Remove resources from current subarray
self.red.delete('coordinator:allocated_hosts:{}'.format(description))
log.info("Released {} hosts; {} hosts available".format(
len(allocated_hosts), len(free_hosts)))
def main():
log = set_logger()
log.info("Starting Katportal Client")
client = BLKATPortalClient()
signal.signal(signal.SIGINT, lambda sig, frame: on_shutdown())
client.start()
def ip_offset(self, product_id):
"""Get IP offset (for ingesting fractions of the band)
"""
try:
offset = int(self.red.get('{}:ip_offset'.format(product_id)))
if (offset > 0):
log.info('Stream IP offset applied: {}'.format(offset))
except:
log.info("No stream IP offset; defaulting to 0")
offset = 0
return offset
def parse_redis_msg(self, message):
"""Split message only twice - the format is as follows:
message_type:description:value
OR message_type:description (if there is no associated value)
If message does not appear to fit format, return it as is.
If message does f
"""
msg_type = ''
description = ''
value = ''
msg_parts = message['data'].split(':', 2)
if len(msg_parts) < 2:
log.info("Not processing this message: {}".format(message))
else:
msg_type = msg_parts[0]
description = msg_parts[1]
if (len(msg_parts) > 2):
value = msg_parts[2]
return msg_type, description, value
def main(port):
FORMAT = "[ %(levelname)s - %(asctime)s - %(filename)s:%(lineno)s] %(message)s"
# logger = logging.getLogger('reynard')
logging.basicConfig(format=FORMAT)
log.setLevel(logging.DEBUG)
log.info("Starting distributor")
red = redis.StrictRedis(port=port)
ps = red.pubsub(ignore_subscribe_messages=True)
ps.subscribe(CHANNEL)
try:
for message in ps.listen():
msg_parts = message['data'].split(':')
if len(msg_parts) != 2:
log.info("Not processing this message --> {}".format(message))
continue
msg_type = msg_parts[0]
product_id = msg_parts[1]
if msg_type == 'configure':
all_streams = json.loads(json_str_formatter(red.get("{}:streams".format(product_id))))
streams = all_streams[STREAM_TYPE]
addr_list, port = parse_spead_addresses(streams.values()[0])
nstreams = len(addr_list)
if nstreams > NCHANNELS:
log.warning("More than {} ({}) stream addresses found".format(NCHANNELS, nstreams))
for i in range(min(nstreams, NCHANNELS)):
msg = "{}:configure:stream:{}".format(product_id, addr_list[i])
red.publish(CHANNELS[i], msg)
except KeyboardInterrupt:
log.info("Stopping distributor")
sys.exit(0)
except Exception as e:
log.error(e)
sys.exit(1)
def main(port, cfg_file, triggermode):
log = set_logger(log_level=logging.DEBUG)
log.info("Starting Coordinator")
coord = Coordinator(port, cfg_file, triggermode)
signal.signal(signal.SIGINT, lambda sig, frame: on_shutdown())
coord.start()
def on_shutdown():
log.info("Coordinator shutting down.")
sys.exit()
def start(self):
"""Start the coordinator as follows:
- The list of available Hashpipe instances and the number of streams per
instance are retrieved from the main configuration .yml file.
- The number of available instances/hosts is read from the appropriate Redis key.
- Subscriptions are made to the three Redis channels.
- Incoming messages trigger the appropriate function for the stage of the
observation. The contents of the messages (if any) are sent to the
appropriate function.
"""
# Configure coordinator
try:
self.hashpipe_instances, self.streams_per_instance = self.config(
self.cfg_file)
log.info('Configured from {}'.format(self.cfg_file))
except:
log.warning(
'Configuration not updated; old configuration might be present.'
)
# Attempt to read list of available hosts. If key does not exist, recreate from
# config file
free_hosts = self.red.lrange('coordinator:free_hosts', 0,
self.red.llen('coordinator:free_hosts'))
if (len(free_hosts) == 0):
redis_tools.write_list_redis(self.red, 'coordinator:free_hosts',
self.hashpipe_instances)
log.info(
'First configuration - no list of available hosts. Retrieving from config file.'
)
# Subscribe to the required Redis channels.
ps = self.red.pubsub(ignore_subscribe_messages=True)
ps.subscribe(ALERTS_CHANNEL)
ps.subscribe(SENSOR_CHANNEL)
ps.subscribe(TRIGGER_CHANNEL)
# Process incoming Redis messages:
try:
for msg in ps.listen():
msg_type, description, value = self.parse_redis_msg(msg)
# If trigger mode is changed on the fly:
if ((msg_type == 'coordinator') &
(description == 'trigger_mode')):
self.triggermode = value
self.red.set('coordinator:trigger_mode', value)
log.info('Trigger mode set to \'{}\''.format(value))
# If all the sensor values required on configure have been
# successfully fetched by the katportalserver
elif (msg_type == 'conf_complete'):
self.conf_complete(description)
# If the current subarray is deconfigured, instruct processing nodes
# to unsubscribe from their respective streams.
# Only instruct processing nodes in the current subarray to unsubscribe.
# Likewise, release hosts only for the current subarray.
elif (msg_type == 'deconfigure'):
self.deconfigure(description)
# Handle the full data-suspect bitmask, one bit per polarisation
# per F-engine.
elif (msg_type == 'data-suspect'):
self.data_suspect(description, value)
# If the current subarray has transitioned to 'track' - that is,
# the antennas are on source and trackign successfully.
elif (msg_type == 'tracking'):
# Note that the description field is equivalent to product_id
# here:
self.tracking_start(description)
# If the current subarray transitions out of the tracking state:
elif (msg_type == 'not-tracking'):
self.tracking_stop(description)
# If pointing updates are received during tracking
elif ('pos_request_base' in description):
self.pointing_update(msg_type, description, value)
except KeyboardInterrupt:
log.info("Stopping coordinator")
sys.exit(0)
except Exception as e:
log.error(e)
sys.exit(1)
def tracking_start(self, product_id):
"""When a subarray is on source and begins tracking, and the F-engine
data is trustworthy, this function instructs the processing nodes
to begin recording data.
Data recording is initiated by issuing a PKTSTART value to the
processing nodes in question via the Hashpipe-Redis gateway [1].
In addition, other appropriate metadata is published to the
processing nodes via the Hashpipe-Redis gateway.
Args:
product_id (str): name of current subarray.
[1] https://arxiv.org/pdf/1906.07391.pdf
"""
# Get list of allocated hosts for this subarray:
array_key = 'coordinator:allocated_hosts:{}'.format(product_id)
allocated_hosts = self.red.lrange(array_key, 0,
self.red.llen(array_key))
# Build list of Hashpipe-Redis Gateway channels to publish to:
chan_list = self.host_list(HPGDOMAIN, allocated_hosts)
# Send messages to these specific hosts:
datadir = self.datadir(product_id)
for i in range(len(chan_list)):
# Publish DATADIR to gateway
self.pub_gateway_msg(self.red, chan_list[i], 'DATADIR', datadir,
log, False)
# Target information:
target_str, ra_str, dec_str = self.target(product_id)
# SRC_NAME:
self.pub_gateway_msg(self.red, chan_list[i], 'SRC_NAME',
target_str, log, False)
# RA_STR and DEC_STR
self.pub_gateway_msg(self.red, chan_list[i], 'RA_STR', ra_str, log,
False)
self.pub_gateway_msg(self.red, chan_list[i], 'DEC_STR', dec_str,
log, False)
# Set PKTSTART separately after all the above messages have
# all been delivered:
pkt_idx_start = self.get_start_idx(allocated_hosts, PKTIDX_MARGIN, log)
for i in range(len(chan_list)):
self.pub_gateway_msg(self.red, chan_list[i], 'PKTSTART',
pkt_idx_start, log, False)
# Alert via slack:
slack_message = "{}::meerkat:: New recording started for {}!".format(
SLACK_CHANNEL, product_id)
self.red.publish(PROXY_CHANNEL, slack_message)
# If armed, reset triggermode to idle after triggering
# once.
# NOTE: need to fix triggermode retrieval? Perhaps done?
triggermode = self.red.get(
'coordinator:trigger_mode:{}'.format(product_id))
if (triggermode == 'armed'):
self.red.set('coordinator:trigger_mode:{}'.format(product_id),
'idle')
log.info(
'Triggermode set to \'idle\' from \'armed\' from {}'.format(
product_id))
elif ('nshot' in triggermode):
nshot = triggermode.split(':')
n = int(nshot[1]) - 1
triggermode = '{}:{}'.format(nshot[0], n)
# If nshot mode, decrement nshot by one and write to Redis.
self.red.set('coordinator:trigger_mode:{}'.format(product_id),
triggermode)
log.info('Triggermode: n shots remaining: {}'.format(n))
if (n <= 0):
# Set triggermode to idle.
triggermode = 'idle'
self.red.set('coordinator:trigger_mode:{}'.format(product_id),
'idle')
log.info('Triggermode set to \'idle\' from \'nshot\'')
# Set subarray state to 'tracking'
self.red.set('coordinator:tracking:{}'.format(product_id), '1')
def conf_complete(self, description):
"""This function is run when a new subarray is configured and the
katportal_server has retrieved all the associated metadata required
for the processing nodes to ingest and record data from the F-engines.
The required metadata is published to the Hashpipe-Redis gateway in
the key-value pair format described in Appendix B of:
https://arxiv.org/pdf/1906.07391.pdf
Notably, the DESTIP value is set for each processing node - the IP
address of the multicast group it is to join.
Args:
description (str): the second field of the Redis message, which
in this case is the name of the current subarray.
"""
# This is the identifier for the subarray that has completed configuration.
product_id = description
tracking = 0 # Initialise tracking state to 0
log.info('New subarray built: {}'.format(product_id))
# Get IP address offset (if there is one) for ingesting only a specific
# portion of the full band.
offset = self.ip_offset(product_id)
# Initialise trigger mode (idle, armed or auto)
self.red.set('coordinator:trigger_mode:{}'.format(description),
self.triggermode)
log.info('Trigger mode for {} on startup: {}'.format(
description, self.triggermode))
# Generate list of stream IP addresses and publish appropriate messages to
# processing nodes:
addr_list, port, n_addrs, n_red_chans = self.ip_addresses(
product_id, offset)
# Allocate hosts:
free_hosts = self.red.lrange('coordinator:free_hosts', 0,
self.red.llen('coordinator:free_hosts'))
# Allocate hosts for the current subarray:
if (len(free_hosts) == 0):
log.warning(
"No free resources, cannot process data from {}".format(
product_id))
else:
allocated_hosts = free_hosts[0:n_red_chans]
redis_tools.write_list_redis(
self.red, 'coordinator:allocated_hosts:{}'.format(product_id),
allocated_hosts)
# Remove allocated hosts from list of available hosts
# NOTE: in future, append/pop with Redis commands instead of write_list_redis
if (len(free_hosts) < n_red_chans):
log.warning(
"Insufficient resources to process full band for {}".
format(product_id))
free_hosts = [] # Empty
else:
free_hosts = free_hosts[n_red_chans:]
redis_tools.write_list_redis(self.red, 'coordinator:free_hosts',
free_hosts)
log.info('Allocated {} hosts to {}'.format(n_red_chans,
product_id))
# Build list of Hashpipe-Redis Gateway channels to publish to:
chan_list = self.host_list(HPGDOMAIN, allocated_hosts)
# Apply to processing nodes
# NOTE: can we address multiple processing nodes more easily?
for i in range(len(chan_list)):
# Port (BINDPORT)
self.pub_gateway_msg(self.red, chan_list[i], 'BINDPORT', port,
log, True)
# Total number of streams (FENSTRM)
self.pub_gateway_msg(self.red, chan_list[i], 'FENSTRM',
n_addrs, log, True)
# Sync time (UNIX, seconds)
t_sync = self.sync_time(product_id)
self.pub_gateway_msg(self.red, chan_list[i], 'SYNCTIME',
t_sync, log, True)
# Centre frequency (FECENTER)
fecenter = self.centre_freq(product_id)
self.pub_gateway_msg(self.red, chan_list[i], 'FECENTER',
fecenter, log, True)
# Total number of frequency channels (FENCHAN)
n_freq_chans = self.red.get('{}:n_channels'.format(product_id))
self.pub_gateway_msg(self.red, chan_list[i], 'FENCHAN',
n_freq_chans, log, True)
# Coarse channel bandwidth (from F engines)
# Note: no sign information!
# (CHAN_BW)
chan_bw = self.coarse_chan_bw(product_id, n_freq_chans)
self.pub_gateway_msg(self.red, chan_list[i], 'CHAN_BW',
chan_bw, log, True)
# Number of channels per substream (HNCHAN)
hnchan = self.chan_per_substream(product_id)
self.pub_gateway_msg(self.red, chan_list[i], 'HNCHAN', hnchan,
log, True)
# Number of spectra per heap (HNTIME)
hntime = self.spectra_per_heap(product_id)
self.pub_gateway_msg(self.red, chan_list[i], 'HNTIME', hntime,
log, True)
# Number of ADC samples per heap (HCLOCKS)
adc_per_heap = self.samples_per_heap(product_id, hntime)
self.pub_gateway_msg(self.red, chan_list[i], 'HCLOCKS',
adc_per_heap, log, True)
# Number of antennas (NANTS)
n_ants = self.antennas(product_id)
self.pub_gateway_msg(self.red, chan_list[i], 'NANTS', n_ants,
log, True)
# Set PKTSTART to 0 on configure
self.pub_gateway_msg(self.red, chan_list[i], 'PKTSTART', 0,
log, True)
# Number of streams for instance i (NSTRM)
n_streams_per_instance = int(addr_list[i][-1]) + 1
self.pub_gateway_msg(self.red, chan_list[i], 'NSTRM',
n_streams_per_instance, log, True)
# Absolute starting channel for instance i (SCHAN)
s_chan = offset * int(
hnchan) + i * n_streams_per_instance * int(hnchan)
self.pub_gateway_msg(self.red, chan_list[i], 'SCHAN', s_chan,
log, True)
# Destination IP addresses for instance i (DESTIP)
self.pub_gateway_msg(self.red, chan_list[i], 'DESTIP',
addr_list[i], log, True)
def main(config):
log = set_logger(log_level=logging.DEBUG)
log.info("Starting Katportal Client")
client = BLKATPortalClient(config)
signal.signal(signal.SIGINT, lambda sig, frame: on_shutdown())
client.start()
def on_shutdown():
log.info("Shutting Down Katportal Clients")
sys.exit()
def on_shutdown():
# TODO: uncomment when you deploy
# notify_slack("KATPortal module at MeerKAT has halted. Might want to check that!")
log.info("Shutting Down Katportal Clients")
sys.exit()
