def start_tbb(stations, mode, subbands):
logging.info('Starting TBB recording')
if mode == 'subband':
mode += 's'
# tbbctl and rspctl expect the mode to be 'subbands'.
rspctl_mode_cmd = [rsp_command, '--tbbmode=%s,%s' % (mode, subbands)]
else:
rspctl_mode_cmd = [rsp_command, '--tbbmode=%s' % (mode)]
cmds = [
rspctl_mode_cmd, [tbb_command, '--mode=%s' % mode],
[tbb_command, '--free'], [tbb_command, '--alloc'],
[tbb_command, '--record']
]
stations = translate_user_station_string_into_station_list(stations)
station_hostname_csv_string = ','.join(
stationname2hostname(s) for s in stations)
relay = lcurun_command + [station_hostname_csv_string]
for cmd in cmds:
cmd = relay + cmd
logging.info('Executing %s' % ' '.join(cmd))
subprocess.check_call(cmd)
time.sleep(2)
def release_tbb(stations):
logging.info('Releasing TBB recording')
stations = translate_user_station_string_into_station_list(stations)
station_hostname_csv_string = ','.join(
stationname2hostname(s) for s in stations)
relay = lcurun_command + [station_hostname_csv_string]
cmd = relay + [tbb_command, '--free']
logging.info('Executing %s' % ' '.join(cmd))
subprocess.check_call(cmd)
def restart_tbb_recording(stations):
stations = translate_user_station_string_into_station_list(stations)
station_hostname_csv_string = ','.join(
stationname2hostname(s) for s in stations)
logging.info("Restarting TBB recording on stations %s", stations)
relay = lcurun_command + [station_hostname_csv_string]
cmd = relay + [tbb_command, "--record"]
logging.info("Executing %s" % " ".join(cmd))
subprocess.check_call(cmd)
time.sleep(2)
def set_tbb_storage(map):
logging.info('Setting TBB storage nodes')
for stations, node in map.items():
stations = translate_user_station_string_into_station_list(stations)
station_hostname_csv_string = ','.join(
stationname2hostname(s) for s in stations)
relay = lcurun_command + [station_hostname_csv_string]
cmds = [[tbb_command, '--storage=%s' % node], [tbb_command, '--udp']]
for cmd in cmds:
cmd = relay + cmd
logging.info('Executing %s' % ' '.join(cmd))
subprocess.check_call(cmd)
def load_tbb_firmware(stations, mode):
logging.info('Loading TBB firmware for mode \"%s\"' % (mode))
# This is hardcoded for now. There is no reliable way to tell from the output
# of tbbctl --imageinfo what a firmware can do. Everything there is a string that
# gets passed by the --writeimage command. So even the image name can be wrong or
# misleading.
# So: the ALERT firmware must be in slot #2!
if mode == "subband":
slot = 2
else:
slot = 1
stations = translate_user_station_string_into_station_list(stations)
station_hostname_csv_string = ','.join(
stationname2hostname(s) for s in stations)
logging.info(
"It is assumed that the firmware for mode \"%s\" is in slot %d!" %
(mode, slot))
relay = lcurun_command + [station_hostname_csv_string]
cmd = [tbb_command, '--config=%s' % slot]
cmd = relay + cmd
logging.info('Executing %s' % ' '.join(cmd))
subprocess.check_call(cmd)
# Wait for 60s. The TBBs will reset when a new firmware gets loaded
# and that takes some time.
wait_time = 60
interval = 10
for sleep_time in range(wait_time, 0, -interval):
logging.info(
"Waited %ds of %ds for the TBB boards to load the firmware for mode \"%s\"..."
% (wait_time - sleep_time, wait_time, mode))
time.sleep(interval)
logging.info(
"TBBs should now have the firmware for mode \"%s\" loaded. Check the output of the following command!"
% (mode))
for board in range(6):
cmd = [tbb_command, '--imageinfo=%s' % str(board)]
cmd = relay + cmd
logging.info('Executing %s' % ' '.join(cmd))
logging.info(check_output_returning_strings(cmd))
def handle_event(self, voevent_xml, voevent_etree):
if voevent_xml is None or voevent_etree is None:
logger.warning("skipping empty vo_event")
return
identifier = voevent_etree.attrib['ivorn']
logger.info('Handling new ALERT event %s...' % identifier)
try:
logger.info('%s' % voevent_xml)
# check if trigger is allowed to be accepted(marshal permissions etc,)
# Note: So we first have to interpret the event at this stage already just to get the actual time when the
# freeze should happen so we can check if the event should be accepted...?
# Meh. I'd prefer this to happen in the recipe, so that we can separate the marshaling from the handling.
# Anyway, here we go:
parset = parse_parset_from_voevent(voevent_xml)
dm = float(
parset['Observation.TBB.TBBsetting.triggerDispersionMeasure'])
triggerid = parset['Observation.TBB.TBBsetting.triggerId']
reference_frequency = float(
parset['Observation.TBB.TBBsetting.referenceFrequency'])
timestr = parset['Observation.TBB.TBBsetting.time']
reference_time = time.mktime(
dateutil.parser.parse(
timestr).timetuple()) # convert time string to seconds
centertime = translate_arrival_time_to_frequency(
reference_time, reference_frequency, dm, target_frequency=200)
duration = DEFAULT_TBB_DUMP_DURATION # tbb can hold this many seconds
starttime = centertime - duration / 2.0
stoptime = centertime + duration / 2.0
# convert float starttime to second and nanosecond component
# todo: Do we need higher precision up to here? Investigate!
# ...We agreed to try this out first, but it could be problematic fr use cases with extremely short recordings.
stoptime_sec, stoptime_nsec = ("%.9f" % stoptime).split(".")
stoptime_sec = int(stoptime_sec)
stoptime_nsec = int(stoptime_nsec)
if self._tbb_trigger_is_acceptable(stoptime):
logger.info('ALERT event %s passed system pre-flight checks' %
triggerid)
# check if trigger is acceptable for PI
decider = lofar.triggerservices.voevent_decider.ALERTDecider()
if decider.is_acceptable(voevent_etree):
logger.info(
'ALERT event %s passed science pre-flight checks' %
triggerid)
# _send_notification('ALERT Broker', ALERT_BROKER_HOST, self.project, triggerid, voevent_xml) # todo: do we want that? do we want it on same bus?
logger.info(
'ALERT event %s is accepted. Initiating TBB dump: starttime %s, duration %ssec, dm %s'
% (triggerid, starttime, duration, dm))
available_stations = self._determine_station_lists(
)['available']
lcus = [
stationname2hostname(station)
for station in available_stations
]
lcu_str = ','.join(lcus)
# do a fast direct freeze call here, so the boards still contain data for this event.
# if we freeze via rpc/service calls, that takes time, so we might loose precious data from the buffers.
freeze_tbb(lcu_str, dm, stoptime_sec, stoptime_nsec)
# initiate the dumping via an rpc call to the tbbservice which takes care of all bookkeeping.
with TBBRPC.create() as rpc:
rpc.do_tbb_subband_dump(starttime,
duration,
dm,
DEFAULT_TBB_PROJECT,
triggerid,
available_stations,
DEFAULT_TBB_SUBBANDS,
DEFAULT_TBB_BOARDS,
DEFAULT_TBB_CEP_NODES,
voevent_xml,
stoptime=stoptime)
else:
raise Exception(
'ALERT event %s rejected by science pre-flight checks!'
% triggerid)
else:
raise Exception(
'ALERT event %s rejected by system pre-flight checks!' %
triggerid)
except Exception as ex:
logger.exception(
"An error occurred while handling ALERT event %s: %s" %
(identifier, ex))
raise
logger.info('...done handling ALERT event %s...' % identifier)