def start_tbb(stations, mode, subbands):
logging.info('Starting TBB recording')
if mode == 'subband':
mode += 's'
# tbbctl and rspctl expect the mode to be 'subbands'.
rspctl_mode_cmd = [rsp_command, '--tbbmode=%s,%s' % (mode, subbands)]
else:
rspctl_mode_cmd = [rsp_command, '--tbbmode=%s' % (mode)]
cmds = [
rspctl_mode_cmd, [tbb_command, '--mode=%s' % mode],
[tbb_command, '--free'], [tbb_command, '--alloc'],
[tbb_command, '--record']
]
stations = translate_user_station_string_into_station_list(stations)
station_hostname_csv_string = ','.join(
stationname2hostname(s) for s in stations)
relay = lcurun_command + [station_hostname_csv_string]
for cmd in cmds:
cmd = relay + cmd
logging.info('Executing %s' % ' '.join(cmd))
subprocess.check_call(cmd)
time.sleep(2)
def release_tbb(stations):
logging.info('Releasing TBB recording')
stations = translate_user_station_string_into_station_list(stations)
station_hostname_csv_string = ','.join(
stationname2hostname(s) for s in stations)
relay = lcurun_command + [station_hostname_csv_string]
cmd = relay + [tbb_command, '--free']
logging.info('Executing %s' % ' '.join(cmd))
subprocess.check_call(cmd)
def restart_tbb_recording(stations):
stations = translate_user_station_string_into_station_list(stations)
station_hostname_csv_string = ','.join(
stationname2hostname(s) for s in stations)
logging.info("Restarting TBB recording on stations %s", stations)
relay = lcurun_command + [station_hostname_csv_string]
cmd = relay + [tbb_command, "--record"]
logging.info("Executing %s" % " ".join(cmd))
subprocess.check_call(cmd)
time.sleep(2)
def create_mapping(stations, nodes):
"""
Map m stations to n nodes. If m>n, some nodes remain unassigned, if n>m, some nodes get assigned multiple times.
:param stations: list of station LCUs
:param nodes: list of nodes
:return: dict mapping stations to nodes, e.g. {station1: node1, station2: node2}
"""
stations = translate_user_station_string_into_station_list(stations)
# zip truncates to shortest list, so make sure there are enough nodes, then map each station to a node
logging.info("Mapping stations %s on %s nodes " % (stations, nodes))
nodes *= (len(stations) // len(nodes) + 1)
map = dict(list(zip(stations, nodes)))
logging.debug('Stations were mapped to nodes as follows: %s' % map)
return map
def set_tbb_storage(map):
logging.info('Setting TBB storage nodes')
for stations, node in map.items():
stations = translate_user_station_string_into_station_list(stations)
station_hostname_csv_string = ','.join(
stationname2hostname(s) for s in stations)
relay = lcurun_command + [station_hostname_csv_string]
cmds = [[tbb_command, '--storage=%s' % node], [tbb_command, '--udp']]
for cmd in cmds:
cmd = relay + cmd
logging.info('Executing %s' % ' '.join(cmd))
subprocess.check_call(cmd)
def load_tbb_firmware(stations, mode):
logging.info('Loading TBB firmware for mode \"%s\"' % (mode))
# This is hardcoded for now. There is no reliable way to tell from the output
# of tbbctl --imageinfo what a firmware can do. Everything there is a string that
# gets passed by the --writeimage command. So even the image name can be wrong or
# misleading.
# So: the ALERT firmware must be in slot #2!
if mode == "subband":
slot = 2
else:
slot = 1
stations = translate_user_station_string_into_station_list(stations)
station_hostname_csv_string = ','.join(
stationname2hostname(s) for s in stations)
logging.info(
"It is assumed that the firmware for mode \"%s\" is in slot %d!" %
(mode, slot))
relay = lcurun_command + [station_hostname_csv_string]
cmd = [tbb_command, '--config=%s' % slot]
cmd = relay + cmd
logging.info('Executing %s' % ' '.join(cmd))
subprocess.check_call(cmd)
# Wait for 60s. The TBBs will reset when a new firmware gets loaded
# and that takes some time.
wait_time = 60
interval = 10
for sleep_time in range(wait_time, 0, -interval):
logging.info(
"Waited %ds of %ds for the TBB boards to load the firmware for mode \"%s\"..."
% (wait_time - sleep_time, wait_time, mode))
time.sleep(interval)
logging.info(
"TBBs should now have the firmware for mode \"%s\" loaded. Check the output of the following command!"
% (mode))
for board in range(6):
cmd = [tbb_command, '--imageinfo=%s' % str(board)]
cmd = relay + cmd
logging.info('Executing %s' % ' '.join(cmd))
logging.info(check_output_returning_strings(cmd))
def upload_tbb_data(stations, dm, start_time, duration, sub_bands, wait_time,
boards):
"""
Set-up and execute tbbctl READBAND commands that will command TBBs in ALERT mode to upload part of their memory to CEP.
:param stations: Only TBBs of these stations will upload their data to CEP.
:param dm: The dispersion measure that was set during data recording.
:param start_time: Designates the start of the recorded data in the TBB memory which will be uploaded to CEP. Earlier data in TBB memory will not be uploaded.
:param duration: The time span for which the data will be uploaded in seconds.
:param sub_bands: The list of sub-bands that will be uploaded.
:param wait_time: The time that has to be waited before another sub-band upload can be commanded.
:param boards: Only these boards will be commanded to upload the spectral data to CEP.
"""
logging.info("Uploadind TBB data...")
stations = translate_user_station_string_into_station_list(stations)
# determine number of tbb boards per station:
stationlists = split_stations_by_boardnumber(stations)
tbb_cmd = tbb_command + " --readband=${board},${channel},%s,%s,%s,%s"
# I have to wait until a sub-band has been uploaded. But I can
# upload multiple boards/channels in parallel. Hence this can
# parallelised down to the sub-bands.
for sub_band in sub_bands:
(adjusted_start_time,
slice_nr) = calculate_adjusted_start_time(dm, start_time,
int(sub_band))
# batch handle all stations with same number of boards through lcurun
for num_boards in list(stationlists.keys()):
logging.debug(
"Creating TBB commands for stations with %s boards..." %
num_boards)
#relay = lcurun_command + [",".join(stationlists[num_boards])]
stations_with_num_boards = stationlists[num_boards]
# iterate over tbb boards
board_list = []
for board in boards:
if int(board) <= num_boards:
board_list.append(board)
else:
logging.error(
"Stations \"%s\" do not have a board #%s! The stations have only %s boards. Continuing without this board."
% (stationlists[num_boards], board, num_boards))
continue
cmd_list = []
for channel in range(15):
for board in board_list:
tbb_cmd = "tbbctl --readband=%s,%s,%s,%s,%s,%s" % (
board, channel, sub_band, adjusted_start_time,
slice_nr, duration * 1000) # milliseconds
cmd_list.append(tbb_cmd)
cmd_list.append("sleep %s" % (wait_time, ))
full_cmd = '\"%s\"' % (' ; '.join(cmd_list), )
execute_in_parallel_over_stations(
full_cmd, stations_with_num_boards, timeout=24 * 60 * 60
) # tbb dumps can take a long time.... timeout of 24 hours is ok
time.sleep(wait_time)
logging.info("Uploading TBB data done.")
def freeze_tbb(stations, dm, timesec, timensec):
"""
:param stations: comma-separated list of stations
:param dm: dispersion measure as float
:param timesec: stop time in seconds (int)
:param timensec: stop time offset in nanoseconds (int)
:return:
"""
stations = translate_user_station_string_into_station_list(stations)
logger.info('Freezing TBB boards for stations: %s', ', '.join(stations))
if dm is not None:
logger.info(
'DM %s provided, performing timed stop on individual boards' % dm)
# determine number of tbb boards per station:
stationlists = split_stations_by_boardnumber(stations)
# batch handle all stations with same number of boards through lcurun
for num_boards in list(stationlists.keys()):
stations_with_num_boards = stationlists[num_boards]
logger.info('Handling stations with %s boards: %s', num_boards,
stations_with_num_boards)
station_str = ','.join(stationlists[num_boards])
relay = lcurun_command + [station_str]
slicenr = int(timensec //
(5 * 1024)) # -> 5.12 microseconds per slice
# string magic to create single cmdline ';' seperated tbbctl commands to set the dispersion measure and the stoptime
set_dm_cmd = " ; ".join([
'%s --dispmeas=%s,%s' % (tbb_command, board, dm)
for board in range(num_boards)
])
set_stoptime_cmd = " ; ".join([
'%s --stoptimed=%s,%s,%s' %
(tbb_command, board, timesec, slicenr)
for board in range(num_boards)
])
# apply all commands for each station
for cmd in [set_dm_cmd, set_stoptime_cmd]:
quoted_cmd = wrap_composite_command(cmd)
execute_in_parallel_over_stations(quoted_cmd,
stations_with_num_boards,
timeout=60,
max_parallel=10)
# Note: Sander says it is still required to tbbctl --stop in subbands mode, although ICD seems to suggest otherwise,
# so we will issue that irrespective of mode in the following.
# Note: This is not even close to nanosecond precision, but this is ridiculous outside the driver/firmware anyway...
# If we really have to be better than this, we could port the sleepuntil.sh that is apparently floating around
# lcuhead to the stations and chain it into the command, so that we are not delayed by lcurun/ssh.
# wait for timestamp, then stop all boards on all stations.
timestamp = float("%d.%09d" % (timesec, timensec))
if dm is not None:
timestamp += 0.32 * dm
sleeptime = timestamp - time.time()
if sleeptime > 0:
logger.info('Waiting %s seconds before stopping TBB boards' %
sleeptime)
time.sleep(sleeptime)
cmd = [tbb_command, '--stop']
execute_in_parallel_over_stations(cmd,
stations,
timeout=60,
max_parallel=10)