def start_tbb(stations, mode, subbands): logging.info('Starting TBB recording') if mode == 'subband': mode += 's' # tbbctl and rspctl expect the mode to be 'subbands'. rspctl_mode_cmd = [rsp_command, '--tbbmode=%s,%s' % (mode, subbands)] else: rspctl_mode_cmd = [rsp_command, '--tbbmode=%s' % (mode)] cmds = [ rspctl_mode_cmd, [tbb_command, '--mode=%s' % mode], [tbb_command, '--free'], [tbb_command, '--alloc'], [tbb_command, '--record'] ] stations = translate_user_station_string_into_station_list(stations) station_hostname_csv_string = ','.join( stationname2hostname(s) for s in stations) relay = lcurun_command + [station_hostname_csv_string] for cmd in cmds: cmd = relay + cmd logging.info('Executing %s' % ' '.join(cmd)) subprocess.check_call(cmd) time.sleep(2)
def release_tbb(stations): logging.info('Releasing TBB recording') stations = translate_user_station_string_into_station_list(stations) station_hostname_csv_string = ','.join( stationname2hostname(s) for s in stations) relay = lcurun_command + [station_hostname_csv_string] cmd = relay + [tbb_command, '--free'] logging.info('Executing %s' % ' '.join(cmd)) subprocess.check_call(cmd)
def restart_tbb_recording(stations): stations = translate_user_station_string_into_station_list(stations) station_hostname_csv_string = ','.join( stationname2hostname(s) for s in stations) logging.info("Restarting TBB recording on stations %s", stations) relay = lcurun_command + [station_hostname_csv_string] cmd = relay + [tbb_command, "--record"] logging.info("Executing %s" % " ".join(cmd)) subprocess.check_call(cmd) time.sleep(2)
def create_mapping(stations, nodes): """ Map m stations to n nodes. If m>n, some nodes remain unassigned, if n>m, some nodes get assigned multiple times. :param stations: list of station LCUs :param nodes: list of nodes :return: dict mapping stations to nodes, e.g. {station1: node1, station2: node2} """ stations = translate_user_station_string_into_station_list(stations) # zip truncates to shortest list, so make sure there are enough nodes, then map each station to a node logging.info("Mapping stations %s on %s nodes " % (stations, nodes)) nodes *= (len(stations) // len(nodes) + 1) map = dict(list(zip(stations, nodes))) logging.debug('Stations were mapped to nodes as follows: %s' % map) return map
def set_tbb_storage(map): logging.info('Setting TBB storage nodes') for stations, node in map.items(): stations = translate_user_station_string_into_station_list(stations) station_hostname_csv_string = ','.join( stationname2hostname(s) for s in stations) relay = lcurun_command + [station_hostname_csv_string] cmds = [[tbb_command, '--storage=%s' % node], [tbb_command, '--udp']] for cmd in cmds: cmd = relay + cmd logging.info('Executing %s' % ' '.join(cmd)) subprocess.check_call(cmd)
def load_tbb_firmware(stations, mode): logging.info('Loading TBB firmware for mode \"%s\"' % (mode)) # This is hardcoded for now. There is no reliable way to tell from the output # of tbbctl --imageinfo what a firmware can do. Everything there is a string that # gets passed by the --writeimage command. So even the image name can be wrong or # misleading. # So: the ALERT firmware must be in slot #2! if mode == "subband": slot = 2 else: slot = 1 stations = translate_user_station_string_into_station_list(stations) station_hostname_csv_string = ','.join( stationname2hostname(s) for s in stations) logging.info( "It is assumed that the firmware for mode \"%s\" is in slot %d!" % (mode, slot)) relay = lcurun_command + [station_hostname_csv_string] cmd = [tbb_command, '--config=%s' % slot] cmd = relay + cmd logging.info('Executing %s' % ' '.join(cmd)) subprocess.check_call(cmd) # Wait for 60s. The TBBs will reset when a new firmware gets loaded # and that takes some time. wait_time = 60 interval = 10 for sleep_time in range(wait_time, 0, -interval): logging.info( "Waited %ds of %ds for the TBB boards to load the firmware for mode \"%s\"..." % (wait_time - sleep_time, wait_time, mode)) time.sleep(interval) logging.info( "TBBs should now have the firmware for mode \"%s\" loaded. Check the output of the following command!" % (mode)) for board in range(6): cmd = [tbb_command, '--imageinfo=%s' % str(board)] cmd = relay + cmd logging.info('Executing %s' % ' '.join(cmd)) logging.info(check_output_returning_strings(cmd))
def upload_tbb_data(stations, dm, start_time, duration, sub_bands, wait_time, boards): """ Set-up and execute tbbctl READBAND commands that will command TBBs in ALERT mode to upload part of their memory to CEP. :param stations: Only TBBs of these stations will upload their data to CEP. :param dm: The dispersion measure that was set during data recording. :param start_time: Designates the start of the recorded data in the TBB memory which will be uploaded to CEP. Earlier data in TBB memory will not be uploaded. :param duration: The time span for which the data will be uploaded in seconds. :param sub_bands: The list of sub-bands that will be uploaded. :param wait_time: The time that has to be waited before another sub-band upload can be commanded. :param boards: Only these boards will be commanded to upload the spectral data to CEP. """ logging.info("Uploadind TBB data...") stations = translate_user_station_string_into_station_list(stations) # determine number of tbb boards per station: stationlists = split_stations_by_boardnumber(stations) tbb_cmd = tbb_command + " --readband=${board},${channel},%s,%s,%s,%s" # I have to wait until a sub-band has been uploaded. But I can # upload multiple boards/channels in parallel. Hence this can # parallelised down to the sub-bands. for sub_band in sub_bands: (adjusted_start_time, slice_nr) = calculate_adjusted_start_time(dm, start_time, int(sub_band)) # batch handle all stations with same number of boards through lcurun for num_boards in list(stationlists.keys()): logging.debug( "Creating TBB commands for stations with %s boards..." % num_boards) #relay = lcurun_command + [",".join(stationlists[num_boards])] stations_with_num_boards = stationlists[num_boards] # iterate over tbb boards board_list = [] for board in boards: if int(board) <= num_boards: board_list.append(board) else: logging.error( "Stations \"%s\" do not have a board #%s! The stations have only %s boards. Continuing without this board." % (stationlists[num_boards], board, num_boards)) continue cmd_list = [] for channel in range(15): for board in board_list: tbb_cmd = "tbbctl --readband=%s,%s,%s,%s,%s,%s" % ( board, channel, sub_band, adjusted_start_time, slice_nr, duration * 1000) # milliseconds cmd_list.append(tbb_cmd) cmd_list.append("sleep %s" % (wait_time, )) full_cmd = '\"%s\"' % (' ; '.join(cmd_list), ) execute_in_parallel_over_stations( full_cmd, stations_with_num_boards, timeout=24 * 60 * 60 ) # tbb dumps can take a long time.... timeout of 24 hours is ok time.sleep(wait_time) logging.info("Uploading TBB data done.")
def freeze_tbb(stations, dm, timesec, timensec): """ :param stations: comma-separated list of stations :param dm: dispersion measure as float :param timesec: stop time in seconds (int) :param timensec: stop time offset in nanoseconds (int) :return: """ stations = translate_user_station_string_into_station_list(stations) logger.info('Freezing TBB boards for stations: %s', ', '.join(stations)) if dm is not None: logger.info( 'DM %s provided, performing timed stop on individual boards' % dm) # determine number of tbb boards per station: stationlists = split_stations_by_boardnumber(stations) # batch handle all stations with same number of boards through lcurun for num_boards in list(stationlists.keys()): stations_with_num_boards = stationlists[num_boards] logger.info('Handling stations with %s boards: %s', num_boards, stations_with_num_boards) station_str = ','.join(stationlists[num_boards]) relay = lcurun_command + [station_str] slicenr = int(timensec // (5 * 1024)) # -> 5.12 microseconds per slice # string magic to create single cmdline ';' seperated tbbctl commands to set the dispersion measure and the stoptime set_dm_cmd = " ; ".join([ '%s --dispmeas=%s,%s' % (tbb_command, board, dm) for board in range(num_boards) ]) set_stoptime_cmd = " ; ".join([ '%s --stoptimed=%s,%s,%s' % (tbb_command, board, timesec, slicenr) for board in range(num_boards) ]) # apply all commands for each station for cmd in [set_dm_cmd, set_stoptime_cmd]: quoted_cmd = wrap_composite_command(cmd) execute_in_parallel_over_stations(quoted_cmd, stations_with_num_boards, timeout=60, max_parallel=10) # Note: Sander says it is still required to tbbctl --stop in subbands mode, although ICD seems to suggest otherwise, # so we will issue that irrespective of mode in the following. # Note: This is not even close to nanosecond precision, but this is ridiculous outside the driver/firmware anyway... # If we really have to be better than this, we could port the sleepuntil.sh that is apparently floating around # lcuhead to the stations and chain it into the command, so that we are not delayed by lcurun/ssh. # wait for timestamp, then stop all boards on all stations. timestamp = float("%d.%09d" % (timesec, timensec)) if dm is not None: timestamp += 0.32 * dm sleeptime = timestamp - time.time() if sleeptime > 0: logger.info('Waiting %s seconds before stopping TBB boards' % sleeptime) time.sleep(sleeptime) cmd = [tbb_command, '--stop'] execute_in_parallel_over_stations(cmd, stations, timeout=60, max_parallel=10)