def _cfgjson2dict(self, config_json):
"""
Returns the provided config as dict if a json object or returns the object if it already is a dict.
"""
if isinstance(config_json, str) or isinstance(config_json,
unicode_type):
log.debug("Received config as string:\n {}".format(config_json))
if (not config_json.strip()) or config_json.strip() == '""':
log.debug("String empty, returning empty dict.")
raise Return({})
try:
cfg = json.loads(config_json)
except:
log.error("Error parsing json")
raise FailReply(
"Cannot handle config string {} - Not valid json!".format(
config_json))
elif isinstance(config_json, dict):
log.debug("Received config as dict")
cfg = config_json
else:
raise FailReply(
"Cannot handle config type {}. Config has to bei either json formatted string or dict!"
.format(type(config_json)))
log.debug("Got cfg: {}, {}".format(cfg, type(cfg)))
raise Return(cfg)
def set(self, config_json):
"""
Add the config_json to the current config. Input / output data streams
will be filled with default values if not provided.
The configuration will be rejected if no corresponding value is present
in the default config. A warnign is emitted on type changes.
The final configuration is stored in self._config for access in derived classes.
Returns:
katcp reply object [[[ !configure ok | (fail [error description]) ]]]
"""
log.debug("Updating configuration: '{}'".format(config_json))
cfg = yield self._cfgjson2dict(config_json)
try:
newcfg = updateConfig(self._config, cfg)
# yield self.check_config(newcfg)
self._config = newcfg
log.debug("Updated config: '{}'".format(self._config))
except FailReply as E:
log.error("Check config failed!")
raise E
except KeyError as error:
raise FailReply("Unknown configuration option: {}".format(
str(error)))
except Exception as error:
raise FailReply("Unknown ERROR: {}".format(str(error)))
def deprovision(self):
log.debug("Deprovision {}".format(self.__provisioned))
if self.__provisioned:
try:
provision_playbook = yaml.load(open(self.__provisioned, 'r'))
except Exception as E:
log.error(E)
raise FailReply(
"Error in deprovisioning, cannot load file: {}".format(E))
try:
subplay_futures = []
log.debug(
"Executing playbook as {} seperate subplays in parallel".
format(len(provision_playbook)))
for play in provision_playbook:
subplay_futures.append(
self.__ansible_subplay_executioner(
play, "--tags=stop"))
yield subplay_futures
except Exception as E:
raise FailReply(
"Error in deprovisioning thrown by ansible {}".format(E))
self.__controller = {}
self.__eddDataStore._products.flushdb()
self.__eddDataStore._dataStreams.flushdb()
self.__provisioned = None
self._provision_sensor.set_value("Unprovisioned")
self._configuration_graph.set_value("")
self._config = self._default_config.copy()
def measurement_prepare(self, config_json=""):
"""
"""
log.debug("Received measurement prepare ... ")
try:
cfg = json.loads(config_json)
except:
log.error("Error parsing json")
raise FailReply(
"Cannot handle config string {} - Not valid json!".format(
config_json))
log.debug("Sending measurement_prepare to {} products: {}".format(
len(self.__controller.keys()),
"\n - ".join(self.__controller.keys())))
futures = []
for cid, controller in self.__controller.items():
if cid in cfg:
log.debug("Sending measurement_prepare to {} with {}".format(
cid, cfg[cid]))
futures.append(controller.measurement_prepare(cfg[cid]))
else:
log.debug("Sending measurement_prepare to {} with {}".format(
cid, ""))
futures.append(controller.measurement_prepare({}))
yield futures
def wrapper(self, *args, **kwargs):
log.debug("Decorator managed state change {} -> {}".format(
self.state, target))
if self.state not in allowed:
log.warning(
"State change to {} requested, but state {} not in allowed states! Doing nothing."
.format(target, self.state))
return
if waitfor:
waiting_since = 0
while (self.state != waitfor):
log.warning(
"Waiting since {}s for state {} to start state change to {}, current state {}. Timeout: {}s"
.format(waiting_since, waitfor, target, self.state,
timeout))
if waiting_since > timeout:
raise RuntimeError(
"Waiting since {}s to assume state {} in preparation to change to {}. Aborting."
.format(waiting_since, waitfor, target))
yield sleep(1)
waiting_since += 1
if self.state in abortwaitfor:
raise FailReply(
"Aborting waiting for state: {} due to state: {}".
format(waitfor, self.state))
if intermediate:
self.state = intermediate
try:
if timeout:
yield with_timeout(datetime.timedelta(seconds=timeout),
func(self, *args, **kwargs))
else:
yield func(self, *args, **kwargs)
except StateChange as E:
self.state = str(E)
except Exception as E:
self.state = error
raise E
else:
self.state = target
def _loadBasicConfig(self, basic_config_file):
"""
Actually loads the basic configuration, called by provision and load
basic configuarion requests.
"""
try:
with open(basic_config_file) as cfg:
basic_config = json.load(cfg)
except Exception as E:
raise FailReply("Error reading config {}".format(E))
log.debug("Read basic config: {}".format(
json.dumps(basic_config, indent=4)))
basic_config = self.__sanitizeConfig(basic_config)
yield self._installController(basic_config['products'])
# Retrieve default configs from products and merge with basic config to
# have full config locally.
self._config = self._default_config.copy()
self._config["products"] = {}
for product in basic_config['products'].values():
log.debug("Retrieve basic config for {}".format(product["id"]))
controller = self.__controller[product["id"]]
log.debug("Checking basic config {}".format(
json.dumps(product, indent=4)))
yield controller.set(product)
cfg = yield controller.getConfig()
log.debug("Got: {}".format(json.dumps(cfg, indent=4)))
cfg['data_store'] = self._default_config['data_store']
self._config["products"][cfg['id']] = cfg
self._configUpdated()
def capture_start(self, config_json=""):
"""@brief start the dspsr instance then turn on dada_junkdb instance."""
log.info("Starting EDD backend")
if self.state != "ready":
raise FailReply(
"pipleine state is not in state = ready, but in state = {} - cannot start the pipeline"
.format(self.state))
#return
self.state = "starting"
try:
mkrecvheader_file = tempfile.NamedTemporaryFile(delete=False)
log.debug("Creating mkrec header file: {}".format(
mkrecvheader_file.name))
mkrecvheader_file.write(mkrecv_header)
# DADA may need this
mkrecvheader_file.write("NBIT {}\n".format(
self._config["input_bit_depth"]))
mkrecvheader_file.write("HEAP_SIZE {}\n".format(
self.input_heapSize))
mkrecvheader_file.write("\n#OTHER PARAMETERS\n")
mkrecvheader_file.write("samples_per_block {}\n".format(
self._config["samples_per_block"]))
mkrecvheader_file.write(
"\n#PARAMETERS ADDED AUTOMATICALLY BY MKRECV\n")
mkrecvheader_file.close()
for i, k in enumerate(self._config['enabled_polarizations']):
cfg = self._config.copy()
cfg.update(self._config[k])
if not self._config['dummy_input']:
numa_node = self._config[k]['numa_node']
physcpu = ",".join(numa.getInfo()[numa_node]['cores'][4:9])
cmd = "taskset {physcpu} mkrecv_nt --quiet --header {mkrecv_header} --idx1-step {samples_per_heap} --dada-key {dada_key} \
--sync-epoch {sync_time} --sample-clock {sample_clock} \
--ibv-if {ibv_if} --port {port_rx} {mcast_sources}".format(
mkrecv_header=mkrecvheader_file.name,
physcpu=physcpu,
**cfg)
mk = ManagedProcess(
cmd,
stdout_handler=self._polarization_sensors[k]
["mkrecv_sensors"].stdout_handler)
else:
log.warning(
"Creating Dummy input instead of listening to network!"
)
cmd = "dummy_data_generator -o {dada_key} -b {input_bit_depth} -d 1000 -s 0".format(
**cfg)
mk = ManagedProcess(cmd)
self.mkrec_cmd.append(mk)
self._subprocessMonitor.add(mk, self._subprocess_error)
except Exception as e:
log.error("Error starting pipeline: {}".format(e))
self.state = "error"
else:
self.state = "running"
self.__watchdogs = []
for i, k in enumerate(self._config['enabled_polarizations']):
wd = SensorWatchdog(
self._polarization_sensors[k]["input-buffer-total-write"],
20, self.watchdog_error)
wd.start()
self.__watchdogs.append(wd)
def configure(self, config_json):
"""@brief destroy any ring buffer and create new ring buffer."""
"""
@brief Configure the EDD CCritical PFB
@param config_json A JSON dictionary object containing configuration information
@detail The configuration dictionary is highly flexible. An example is below:
"""
log.info("Configuring EDD backend for processing")
log.debug("Configuration string: '{}'".format(config_json))
if self.state != "idle":
raise FailReply(
'Cannot configure pipeline. Pipeline state {}.'.format(
self.state))
# alternatively we should automatically deconfigure
#yield self.deconfigure()
self.state = "configuring"
# Merge retrieved config into default via recursive dict merge
def __updateConfig(oldo, new):
old = oldo.copy()
for k in new:
if isinstance(old[k], dict):
old[k] = __updateConfig(old[k], new[k])
else:
old[k] = new[k]
return old
if isinstance(config_json, str):
cfg = json.loads(config_json)
elif isinstance(config_json, dict):
cfg = config_json
else:
self.state = "idle" # no states changed
raise FailReply(
"Cannot handle config type {}. Config has to bei either json formatted string or dict!"
.format(type(config_json)))
try:
self._config = __updateConfig(DEFAULT_CONFIG, cfg)
except KeyError as error:
self.state = "idle" # no states changed
raise FailReply("Unknown configuration option: {}".format(
str(error)))
cfs = json.dumps(self._config, indent=4)
log.info("Received configuration:\n" + cfs)
self._edd_config_sensor.set_value(cfs)
# calculate input buffer parameters
self.input_heapSize = self._config["samples_per_heap"] * self._config[
'input_bit_depth'] / 8
nHeaps = self._config["samples_per_block"] / self._config[
"samples_per_heap"]
input_bufferSize = nHeaps * (self.input_heapSize)
log.info('Input dada parameters created from configuration:\n\
heap size: {} byte\n\
heaps per block: {}\n\
buffer size: {} byte'.format(self.input_heapSize, nHeaps,
input_bufferSize))
# calculate output buffer parameters
nSlices = max(
self._config["samples_per_block"] / self._config['fft_length'], 1)
nChannels = self._config['fft_length'] / 2
# on / off spectrum + one side channel item per spectrum
output_bufferSize = nSlices * 2 * nChannels * self._config[
'output_bit_depth'] / 8
output_heapSize = output_bufferSize
#output_bufferSize
rate = output_bufferSize * float(
self._config['sample_clock']
) / self._config[
"samples_per_block"] # in spead documentation BYTE per second and not bit!
rate *= self._config[
"output_rate_factor"] # set rate to (100+X)% of expected rate
self._output_rate_status.set_value(rate / 1E9)
log.info('Output parameters calculated from configuration:\n\
spectra per block: {} \n\
nChannels: {} \n\
buffer size: {} byte \n\
heap size: {} byte\n\
rate ({:.0f}%): {} Gbps'.format(
nSlices, nChannels, output_bufferSize, output_heapSize,
self._config["output_rate_factor"] * 100, rate / 1E9))
self._subprocessMonitor = SubprocessMonitor()
for i, k in enumerate(self._config['enabled_polarizations']):
numa_node = self._config[k]['numa_node']
# configure dada buffer
bufferName = self._config[k]['dada_key']
yield self._create_ring_buffer(input_bufferSize, 64, bufferName,
numa_node)
ofname = bufferName[::-1]
# we write nSlice blocks on each go
yield self._create_ring_buffer(output_bufferSize, 64, ofname,
numa_node)
# Configure + launch
# here should be a smarter system to parse the options from the
# controller to the program without redundant typing of options
physcpu = numa.getInfo()[numa_node]['cores'][0]
cmd = "taskset {physcpu} pfb --input_key={dada_key} --inputbitdepth={input_bit_depth} --fft_length={fft_length} --ntaps={ntaps} -o {ofname} --log_level={log_level} --outputbitdepth={output_bit_depth} --output_type=dada".format(
dada_key=bufferName,
ofname=ofname,
heapSize=self.input_heapSize,
numa_node=numa_node,
physcpu=physcpu,
**self._config)
log.debug("Command to run: {}".format(cmd))
cudaDevice = numa.getInfo()[self._config[k]
["numa_node"]]["gpus"][0]
cli = ManagedProcess(cmd, env={"CUDA_VISIBLE_DEVICES": cudaDevice})
self._subprocessMonitor.add(cli, self._subprocess_error)
self._subprocesses.append(cli)
cfg = self._config.copy()
cfg.update(self._config[k])
if self._config["output_type"] == 'dada':
mksend_header_file = tempfile.NamedTemporaryFile(delete=False)
mksend_header_file.write(mksend_header)
mksend_header_file.close()
timestep = input_bufferSize * 8 / cfg['input_bit_depth']
physcpu = ",".join(numa.getInfo()[numa_node]['cores'][1:4])
cmd = "taskset {physcpu} mksend --header {mksend_header} --nthreads 3 --dada-key {ofname} --ibv-if {ibv_if} --port {port_tx} --sync-epoch {sync_time} --sample-clock {sample_clock} --item1-step {timestep} --item2-list {polarization} --item3-list {fft_length} --item4-list {ntaps} --item6-list {sample_clock} --item5-list {sync_time} --rate {rate} --heap-size {heap_size} {mcast_dest}".format(
mksend_header=mksend_header_file.name,
timestep=timestep,
ofname=ofname,
polarization=i,
nChannels=nChannels,
physcpu=physcpu,
rate=rate,
heap_size=output_heapSize,
**cfg)
elif self._config["output_type"] == 'disk':
cmd = "dada_dbnull -z -k {}".format(ofname)
if not os.path.isdir("./{ofname}".format(ofname=ofname)):
os.mkdir("./{ofname}".format(ofname=ofname))
cmd = "dada_dbdisk -k {ofname} -D ./{ofname} -W".format(
ofname=ofname, **cfg)
else:
log.warning("Selected null output. Not sending data!")
cmd = "dada_dbnull -z -k {}".format()
log.debug("Command to run: {}".format(cmd))
mks = ManagedProcess(cmd)
self._subprocessMonitor.add(mks, self._subprocess_error)
self._subprocesses.append(mks)
self._subprocessMonitor.start()
self.state = "ready"
def configure(self, config_json):
"""
Configure the Skarab PFb Pipeline
Args:
config_json A JSON dictionary object containing configuration information
"""
log.info("Configuring EDD backend for processing")
log.debug("Configuration string: '{}'".format(config_json))
yield self.set(config_json)
# Convert arbitrary output parts to input list
iplist = []
for l in self._config["output_data_streams"].values():
iplist.extend(ip_utils.ipstring_to_list(l["ip"]))
output_string = ip_utils.ipstring_from_list(iplist)
output_ip, Noutput_streams, port = ip_utils.split_ipstring(
output_string)
port = set(
[l["port"] for l in self._config["output_data_streams"].values()])
if len(port) != 1:
raise FailReply("Output data streams have to stream to same port")
# update sync tim based on input
for l in self._config["output_data_streams"].values():
l["sync_time"] = self._config["input_data_streams"][
"polarization_0"]["sync_time"]
self._configUpdated()
cfs = json.dumps(self._config, indent=4)
log.info("Final configuration:\n" + cfs)
if self._config["skip_device_config"]:
log.warning(
"Skipping device configuration because debug mode is active!")
raise Return
log.debug("Setting firmware string")
self._client.setFirmware(
os.path.join(self._config["firmware_directory"],
self._config['firmware']))
log.debug("Connecting to client")
self._client.connect()
if self._config['force_program']:
log.debug("Forcing reprogramming")
yield self._client.program()
yield self._client.initialize()
yield self._client.configure_inputs(
self._config["input_data_streams"]["polarization_0"]["ip"],
self._config["input_data_streams"]["polarization_1"]["ip"],
int(self._config["input_data_streams"]["polarization_0"]["port"]))
yield self._client.configure_output(output_ip, int(port.pop()),
Noutput_streams,
self._config["channels_per_group"],
self._config["board_id"])
yield self._client.configure_quantization_factor(
self._config["initial_quantization_factor"])
yield self._client.configure_fft_shift(
self._config["initial_fft_shift"])
def configure(self, config_json):
"""
Configure the EDD VLBi pipeline
Args:
config_json A JSON dictionary object containing configuration information
"""
log.info("Configuring EDD backend for processing")
log.debug("Configuration string: '{}'".format(config_json))
yield self.set(config_json)
cfs = json.dumps(self._config, indent=4)
log.info("Final configuration:\n" + cfs)
self.__numa_node_pool = []
# remove numa nodes with missing capabilities
for node in numa.getInfo():
if len(numa.getInfo()[node]['gpus']) < 1:
log.debug("Not enough gpus on numa node {} - removing from pool.".format(node))
continue
elif len(numa.getInfo()[node]['net_devices']) < 1:
log.debug("Not enough nics on numa node {} - removing from pool.".format(node))
continue
else:
self.__numa_node_pool.append(node)
log.debug("{} numa nodes remaining in pool after cosntraints.".format(len(self.__numa_node_pool)))
if len(self._config['input_data_streams']) > len(self.__numa_node_pool):
raise FailReply("Not enough numa nodes to process {} polarizations!".format(len(self._config['input_data_streams'])))
self._subprocessMonitor = SubprocessMonitor()
#ToDo: Check that all input data streams have the same format, or allow different formats
for i, streamid in enumerate(self._config['input_data_streams']):
# calculate input buffer parameters
stream_description = self._config['input_data_streams'][streamid]
stream_description["dada_key"] = ["dada", "dadc"][i]
self.add_input_stream_sensor(streamid)
self.input_heapSize = stream_description["samples_per_heap"] * stream_description['bit_depth'] / 8
nHeaps = self._config["samples_per_block"] / stream_description["samples_per_heap"]
input_bufferSize = nHeaps * (self.input_heapSize)
log.info('Input dada parameters created from configuration:\n\
heap size: {} byte\n\
heaps per block: {}\n\
buffer size: {} byte'.format(self.input_heapSize, nHeaps, input_bufferSize))
final_payloads, final_fpss, final_framens = EDD_VDIF_Frame_Size(stream_description['sample_rate'])
if self._config['payload_size'] == 'auto':
payload_size = final_payloads[-1]
else:
payload_size = int(self._config['payload_size'])
log.info('Possible frame payload sizes (add 32 for framesize):')
for k in range(final_payloads.size):
if payload_size == final_payloads[k]:
M = "*"
else:
M = " "
log.info(' {}{:5.0f} byte {:8.0f} frames per sec {:6.3f} nsec/frame'.format(M, final_payloads[k], final_fpss[k], final_framens[k]))
if payload_size not in final_payloads:
log.warning("Payload size {} possibly not conform with VDIF format!".format(payload_size))
# calculate output buffer parameters
size_of_samples = ceil(1. * self._config["samples_per_block"] * 2 / 8.) # byte for two bit mode
number_of_packages = ceil(size_of_samples / float(payload_size))
output_buffer_size = number_of_packages * (payload_size + self._config['vdif_header_size'])
integration_time = self._config["samples_per_block"] / float(stream_description["sample_rate"])
self._integration_time_status.set_value(integration_time)
rate = output_buffer_size/ integration_time # in spead documentation BYTE per second and not bit!
rate *= self._config["output_rate_factor"] # set rate to (100+X)% of expected rate
self._output_rate_status.set_value(rate / 1E9)
log.info('Output parameters calculated from configuration:\n\
total size of data samples: {} byte\n\
number_of_packages: {}\n\
size of output buffer: {} byte\n\
rate ({:.0f}%): {} Gbps'.format(size_of_samples,
number_of_packages, output_buffer_size,
self._config["output_rate_factor"]*100, rate / 1E9))
numa_node = self.__numa_node_pool[i]
log.debug("Associating {} with numa node {}".format(streamid, numa_node))
# configure dada buffer
bufferName = stream_description['dada_key']
yield self._create_ring_buffer(input_bufferSize, 64, bufferName, numa_node)
ofname = bufferName[::-1]
# we write nSlice blocks on each go
yield self._create_ring_buffer(output_buffer_size, 8, ofname, numa_node)
# Configure + launch
physcpu = numa.getInfo()[numa_node]['cores'][0]
thread_id = self._config['thread_id'][streamid]
station_id = self._config['thread_id'][streamid]
cmd = "taskset -c {physcpu} VLBI --input_key={dada_key} --speadheap_size={heapSize} --thread_id={thread_id} --station_id={station_id} --payload_size={payload_size} --sample_rate={sample_rate} --nbits={bit_depth} -o {ofname} --log_level={log_level} --output_type=dada".format(ofname=ofname, heapSize=self.input_heapSize, numa_node=numa_node, physcpu=physcpu, thread_id=thread_id, station_id=station_id, payload_size=payload_size, log_level=self._config['log_level'], **stream_description)
log.debug("Command to run: {}".format(cmd))
cudaDevice = numa.getInfo()[numa_node]['gpus'][0]
cli = ManagedProcess(cmd, env={"CUDA_VISIBLE_DEVICES": cudaDevice})
self._subprocessMonitor.add(cli, self._subprocess_error)
self._subprocesses.append(cli)
cfg = self._config.copy()
cfg.update(stream_description)
ip_range = []
port = set()
for key in self._config["output_data_streams"]:
if streamid in key:
ip_range.append(self._config["output_data_streams"][key]['ip'])
port.add(self._config["output_data_streams"][key]['port'])
if len(port)!=1:
raise FailReply("Output data for one plarization has to be on the same port! ")
if self._config["output_type"] == 'network':
physcpu = ",".join(numa.getInfo()[numa_node]['cores'][1:2])
fastest_nic, nic_params = numa.getFastestNic(numa_node)
log.info("Sending data for {} on NIC {} [ {} ] @ {} Mbit/s".format(streamid, fastest_nic, nic_params['ip'], nic_params['speed']))
cmd = "taskset -c {physcpu} vdif_send --input_key {ofname} --if_ip {ibv_if} --dest_ip {mcast_dest} --port {port_tx} --max_rate {rate}".format(ofname=ofname,
physcpu=physcpu, ibv_if=nic_params['ip'], mcast_dest=" ".join(ip_range), port_tx=port.pop(), rate=rate)
log.debug("Command to run: {}".format(cmd))
elif self._config["output_type"] == 'disk':
ofpath = os.path.join(cfg["output_directory"], ofname)
log.debug("Writing output to {}".format(ofpath))
if not os.path.isdir(ofpath):
os.makedirs(ofpath)
cmd = "dada_dbdisk -k {ofname} -D {ofpath} -W".format(ofname=ofname, ofpath=ofpath, **cfg)
else:
log.warning("Selected null output. Not sending data!")
cmd = "dada_dbnull -z -k {}".format(ofname)
log.debug("Command to run: {}".format(cmd))
mks = ManagedProcess(cmd, env={"CUDA_VISIBLE_DEVICES": cudaDevice})
self._subprocessMonitor.add(mks, self._subprocess_error)
self._subprocesses.append(mks)
self._subprocessMonitor.start()
def configure(self, config_json):
"""
Configure the EDD gated spectrometer
Args:
config_json: A JSON dictionary object containing configuration information
"""
log.info("Configuring EDD backend for processing")
log.debug("Configuration string: '{}'".format(config_json))
yield self.set(config_json)
cfs = json.dumps(self._config, indent=4)
log.info("Final configuration:\n" + cfs)
self.__numa_node_pool = []
# remove numa nodes with missing capabilities
for node in numa.getInfo():
if len(numa.getInfo()[node]['gpus']) < 1:
log.debug(
"Not enough gpus on numa node {} - removing from pool.".
format(node))
continue
elif len(numa.getInfo()[node]['net_devices']) < 1:
log.debug(
"Not enough nics on numa node {} - removing from pool.".
format(node))
continue
else:
self.__numa_node_pool.append(node)
log.debug("{} numa nodes remaining in pool after constraints.".format(
len(self.__numa_node_pool)))
if len(self.__numa_node_pool) == 0:
if self._config['nonfatal_numacheck']:
log.warning("Not enough numa nodes to process data!")
self.__numa_node_pool = numa.getInfo().keys()
else:
raise FailReply("Not enough numa nodes to process data!")
self._subprocessMonitor = SubprocessMonitor()
if len(self._config['input_data_streams']) != 2:
raise FailReply("Require 2 polarization input, got {}".format(
len(self._config['input_data_streams'])))
log.debug("Merging ip ranges")
self.stream_description = copy.deepcopy(
self._config['input_data_streams'].items()[0][1])
self.stream_description["ip"] += ",{}".format(
self._config['input_data_streams'].items()[1][1]["ip"])
log.debug("Merged ip ranges: {}".format(self.stream_description["ip"]))
self.input_heapSize = self.stream_description[
"samples_per_heap"] * self.stream_description['bit_depth'] // 8
nHeaps = self._config["samples_per_block"] // self.stream_description[
"samples_per_heap"]
input_bufferSize = nHeaps * (self.input_heapSize + 64 // 8)
log.info('Input dada parameters created from configuration:\n\
heap size: {} byte\n\
heaps per block: {}\n\
buffer size: {} byte'.format(self.input_heapSize, nHeaps,
input_bufferSize))
# calculate output buffer parameters
nSlices = max(
self._config["samples_per_block"] // 2 //
self._config['fft_length'] // self._config['naccumulate'], 1)
nChannels = self._config['fft_length'] // 2 + 1
# on / off spectrum + one side channel item per spectrum
output_bufferSize = nSlices * (8 * (nChannels * 32 // 8 + 2 * 8))
output_heapSize = nChannels * 32 // 8
integrationTime = self._config['fft_length'] * self._config[
'naccumulate'] / (float(self.stream_description["sample_rate"]))
self._integration_time_status.set_value(integrationTime)
rate = output_heapSize / integrationTime # in spead documentation BYTE per second and not bit!
rate *= self._config[
"output_rate_factor"] # set rate to (100+X)% of expected rate
self._output_rate_status.set_value(rate / 1E9)
log.info('Output parameters calculated from configuration:\n\
spectra per block: {} \n\
nChannels: {} \n\
buffer size: {} byte \n\
integrationTime : {} s \n\
heap size: {} byte\n\
rate ({:.0f}%): {} Gbps'.format(
nSlices, nChannels, output_bufferSize, integrationTime,
output_heapSize, self._config["output_rate_factor"] * 100,
rate / 1E9))
numa_node = self.__numa_node_pool[0]
log.debug("Associating with numa node {}".format(numa_node))
# configure dada buffer
yield self._create_ring_buffer(input_bufferSize, 64, self.__dada_key,
numa_node)
ofname = self.__dada_key[::-1]
# we write nSlice blocks on each go
yield self._create_ring_buffer(output_bufferSize, 8 * nSlices, ofname,
numa_node)
## specify all subprocesses
self.__coreManager = CoreManager(numa_node)
self.__coreManager.add_task("gated_spectrometer", 1)
N_inputips = 0
for p in self.stream_description["ip"].split(','):
N_inputips += len(ipstring_to_list(p))
log.debug("Found {} input ips".format(N_inputips))
if not self._config["dummy_input"]:
self.__coreManager.add_task("mkrecv",
N_inputips + 1,
prefere_isolated=True)
if self._config["output_type"] == "network":
self.__coreManager.add_task("mksend", 2)
# Configure + launch
cmd = "taskset -c {physcpu} gated_spectrometer --nsidechannelitems=1 --input_key={dada_key} --speadheap_size={heapSize} --selected_sidechannel=0 --nbits={bit_depth} --fft_length={fft_length} --naccumulate={naccumulate} -o {ofname} --log_level={log_level} --output_format=Stokes --input_polarizations=Dual --output_type=dada".format(
dada_key=self.__dada_key,
ofname=ofname,
heapSize=self.input_heapSize,
numa_node=numa_node,
bit_depth=self.stream_description['bit_depth'],
physcpu=self.__coreManager.get_coresstr('gated_spectrometer'),
**self._config)
log.debug("Command to run: {}".format(cmd))
cudaDevice = numa.getInfo()[numa_node]['gpus'][0]
gated_cli = ManagedProcess(cmd,
env={"CUDA_VISIBLE_DEVICES": cudaDevice})
log.debug("Visble Cuda Device: {}".format(cudaDevice))
self._subprocessMonitor.add(gated_cli, self._subprocess_error)
self._subprocesses.append(gated_cli)
cfg = self._config.copy()
cfg.update(self.stream_description)
cfg["dada_key"] = self.__dada_key
ip_range = []
port = set()
for key in self._config["output_data_streams"]:
ip_range.append(self._config["output_data_streams"][key]['ip'])
port.add(self._config["output_data_streams"][key]['port'])
if len(port) != 1:
raise FailReply("Output data has to be on the same port! ")
if self._config["output_type"] == 'network':
mksend_header_file = tempfile.NamedTemporaryFile(delete=False)
mksend_header_file.write(_mksend_header)
mksend_header_file.close()
nhops = len(ip_range)
timestep = cfg["fft_length"] * cfg["naccumulate"]
#select network interface
fastest_nic, nic_params = numa.getFastestNic(numa_node)
heap_id_start = 0 #2 * i # two output spectra per pol
log.info("Sending data on NIC {} [ {} ] @ {} Mbit/s".format(
fastest_nic, nic_params['ip'], nic_params['speed']))
cmd = "taskset -c {physcpu} mksend --header {mksend_header} --heap-id-start {heap_id_start} --dada-key {ofname} --ibv-if {ibv_if} --port {port_tx} --sync-epoch {sync_time} --sample-clock {sample_rate} --item1-step {timestep} --item4-list {fft_length} --item6-list {sync_time} --item7-list {sample_rate} --item8-list {naccumulate} --rate {rate} --heap-size {heap_size} --nhops {nhops} {mcast_dest}".format(
mksend_header=mksend_header_file.name,
heap_id_start=heap_id_start,
timestep=timestep,
ofname=ofname,
nChannels=nChannels,
physcpu=self.__coreManager.get_coresstr('mksend'),
integrationTime=integrationTime,
rate=rate,
nhops=nhops,
heap_size=output_heapSize,
ibv_if=nic_params['ip'],
mcast_dest=" ".join(ip_range),
port_tx=port.pop(),
**cfg)
log.debug("Command to run: {}".format(cmd))
elif self._config["output_type"] == 'disk':
ofpath = os.path.join(cfg["output_directory"], ofname)
log.debug("Writing output to {}".format(ofpath))
if not os.path.isdir(ofpath):
os.makedirs(ofpath)
cmd = "dada_dbdisk -k {ofname} -D {ofpath} -W".format(
ofname=ofname, ofpath=ofpath, **cfg)
else:
log.warning("Selected null output. Not sending data!")
cmd = "dada_dbnull -z -k {}".format(ofname)
mks = ManagedProcess(cmd, env={"CUDA_VISIBLE_DEVICES": cudaDevice})
self._subprocessMonitor.add(mks, self._subprocess_error)
self._subprocesses.append(mks)
self._subprocessMonitor.start()
def provision(self, description):
"""
Provision the EDD using the provided provision description.
Args:
description: description of the provision. This has to be a string of format
- ::`NAME` to load NAME.json and NAME.yml, or
- ::`NAME1.yml;NAME2.json` to load different yml / json configs
"""
os.chdir(self.__edd_ansible_git_repository_folder)
log.debug("Provision description {} from directory {}".format(
description, os.getcwd()))
if description.startswith('"'):
description = description.lstrip('"')
description = description.rstrip('"')
descr_subfolder = "provison_descriptions"
if ";" in description:
description = description.split(';')
if description[0].endswith("yml"):
playbook_file = os.path.join(descr_subfolder, description[0])
basic_config_file = os.path.join(descr_subfolder,
description[1])
else:
playbook_file = os.path.join(descr_subfolder, description[1])
basic_config_file = os.path.join(descr_subfolder,
description[0])
else:
playbook_file = os.path.join(descr_subfolder, description + ".yml")
basic_config_file = os.path.join(descr_subfolder,
description + ".json")
log.debug("Loading provision description files: {} and {}".format(
playbook_file, basic_config_file))
if not os.path.isfile(playbook_file):
raise FailReply(
"cannot find playbook file {}".format(playbook_file))
if not os.path.isfile(basic_config_file):
raise FailReply(
"cannot find config file {}".format(basic_config_file))
try:
provision_playbook = yaml.load(open(playbook_file, 'r'))
except Exception as E:
log.error(E)
raise FailReply(
"Error in provisioning, cannot load file: {}".format(E))
try:
subplay_futures = []
log.debug("Executing playbook as {} seperate subplays in parallel".
format(len(provision_playbook)))
self.__provisioned = playbook_file
self._provision_sensor.set_value(playbook_file)
for play in provision_playbook:
subplay_futures.append(
self.__ansible_subplay_executioner(play))
yield subplay_futures
except Exception as E:
raise FailReply(
"Error in provisioning thrown by ansible {}".format(E))
yield self._loadBasicConfig(basic_config_file)
def configure(self, config_json):
"""
Configure the EDD backend
Args:
config_json: A JSON dictionary object containing configuration information
"""
log.info("Configuring EDD backend for processing")
#log.info("Resetting data streams")
#TODo: INterface? Decide if this is always done
#self.__eddDataStore._dataStreams.flushdb()
log.debug("Received configuration string: '{}'".format(config_json))
try:
cfg = json.loads(config_json)
except:
log.error("Error parsing json")
raise FailReply(
"Cannot handle config string {} - Not valid json!".format(
config_json))
if not self.__provisioned:
log.debug("Not provisioned. Using full config.")
# Do not use set here, as there might not be a basic config from
# provisioning
cfg = self.__sanitizeConfig(cfg)
self._config = cfg
else:
yield EDDPipeline.set(self, cfg)
yield self._installController(self._config['products'])
cfs = json.dumps(self._config, indent=4)
log.debug("Starting configuration:\n" + cfs)
# Data streams are only filled in on final configure as they may
# require data from the configure command of previous products. As example, the packetizer
# data stream has a sync time that is propagated to other components
# The components are thus configured following the dependency tree,
# which is a directed acyclic graph (DAG)
log.debug("Build DAG from config")
dag = nx.DiGraph()
for product, product_config in self._config['products'].items():
log.debug("Adding node: {}".format(product))
dag.add_node(product)
if "input_data_streams" in product_config:
for stream in value_list(product_config["input_data_streams"]):
if not stream["source"]:
log.warning(
"Ignoring stream without source for DAG from {}".
format(product))
continue
source_product = stream["source"].split(":")[0]
if source_product not in self._config['products']:
raise FailReply(
"{} requires data stream of unknown product {}".
format(product, stream["source"]))
log.debug("Connecting: {} -> {}".format(
source_product, product))
dag.add_edge(source_product, product)
log.debug("Checking for loops in graph")
try:
cycle = nx.find_cycle(dag)
FailReply("Cycle detected in dependency graph: {}".format(cycle))
except nx.NetworkXNoCycle:
log.debug("No loop on graph found")
pass
graph = "\n".join(
[" {} --> {}".format(k[0], k[1]) for k in dag.edges()])
log.info("Dependency graph of products:\n{}".format(graph))
self._configuration_graph.set_value(graph)
configure_results = {}
configure_futures = []
@coroutine
def __process_node(node):
"""
Wrapper to parallelize configuration of nodes. Any Node will wait for its predecessors to be done.
"""
#Wait for all predecessors to be finished
log.debug("DAG Processing {}: Waiting for {} predecessors".format(
node, len(list(dag.predecessors(node)))))
for pre in dag.predecessors(node):
log.debug('DAG Processing {}: waiting for {}'.format(
node, pre))
while not pre in configure_results:
# python3 asyncio coroutines would not run until awaited,
# so we could build the graph up front and then execute it
# without waiting
yield tornado.gen.sleep(0.5)
log.debug('DAG Processing {}: Predecessor {} done.'.format(
node, pre))
if not configure_results[pre]:
log.error(
'DAG Processing {}: fails due to error in predecessor {}'
.format(node, pre))
configure_results[node] = False
raise Return
log.debug('DAG Processing {}: Predecessor {} was successfull.'.
format(node, pre))
log.debug("DAG Processing {}: All predecessors done.".format(node))
try:
log.debug(
"DAG Processing {}: Checking input data streams for updates."
.format(node))
if "input_data_streams" in self._config['products'][node]:
log.debug(
'DAG Processing {}: Update input streams'.format(node))
for stream in value_list(self._config['products'][node]
["input_data_streams"]):
product_name, stream_name = stream["source"].split(":")
stream.update(self._config['products'][product_name]
["output_data_streams"][stream_name])
log.debug('DAG Processing {}: Set Final config'.format(node))
yield self.__controller[node].set(
self._config['products'][node])
log.debug(
'DAG Processing {}: Staring configuration'.format(node))
yield self.__controller[node].configure()
log.debug(
"DAG Processing {}: Getting updated config".format(node))
cfg = yield self.__controller[node].getConfig()
log.debug("Got: {}".format(json.dumps(cfg, indent=4)))
self._config["products"][node] = cfg
except Exception as E:
log.error(
'DAG Processing: {} Exception cought during configuration:\n {}:{}'
.format(node,
type(E).__name__, E))
configure_results[node] = False
else:
log.debug(
'DAG Processing: {} Successfully finished configuration'.
format(node))
configure_results[node] = True
log.debug("Creating processing futures")
configure_futures = [__process_node(node) for node in dag.nodes()]
yield configure_futures
self._configUpdated()
log.debug("Final configuration:\n '{}'".format(
json.dumps(self._config, indent=2)))
failed_prcts = [
k for k in configure_results if not configure_results[k]
]
if failed_prcts:
raise FailReply("Failed products: {}".format(
",".join(failed_prcts)))
log.info("Updating data streams in database")
for productname, product in self._config["products"].items():
log.debug(" - Checking {}".format(productname))
if "output_data_streams" in product and isinstance(
product["output_data_streams"], dict):
for stream, streamcfg in product["output_data_streams"].items(
):
key = "{}:{}".format(productname, stream)
self.__eddDataStore.addDataStream(key, streamcfg)
log.info("Successfully configured EDD")
raise Return("Successfully configured EDD")