def forward(Q, cfg, args, timeout): """To be executed by a local thread. Pops items from the queue and forwards them.""" global comm, rank logger = logging.getLogger("middleman") logger.info( f"Worker: Creating writer_gen: engine={cfg[args.transport_tx]['engine']}" ) # suffix = "" # if not args.debug else '-MM' ch_name = gen_channel_name(cfg["diagnostic"]) writer = writer_gen(cfg[args.transport_tx], ch_name) logger.info(f"Worker: Streaming channel name = {ch_name}") tx_list = [] is_first = True while True: # msg = None try: msg = Q.get(timeout=timeout) logger.info( f"Worker: Receiving from Queue: {msg} - {msg.data.shape}, {msg.data.dtype}" ) if is_first: writer.DefineVariable( gen_var_name(cfg)[rank], msg.data.shape, msg.data.dtype) #if msg.attrs is not None: writer.DefineAttributes("stream_attrs", msg.attrs) logger.info( f"Worker: Defining stream_attrs for forwarded stream: {msg.attrs}" ) writer.Open() logger.info("Worker: Starting forwarding process") is_first = False except queue.Empty: logger.info( "Worker: Empty queue after waiting until time-out. Exiting") break logger.info( f"Worker Forwarding chunk {msg.tstep_idx}. Data = {msg.data.shape}" ) writer.BeginStep() writer.put_data(msg) writer.EndStep() logger.info(f"Worker: Done writing chunk {msg.tstep_idx}.") tx_list.append(msg.tstep_idx) Q.task_done() logger.info(f"Consumed tidx={msg.tstep_idx}") writer.Close() logger.info( f"Worker: Exiting send loop. Transmitted {len(tx_list)} time chunks: {tx_list}" ) logger.info(writer.transfer_stats())
'LoFreq': dobj.lo, 'LensFocus': dobj.sf, 'LensZoom': dobj.sz }) # Trying to load all h5 data into memory print("Loading h5 data into memory") timebase = dobj.time_base_full() tstarts = timebase[::batch_size] tstops = timebase[batch_size - 1::batch_size] _, data = dobj.get_data(trange=[timebase[0], timebase[-1]], norm=1, verbose=0) data_all = np.array_split(data, num_batches, axis=-1) #writer = writer_dataman(shotnr, gen_id) #writer = writer_gen(shotnr, gen_id, cfg["engine"], cfg["params"]) writer = writer_gen(cfg["transport_nersc"]) data_arr = data_all[0] writer.DefineVariable("tstep", np.array(0)) writer.DefineVariable("floats", data_arr) writer.DefineVariable("trange", np.array([0.0, 0.0])) writer.DefineAttributes("cfg", cfg) writer.Open() print("Start sending:") t0 = time.time() for i in range(nstep): if (rank == 0): print("Sending: {0:d} / {1:d}".format(i, nstep)) with writer.step() as w: w.put_data("tstep", np.array(i))
"params": { "IPAddress": "128.55.205.18", "Timeout": "120", "Port": "50001", "TransportMode": "reliable" } } channel_name = gen_channel_name(2408, rank) if rank == 0: logging.info( "==================I am test_reader_multichannel===================") logging.info(f"Channel_name = {channel_name}") w = writer_gen(cfg_transport, channel_name) w.DefineVariable("dummy", (192, 10_000), np.float64) w.DefineVariable("tstep", (1, 1), np.int32) w.Open() w.DefineAttributes("strem_attrs", {"test": "yes"}) for tstep in range(1, 100): if rank == tstep % size: data = np.random.normal(1000.0 * (rank + 1) + tstep, 0.1, size=(192, 10_000)) chunk = twod_chunk(data) tdata = twod_chunk( np.array([[1]], dtype=np.int32) * np.int32(tstep)) # Clunky way to send time-step data :)
with open(args.config, "r") as df: cfg = json.load(df) # Set up the logger with open('configs/logger.yaml', 'r') as f: log_cfg = yaml.safe_load(f.read()) logging.config.dictConfig(log_cfg) logger = logging.getLogger("generator") logger.info("Starting up...") # Instantiate a dataloader dataloader = get_loader(cfg) configname = "transport_tx" if not args.kstar else "transport_rx" logger.info(f"Creating writer_gen: engine={cfg[configname]['engine']}") writer = writer_gen(cfg[configname], gen_channel_name(cfg["diagnostic"])) logger.info(f"Streaming channel name = {gen_channel_name(cfg['diagnostic'])}") # Give the writer hints on what kind of data to transfer writer.DefineVariable( gen_var_name(cfg)[rank], dataloader.get_chunk_shape(), dataloader.dtype) # TODO: Clean up naming conventions for stream attributes logger.info(f"Writing attributes: {dataloader.attrs}") writer.Open() writer.DefineAttributes("stream_attrs", dataloader.attrs) logger.info("Start sending on channel:") batch_gen = dataloader.batch_generator() for nstep, chunk in enumerate(batch_gen): # TODO: Do we want to place filtering in the generator? This would allow us to
with open(args.config, "r") as df: cfg = json.load(df) # Set up the logger with open('configs/logger.yaml', 'r') as f: log_cfg = yaml.safe_load(f.read()) logging.config.dictConfig(log_cfg) logger = logging.getLogger("generator") logger.info("Starting up...") # Instantiate a dataloader dataloader = get_loader(cfg) sectionname = "transport_tx" if not args.kstar else "transport_rx" logger.info(f"Creating writer_gen: engine={cfg[sectionname]['engine']}") writer = writer_gen(cfg[sectionname], gen_channel_name(cfg["diagnostic"])) logger.info(f"Streaming channel name = {gen_channel_name(cfg['diagnostic'])}") # Give the writer hints on what kind of data to transfer writer.DefineVariable(gen_var_name(cfg)[rank], dataloader.get_chunk_shape(), dataloader.dtype) # TODO: Clean up naming conventions for stream attributes logger.info(f"Writing attributes: {dataloader.attrs}") writer.Open() writer.DefineAttributes("stream_attrs", dataloader.attrs) logger.info("Start sending on channel:") batch_gen = dataloader.batch_generator() for nstep, chunk in enumerate(batch_gen):
# Get a data_loader dobj = KstarEcei(shot=shotnr,data_path=datapath,clist=my_channel_range,verbose=False) cfg.update({'TriggerTime':dobj.tt.tolist(),'SampleRate':[dobj.fs/1e3], 'TFcurrent':dobj.itf/1e3,'Mode':dobj.mode, 'LoFreq':dobj.lo,'LensFocus':dobj.sf,'LensZoom':dobj.sz}) # Trying to load all h5 data into memory print("Loading h5 data into memory") timebase = dobj.time_base_full() tstarts = timebase[::batch_size] tstops = timebase[batch_size-1::batch_size] _,data = dobj.get_data(trange=[timebase[0],timebase[-1]],norm=1,verbose=0) data_all = np.array_split(data,num_batches,axis=-1) #writer = writer_dataman(shotnr, gen_id) writer = writer_gen(shotnr, gen_id, cfg["engine"], cfg["params"]) data_arr = data_all[0] varData = writer.DefineVariable("floats",data_arr) varTime = writer.DefineVariable("trange",np.array([0.0,0.0])) writer.DefineAttributes("cfg",cfg) writer.Open() print("Start sending:") t0 = time.time() for i in range(nstep): if(rank == 0): print("Sending: {0:d} / {1:d}".format(i, nstep)) with writer.step() as w: w.put_data(varTime,np.array([tstarts[i],tstops[i]])) w.put_data(varData,data_all[i])