def __init__(self, task_config, fft_config):
"""Initialize the object with a fixed channel list, a fixed name of the analysis to be performed
and a fixed set of parameters for the analysis routine.
Inputs:
=======
channel_range: list of strings, defines the name of the channels. This should probably match the
name of the channels in the BP file.
task_config: dict, defines parameters of the analysis to be performed
fft_config dict, gives parameters of the fourier-transformed data
"""
# Stores the description of the task. This can be arbitrary
self.description = task_config["description"]
# Stores the name of the analysis we are going to execute
self.analysis = task_config["analysis"]
# Parse the reference and cross channels.
try:
kwargs = task_config["kwargs"]
# These channels serve as reference for the spectral diagnostics
self.ref_channels = channel_range.from_str(
kwargs["ref_channels"][0])
# These channels serve as the cross-data for the spectrail diagnostics
self.x_channels = channel_range.from_str(kwargs["x_channels"][0])
except KeyError:
self.kwargs = None
self.fft_config = fft_config
self.storage_scheme = {
"ref_channels": self.ref_channels.to_str(),
"cross_channels": self.x_channels.to_str()
}
def __init__(self, cfg: dict):
print(
"++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++"
)
print(
"+ +"
)
print(
"+ readers.py is deprecated. Use reader_mpi.py or reader_nompi.py +"
)
print(
"+ +"
)
print(
"++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++"
)
comm = MPI.COMM_WORLD
self.rank = comm.Get_rank()
self.size = comm.Get_size()
self.logger = logging.getLogger("simple")
self.shotnr = cfg["shotnr"]
self.adios = adios2.ADIOS(MPI.COMM_WORLD)
self.IO = self.adios.DeclareIO(gen_io_name(self.shotnr))
self.reader = None
# Generate a descriptive channel name
chrg = channel_range.from_str(
cfg["transport"]["channel_range"][self.rank])
self.channel_name = gen_channel_name_v2(self.shotnr, chrg.to_str())
self.logger.info(f"reader_base: channel_name = {self.channel_name}")
def __init__(self, cfg: dict, shotnr: int = 18431):
"""Generates a reader for KSTAR ECEI data.
Parameters:
-----------
cfg: delta config dictionary
"""
self.adios = adios2.ADIOS()
self.logger = logging.getLogger("simple")
self.shotnr = shotnr
self.IO = self.adios.DeclareIO(gen_io_name(self.shotnr))
self.reader = None
# Generate a descriptive channel name
if len(cfg["channel_range"]) > 1:
self.logger.error(
"reader_base is not using MPI. The following channel_ranges are ignored:"
)
for crg in cfg["channel_range"][1:]:
self.logger.error(f"Ignoring channel range {crg}")
self.chrg = channel_range.from_str(cfg["channel_range"][0])
self.channel_name = gen_channel_name_v3(cfg["datapath"], self.shotnr,
self.chrg.to_str())
self.logger.info(f"reader_base: channel_name = {self.channel_name}")
def __init__(self, task_config, fft_config, ecei_config):
"""Initialize the object with a fixed channel list, a fixed name of the analysis to be performed
and a fixed set of parameters for the analysis routine.
Inputs:
=======
channel_range: list of strings, defines the name of the channels. This should probably match the
name of the channels in the BP file.
task_config: dict, defines parameters of the analysis to be performed
fft_config dict, gives parameters of the fourier-transformed data
"""
# Stores the description of the task. This can be arbitrary
self.description = task_config["description"]
# Stores the name of the analysis we are going to execute
self.analysis = task_config["analysis"]
# Parse the reference and cross channels.
self.ref_channels = channel_range.from_str(task_config["ref_channels"])
# These channels serve as the cross-data for the spectral diagnostics
self.cmp_channels = channel_range.from_str(task_config["cmp_channels"])
self.task_config = task_config
self.fft_config = fft_config
self.ecei_config = ecei_config
self.storage_scheme = {
"ref_channels": self.ref_channels.to_str(),
"cmp_channels": self.cmp_channels.to_str()
}
# Construct a list of unique channels
# F.ex. we have ref_channels [(1,1), (1,2), (1,3)] and cmp_channels = [(1,1), (1,2)]
# The unique list of channels is then
# (1,1) x (1,1), (1,1) x (1,2)
# (1,2) x (1,2) !!! Omit (1,2) x (1,1)
# (1,3) x (1,1)
# (1,3) x (1,2)
channel_pairs = [
channel_pair(cr, cx) for cr in self.ref_channels
for cx in self.cmp_channels
]
# Make a list, so that we don't exhause the iterator after the first call.
self.unique_channels = list(
more_itertools.distinct_combinations(channel_pairs, 1))
self.channel_chunk_size = task_config["channel_chunk_size"]
def __init__(self, cfg: dict, shotnr: int=18431):
self.logger = logging.getLogger("simple")
self.shotnr = shotnr
self.adios = adios2.ADIOS()
self.IO = self.adios.DeclareIO(gen_io_name(0))
self.writer = None
# Adios2 variable that is defined in DefineVariable
self.variable = None
# The shape used to define self.variable
self.shape = None
# Generate a descriptive channel name
self.chrg = channel_range.from_str(cfg["channel_range"][0])
self.channel_name = gen_channel_name_v3(cfg["datapath"], self.shotnr, self.chrg.to_str())
self.logger.info(f"writer_base: channel_name = {self.channel_name}")
def __init__(self, cfg):
"""
Old: filename: str, ch_range: channel_range, chunk_size:int):
Inputs:
=======
cfg: Delta configuration with loader and ECEI section.
"""
self.ch_range = channel_range.from_str(
cfg["source"]["channel_range"][0])
# Create a list of paths in the HDF5 file, corresponding to the specified channels
self.filename = cfg["source"]["source_file"]
self.chunk_size = cfg["source"]["chunk_size"]
self.ecei_cfg = cfg["ECEI_cfg"]
self.num_chunks = 500
self.current_chunk = 0
self.tnorm = cfg["ECEI_cfg"]["t_norm"]
self.got_normalization = False
def __init__(self, cfg: dict, shotnr: int=18431):
"""Generates a reader for KSTAR ECEI data.
Parameters:
-----------
cfg: delta config dictionary
"""
comm = MPI.COMM_SELF
self.rank = comm.Get_rank()
self.size = comm.Get_size()
# This should be MPI.COMM_SELF, not MPI.COMM_WORLD
self.adios = adios2.ADIOS(MPI.COMM_SELF)
self.logger = logging.getLogger("simple")
self.shotnr = shotnr
self.IO = self.adios.DeclareIO(gen_io_name(self.shotnr))
# Keeps track of the past chunk sizes. This allows to construct a dummy time base
self.reader = None
# Generate a descriptive channel name
self.chrg = channel_range.from_str(cfg["channel_range"][self.rank])
self.channel_name = gen_channel_name_v3(cfg["datapath"], self.shotnr, self.chrg.to_str())
self.logger.info(f"reader_base: channel_name = {self.channel_name}")
def main():
comm = MPI.COMM_WORLD
rank = comm.Get_rank()
size = comm.Get_size()
# Parse command line arguments and read configuration file
parser = argparse.ArgumentParser(
description="Receive data and dispatch analysis tasks to a mpi queue")
parser.add_argument('--config',
type=str,
help='Lists the configuration file',
default='configs/config_null.json')
parser.add_argument('--benchmark', action="store_true")
args = parser.parse_args()
global cfg
with open(args.config, "r") as df:
cfg = json.load(df)
df.close()
# Load logger configuration from file:
# http://zetcode.com/python/logging/
with open("configs/logger.yaml", "r") as f:
log_cfg = yaml.safe_load(f.read())
logging.config.dictConfig(log_cfg)
logger = logging.getLogger('simple')
# Create a global executor
#executor = concurrent.futures.ThreadPoolExecutor(max_workers=60)
executor_fft = MPIPoolExecutor(max_workers=16)
executor_anl = MPIPoolExecutor(max_workers=16)
#executor = MPIPoolExecutor(max_workers=120)
adios2_varname = channel_range.from_str(
cfg["transport_nersc"]["channel_range"][0])
cfg["run_id"] = ''.join(
random.choice(string.ascii_uppercase + string.digits)
for _ in range(6))
cfg["run_id"] = "ABC128"
cfg["storage"]["run_id"] = cfg["run_id"]
logger.info(f"Starting run {cfg['run_id']}")
# Instantiate a storage backend and store the run configuration and task configuration
if cfg['storage']['backend'] == "numpy":
store_backend = backends.backend_numpy(cfg['storage'])
elif cfg['storage']['backend'] == "mongo":
store_backend = backends.backend_mongodb(cfg["storage"])
elif cfg['storage']['backend'] == "null":
store_backend = backends.backend_null(cfg['storage'])
else:
raise NameError(
f"Unknown storage backend requested: {cfg['storage']['backend']}")
store_backend.store_one({"run_id": cfg['run_id'], "run_config": cfg})
logger.info(f"Stored one")
# Create ADIOS reader object
reader = reader_gen(cfg["transport_nersc"])
task_list = task_list_spectral(executor_anl, executor_fft,
cfg["task_list"], cfg["fft_params"],
cfg["ECEI_cfg"], cfg["storage"])
#task_list = task_list_spectral(executor, cfg["task_list"], cfg["fft_params"], cfg["ECEI_cfg"], cfg["storage"])
dq = queue.Queue()
msg = None #
tic_main = timeit.default_timer()
workers = []
for _ in range(4):
worker = threading.Thread(target=consume, args=(dq, task_list))
worker.start()
workers.append(worker)
# reader.Open() is blocking until it opens the data file or receives the
# data stream. Put this right before entering the main loop
logger.info(f"{rank} Waiting for generator")
reader.Open()
logger.info(f"Starting main loop")
rx_list = []
while True:
stepStatus = reader.BeginStep()
if stepStatus:
# Read data
stream_data = reader.Get(adios2_varname, save=False)
rx_list.append(reader.CurrentStep())
# Generate message id and publish
msg = AdiosMessage(tstep_idx=reader.CurrentStep(),
data=stream_data)
dq.put_nowait(msg)
logger.info(f"Published tidx {msg.tstep_idx}")
reader.EndStep()
else:
logger.info(f"Exiting: StepStatus={stepStatus}")
break
if reader.CurrentStep() > 100:
break
dq.join()
logger.info("Queue joined")
logger.info("Exiting main loop")
for thr in workers:
thr.join()
logger.info("Workers have joined")
# Shotdown the executioner
executor_anl.shutdown(wait=True)
executor_fft.shutdown(wait=True)
#executor.shutdown(wait=True)
toc_main = timeit.default_timer()
logger.info(
f"Run {cfg['run_id']} finished in {(toc_main - tic_main):6.4f}s")
logger.info(f"Processed {len(rx_list)} time_chunks: {rx_list}")
cfg = json.load(df)
with open('configs/logger.yaml', 'r') as f:
log_cfg = yaml.safe_load(f.read())
logging.config.dictConfig(log_cfg)
logger = logging.getLogger("generator")
datapath = cfg["transport_nersc"]["datapath"]
nstep = cfg["transport_nersc"]["nstep"]
shotnr = cfg["shotnr"]
# Enforce 1:1 mapping of channels and tasks
assert (len(cfg["transport_nersc"]["channel_range"]) == size)
# Channels this process is reading
ch_rg = channel_range.from_str(cfg["transport_nersc"]["channel_range"][rank])
# Hard-code the total number of data points
#data_pts = int(5e6)
# Hard-code number of data points per data packet
#data_per_batch = int(1e1)
# Calculate the number of required data batches we send over the channel
#num_batches = data_pts // data_per_batch
# Get a data_loader
logger.info("Loading h5 data into memory")
dl = loader_ecei(cfg)
dl.cache()
batch_gen = dl.batch_generator()
logger.info(
from analysis.channels import channel, channel_range
ch_start = channel("L", 1, 3)
ch_end = channel("L", 4, 5)
# We often need a rectangular selection. This mode refers to the rectangular configuration
# of the channel views. Basically, we specify a recatngle by giving a lower left corner
# and an upper right corner
# Then iterate within the rectangular bounds defined by this rectangle.
clist = (ch_start, ch_end, mode="rectangle")
for c in clist:
print(c)
# Test naive iteration where we just take the outer product of two lists:
crg1 = channel_range.from_str("L0101-0104")
crg2 = channel_range.from_str("L0101-0104")
print("Testing naive iteration...")
res = ["{0:d} x {1:d}".format(c1.idx(), c2.idx()) for c1 in clist1 for c2 in clist2]
print(res)
# Test iteration over groups of channels.
# In the spectral analysis we often want to iterate over a list of unique channel pairs.
# That is, the pair (ch1, ch2) is identical to the pair (ch2, ch1).
# We can do this using combinations_with_replacement from itertools as follows.
#
print("Testing iteration over channel lists...")
crg1 = channel_range.from_str("L0101-0104")
combs = list(itertools.combinations_with_replacement(crg1, 2))
def __init__(self, task_config, fft_config, ecei_config, storage_config):
"""Initialize the object with a fixed channel list, a fixed name of the analysis to be performed
and a fixed set of parameters for the analysis routine.
Inputs:
=======
task_config: dict, defines parameters of the analysis to be performed
fft_config: dict, gives parameters of the fourier-transformed data
ecei_config: dict, information on ecei diagnostic
"""
self.task_config = task_config
self.ecei_config = ecei_config
self.storage_config = storage_config
self.logger = logging.getLogger("simple")
# Stores the description of the task. This can be arbitrary
self.description = task_config["task_description"]
# Stores the name of the analysis we are going to execute
self.analysis = task_config["analysis"]
if self.analysis == "cross_phase":
self.kernel = kernel_crossphase_64_cy
elif self.analysis == "cross_power":
self.kernel = kernel_crosspower_64_cy
elif self.analysis == "cross_correlation":
self.kernel = kernel_crosscorr
elif self.analysis == "coherence":
self.kernel = kernel_coherence_64_cy
elif self.analysis == "skw":
self.kernel = kernel_skw
elif self.analysis == "bicoherence":
self.kernel = kernel_bicoherence
elif self.analysis == "null":
self.kernel = kernel_null
else:
raise NameError(f"Unknown analysis task {self.analysis}")
# Parse the reference and cross channels.
self.ref_channels = channel_range.from_str(task_config["ref_channels"])
# These channels serve as the cross-data for the spectral diagnostics
self.cmp_channels = channel_range.from_str(task_config["cmp_channels"])
# Construct a list of unique channels
# F.ex. we have ref_channels [(1,1), (1,2), (1,3)] and cmp_channels = [(1,1), (1,2)]
# The unique list of channels is then
# (1,1) x (1,1), (1,1) x (1,2)
# (1,2) x (1,2) !!! Omits (1,2) x (1,1)
# (1,3) x (1,1)
# (1,3) x (1,2)
channel_pairs = [channel_pair(cr, cx) for cr in self.ref_channels for cx in self.cmp_channels]
# Make a list, so that we don't exhaust the iterator after the first call.
self.unique_channels = [i[0] for i in more_itertools.distinct_combinations(channel_pairs, 1)]
# Number of channel pairs per future
self.channel_chunk_size = task_config["channel_chunk_size"]
# Total number of chunks, i.e. the number of futures appended to the list per call to calculate
self.num_chunks = (len(self.unique_channels) + self.channel_chunk_size - 1) // self.channel_chunk_size
# Get the configuration from task_fft_scipy, but don't store the object.
fft_config["fsample"] = ecei_config["SampleRate"] * 1e3
self.my_fft = task_fft_scipy(self.channel_chunk_size, fft_config, normalize=True, detrend=True)
self.fft_params = self.my_fft.get_fft_params()
self.storage_backend = None
if self.storage_config["backend"] == "numpy":
self.storage_backend = backends.backend_numpy(self.storage_config)
elif self.storage_config["backend"] == "mongo":
self.storage_backend = backends.backend_mongodb(self.storage_config)
elif self.storage_config["backend"] == "null":
self.storage_backend = backends.backend_null(self.storage_config)
else:
raise NameError(f"Unknown storage backend requested: {self.storage_config}")
self.storage_backend.store_metadata(self.task_config, self.get_dispatch_sequence())
#with open(args.config, "r") as df:
# cfg = json.load(df)
cfg = json.loads(config_str)
with open('configs/logger.yaml', 'r') as f:
log_cfg = yaml.safe_load(f.read())
logging.config.dictConfig(log_cfg)
logger = logging.getLogger("simple")
logger.info(f"{cfg['channel_range']}")
gen_id = 1_000 * rank
my_channel_range = channel_range.from_str(cfg["channel_range"][rank])
logger.info(f"rank: {rank} config: {cfg}")
reader = reader_dataman(cfg)
logger.info("Waiting")
reader.Open()
step = 0
while(True):
stepStatus = reader.BeginStep()
logger.info(stepStatus)
if stepStatus == adios2.StepStatus.OK:
channel_data = reader.get_data("FloatArray")
reader.EndStep()
logger.info(f"rank {rank:d}: Step {reader.CurrentStep():04d} data = {channel_data}")
def main():
# Parse command line arguments and read configuration file
parser = argparse.ArgumentParser(
description="Receive data and dispatch analysis tasks to a mpi queue")
parser.add_argument('--config',
type=str,
help='Lists the configuration file',
default='configs/config_null.json')
parser.add_argument('--benchmark', action="store_true")
args = parser.parse_args()
global cfg
with open(args.config, "r") as df:
cfg = json.load(df)
df.close()
# Load logger configuration from file:
# http://zetcode.com/python/logging/
with open("configs/logger.yaml", "r") as f:
log_cfg = yaml.safe_load(f.read())
logging.config.dictConfig(log_cfg)
comm = MPI.COMM_WORLD
rank = comm.Get_rank()
size = comm.Get_size()
# Create a global executor
#executor = concurrent.futures.ThreadPoolExecutor(max_workers=60)
#executor = MPIPoolExecutor(max_workers=24)
adios2_varname = channel_range.from_str(
cfg["transport"]["channel_range"][0])
with MPICommExecutor(MPI.COMM_WORLD) as executor:
if executor is not None:
logger = logging.getLogger('simple')
cfg["run_id"] = ''.join(
random.choice(string.ascii_uppercase + string.digits)
for _ in range(6))
cfg["run_id"] = "ABC125"
cfg["storage"]["run_id"] = cfg["run_id"]
logger.info(f"Starting run {cfg['run_id']}")
# Instantiate a storage backend and store the run configuration and task configuration
if cfg['storage']['backend'] == "numpy":
store_backend = backends.backend_numpy(cfg['storage'])
elif cfg['storage']['backend'] == "mongo":
store_backend = backends.backend_mongodb(cfg)
elif cfg['storage']['backend'] == "null":
store_backend = backends.backend_null(cfg['storage'])
logger.info("Storing one")
store_backend.store_one({
"run_id": cfg['run_id'],
"run_config": cfg
})
logger.info("Done storing. Continuing:")
# Create the FFT task
cfg["fft_params"]["fsample"] = cfg["ECEI_cfg"]["SampleRate"] * 1e3
my_fft = task_fft_scipy(10_000,
cfg["fft_params"],
normalize=True,
detrend=True)
fft_params = my_fft.get_fft_params()
# Create ADIOS reader object
reader = reader_gen(cfg["transport"])
# Create a list of individual spectral tasks
#task_list = []
#for task_config in cfg["task_list"]:
# #task_list.append(task_spectral(task_config, fft_params, cfg["ECEI_cfg"]))
# #task_list.append(task_spectral(task_config, cfg["fft_params"], cfg["ECEI_cfg"], cfg["storage"]))
# #store_backend.store_metadata(task_config, task_list[-1].get_dispatch_sequence())
dq = queue.Queue()
msg = None
tic_main = timeit.default_timer()
workers = []
for _ in range(16):
#thr = ConsumeThread(dq, executor, task_list, cfg)
worker = threading.Thread(target=consume,
args=(dq, executor, my_fft,
task_list))
worker.start()
workers.append(worker)
# logger.info(f"Started thread {thr}")
# reader.Open() is blocking until it opens the data file or receives the
# data stream. Put this right before entering the main loop
logger.info(f"{rank} Waiting for generator")
reader.Open()
last_step = 0
logger.info(f"Starting main loop")
rx_list = []
while True:
stepStatus = reader.BeginStep()
logger.info(f"currentStep = {reader.CurrentStep()}")
if stepStatus:
# Read data
stream_data = reader.Get(adios2_varname, save=False)
rx_list.append(reader.CurrentStep())
# Generate message id and publish is
msg = AdiosMessage(tstep_idx=reader.CurrentStep(),
data=stream_data)
dq.put_nowait(msg)
logger.info(f"Published message {msg}")
reader.EndStep()
else:
logger.info(f"Exiting: StepStatus={stepStatus}")
break
#Early stopping for debug
if reader.CurrentStep() > 100:
logger.info(
f"Exiting: CurrentStep={reader.CurrentStep()}, StepStatus={stepStatus}"
)
dq.put(AdiosMessage(tstep_idx=None, data=None))
break
last_step = reader.CurrentStep()
dq.join()
logger.info("Queue joined")
logger.info("Exiting main loop")
for thr in workers:
thr.join()
logger.info("Workers have joined")
# Shotdown the executioner
executor.shutdown(wait=True)
toc_main = timeit.default_timer()
logger.info(
f"Run {cfg['run_id']} finished in {(toc_main - tic_main):6.4f}s"
)
logger.info(f"Processed time_chunks {rx_list}")
#my_fft = task_fft_scipy(10_000, config_fft["fft_params"], normalize=True, detrend=True)
#fft_data = my_fft.do_fft_local(io_array_tr)
with np.load("/global/homes/r/rkube/repos/delta/test_data/fft_array_s0001.npz"
) as df:
fft_data = df["fft_data"]
fft_data_64 = np.ascontiguousarray(fft_data)
fft_data_32 = np.require(fft_data_64,
dtype=np.complex64,
requirements=['A', 'C'])
###################################################
# Generate channels to iterate over
ref_chrg = channel_range.from_str("L0101-2408")
cmp_chrg = channel_range.from_str("L0101-2408")
channel_pairs = [channel_pair(cr, cx) for cr in ref_chrg for cx in cmp_chrg]
unique_channels = [
i[0] for i in more_itertools.distinct_combinations(channel_pairs, 1)
]
#ch_it = [channel_pair(channel("L", i, 1), channel("L", i, 2)) for i in range(1, 25)]
#ch_it = ch_it + [channel_pair(channel("L", i, 1), channel("L", i, 3)) for i in range(1, 25)]
#h_it = ch_it + [channel_pair(channel("L", i, 1), channel("L", i, 4)) for i in range(1, 25)]
#ch_it = ch_it + [channel_pair(channel("L", i, 1), channel("L", i, 4)) for i in range(1, 25)]
#ch_it = ch_it + [channel_pair(channel("L", i, 1), channel("L", i, 5)) for i in range(1, 25)]
#ch_it = ch_it + [channel_pair(channel("L", i, 1), channel("L", i, 6)) for i in range(1, 25)]
#ch_it = ch_it + [channel_pair(channel("L", i, 1), channel("L", i, 7)) for i in range(1, 25)]
#ch_it = ch_it + [channel_pair(channel("L", i, 1), channel("L", i, 8)) for i in range(1, 25)]
def main():
"""Reads items from a ADIOS2 connection and forwards them."""
parser = argparse.ArgumentParser(description="Receive data and dispatch analysis tasks to a mpi queue")
parser.add_argument('--config', type=str, help='Lists the configuration file', default='configs/config-middle.json')
args = parser.parse_args()
with open(args.config, "r") as df:
cfg = json.load(df)
df.close()
timeout = 30
# The middleman uses both a reader and a writer. Each is configured with using their respective section
# of the config file. Therefore some keys are duplicated, such as channel_range. Make sure that these
# items are the same in both sections
assert(cfg["transport_kstar"]["channel_range"] == cfg["transport_nersc"]["channel_range"])
with open("configs/logger.yaml", "r") as f:
log_cfg = yaml.safe_load(f.read())
logging.config.dictConfig(log_cfg)
logger = logging.getLogger('middleman')
# Create ADIOS reader object
reader = reader_gen(cfg["transport_kstar"])
reader.Open()
dq = queue.Queue()
msg = None
worker = threading.Thread(target=forward, args=(dq, cfg, timeout))
worker.start()
tic = timeit.default_timer()
nstep = 0
rx_list = []
while True:
stepStatus = reader.BeginStep()
logger.info(f"stepStatus = {stepStatus}, currentStep = {reader.CurrentStep()}")
if stepStatus:
if reader.CurrentStep() == 0:
tic = timeit.default_timer()
# Read data
stream_data = reader.Get(channel_range.from_str(cfg["transport_kstar"]["channel_range"][0]),
save=False)
rx_list.append(reader.CurrentStep())
# Generate message id and publish is
msg = AdiosMessage(tstep_idx=reader.CurrentStep(), data=stream_data)
dq.put_nowait(msg)
logger.info(f"Published message {msg}")
reader.EndStep()
nstep += 1
else:
logger.info(f"Exiting: StepStatus={stepStatus}")
dq.put_nowait(AdiosMessage(tstep_idx=None, data=None))
break
last_step = reader.CurrentStep()
logger.info("Exiting main loop")
worker.join()
logger.info("Workers have joined")
dq.join()
logger.info("Queue joined")
toc = timeit.default_timer()
deltat = toc - tic
chunk_size = np.prod(stream_data.shape) * stream_data.itemsize / 1024 / 1024
logger.info(f"Received {len(rx_list)} time chunks: {rx_list}")
logger.info("")
logger.info("Summary:")
logger.info(f" chunk shape: {stream_data.shape}")
logger.info(f" chunk size (MB): {chunk_size:.03f}")
logger.info(f" total nstep: {nstep:d}")
logger.info(f" total data (MB): {(chunk_size * nstep):03f}")
logger.info(f" time (sec): {(deltat):.03f}")
logger.info(f" throughput (MB/sec): {(chunk_size * nstep)/(deltat):.03f}")
logger.info("Finished")
