def test_data_transfer(self):
master = MockRingBufferMaster()
master.create_buffer()
jf_test_det = DetectorDefinition(
detector_name="0.5 JF",
detector_model=JUNGFRAU,
geometry=[1, 1],
bit_depth=16
)
writer = MockRingBufferClient(
process_id=1,
follower_ids=[],
detector_def=jf_test_det,
as_reader=False,
use_assembly_buffer=True
)
writer.init_buffer()
receiver = MockRingBufferClient(
process_id=2,
follower_ids=[1],
detector_def=jf_test_det,
use_assembly_buffer=True
)
receiver.init_buffer()
writer_current_slot = rb.claim_next_slot(writer.rb_consumer_id)
self.assertNotEqual(writer_current_slot, -1, "The writer should be able to get the slot.")
receiver_current_slot = rb.claim_next_slot(receiver.rb_consumer_id)
self.assertEqual(receiver_current_slot, -1, "The reader should not be able to get the slot yet.")
sent_raw_data = numpy.random.randint(low=0, high=128,
size=jf_test_det.raw_image_data_n_bytes, dtype="uint16")
raw_write_pointer = rb.get_buffer_slot(writer.rb_raw_dbuffer_id, writer_current_slot)
ctypes.memmove(raw_write_pointer, sent_raw_data.ctypes.data, sent_raw_data.nbytes)
sent_assembled_data = numpy.random.randint(low=0, high=128,
size=jf_test_det.image_data_n_bytes, dtype="uint16")
assembled_write_pointer = rb.get_buffer_slot(writer.rb_assembled_dbuffer_id, writer_current_slot)
ctypes.memmove(assembled_write_pointer, sent_assembled_data.ctypes.data, sent_assembled_data.nbytes)
rb.commit_slot(writer.rb_consumer_id, writer_current_slot)
receiver_current_slot = rb.claim_next_slot(receiver.rb_consumer_id)
self.assertEqual(receiver_current_slot, writer_current_slot, "Slot should be ready for the receiver.")
raw_receive_pointer = rb.get_buffer_slot(receiver.rb_raw_dbuffer_id, receiver_current_slot)
raw_received_data = get_frame_data(raw_receive_pointer, [jf_test_det.raw_image_data_n_bytes])
numpy.testing.assert_array_equal(sent_raw_data, raw_received_data)
assembled_receive_pointer = rb.get_buffer_slot(receiver.rb_assembled_dbuffer_id, receiver_current_slot)
assembled_received_data = get_frame_data(assembled_receive_pointer, [jf_test_det.image_data_n_bytes])
numpy.testing.assert_array_equal(sent_assembled_data, assembled_received_data)
def start_rb_assembler(name, detector_def: DetectorDefinition,
ringbuffer: MpiRingBufferClient, assembler_index,
n_total_assemblers):
_logger.info("Starting assembler with name='%s'." % name)
ringbuffer.init_buffer()
control_client = MpiControlClient()
mpi_ref_time = time()
image_assembler = ImageAssembler(det_assembler=detector_def,
assembler_index=assembler_index,
n_total_assemblers=n_total_assemblers)
# Function signature:
# char* source_root, char* destination_root, size_t* dest_move_offsets, uint32_t n_moves, size_t n_bytes_per_move
assemble_image = get_image_assembler_function()
dest_move_offsets = image_assembler.move_offsets
n_moves = image_assembler.n_moves
n_bytes_per_move = image_assembler.n_bytes_per_move
while True:
if (time() - mpi_ref_time) > MPI_COMM_DELAY:
# TODO: Currently the only message is a reset message.
if control_client.is_message_ready():
control_client.get_message()
ringbuffer.reset()
_logger.info("[%s] Ringbuffer reset." % name)
mpi_ref_time = time()
rb_current_slot = rb.claim_next_slot(ringbuffer.rb_consumer_id)
if rb_current_slot == -1:
sleep(config.RB_RETRY_DELAY)
continue
raw_data_pointer = rb.get_buffer_slot(ringbuffer.rb_raw_dbuffer_id,
rb_current_slot)
assembled_data_pointer = rb.get_buffer_slot(
ringbuffer.rb_assembled_dbuffer_id, rb_current_slot)
assemble_image(raw_data_pointer, assembled_data_pointer,
dest_move_offsets, n_moves, n_bytes_per_move)
if not rb.commit_slot(ringbuffer.rb_consumer_id, rb_current_slot):
error_message = "[%s] Cannot commit rb slot %d." % (
name, rb_current_slot)
_logger.error(error_message)
raise RuntimeError(error_message)
def start_h5_writer(name, detector_def: DetectorDefinition,
ringbuffer: MpiRingBufferClient):
_logger.info("Starting h5 writer with name='%s'" % name)
ringbuffer.init_buffer()
writer = H5Writer(detector_def)
control_client = MpiControlClient()
mpi_ref_time = time()
while True:
if (time() - mpi_ref_time) > MPI_COMM_DELAY:
# TODO: Currently the only message is a reset message.
if control_client.is_message_ready():
control_client.get_message()
writer.close()
writer = H5Writer(detector_def)
_logger.info("[%s] Ringbuffer reset." % name)
mpi_ref_time = time()
rb_current_slot = rb.claim_next_slot(ringbuffer.rb_consumer_id)
if rb_current_slot == -1:
sleep(config.RB_RETRY_DELAY)
continue
metadata, data = read_data_from_rb(
rb_current_slot=rb_current_slot,
rb_hbuffer_id=ringbuffer.rb_hbuffer_id,
rb_dbuffer_id=ringbuffer.rb_consumer_id,
n_submodules=detector_def.n_submodules_total,
image_size=detector_def.detector_size)
writer.write_image(data.tobytes())
writer.write_metadata(metadata)
if not rb.commit_slot(ringbuffer.rb_consumer_id, rb_current_slot):
error_message = "[%s] Cannot commit rb slot %d." % (
name, rb_current_slot)
_logger.error(error_message)
raise RuntimeError(error_message)
def start_writer_sender(name, bind_url, zmq_mode,
detector_def: DetectorDefinition,
ringbuffer: MpiRingBufferClient):
_logger.info(
"Starting sender with name='%s', bind_url='%s', zmq_mode='%s'" %
(name, bind_url, zmq_mode))
ringbuffer.init_buffer()
context = zmq.Context(io_threads=ZMQ_IO_THREADS)
socket = context.socket(zmq.__getattribute__(zmq_mode))
socket.bind(bind_url)
zmq_sender = DetectorZMQSender(name, socket, ringbuffer, detector_def)
control_client = MpiControlClient()
mpi_ref_time = time()
while True:
if (time() - mpi_ref_time) > MPI_COMM_DELAY:
# TODO: Currently the only message is a reset message.
if control_client.is_message_ready():
control_client.get_message()
ringbuffer.reset()
zmq_sender.reset()
_logger.info("[%s] Ringbuffer reset." % name)
mpi_ref_time = time()
rb_current_slot = rb.claim_next_slot(ringbuffer.rb_consumer_id)
if rb_current_slot == -1:
sleep(config.RB_RETRY_DELAY)
continue
metadata, data = zmq_sender.read_data(rb_current_slot)
zmq_sender.send_data(metadata, data)
if not rb.commit_slot(ringbuffer.rb_consumer_id, rb_current_slot):
error_message = "[%s] Cannot commit rb slot %d." % (
name, rb_current_slot)
_logger.error(error_message)
raise RuntimeError(error_message)
def test_prefilled_buffer(n_buffer_slots, rb_folder,
detector: DetectorDefinition, n_slots_to_fill):
create_rb_files(n_buffer_slots,
detector.image_header_n_bytes,
detector.image_data_n_bytes,
rb_folder=rb_folder)
rb_master = MockRingBufferMaster(rb_folder=rb_folder)
rb_master.create_buffer()
rb_client_writer = MockRingBufferClient(process_id=0,
follower_ids=[],
detector_def=detector,
as_reader=False,
rb_folder=rb_folder)
rb_client_reader = MockRingBufferClient(process_id=1,
follower_ids=[0],
detector_def=detector,
as_reader=True,
rb_folder=rb_folder)
rb_client_writer.init_buffer()
rb_client_reader.init_buffer()
for i_slot in range(n_slots_to_fill):
rb_current_slot = rb.claim_next_slot(rb_client_writer.rb_consumer_id)
if rb_current_slot == -1:
raise RuntimeError(
"Cannot get next slot from ringbuffer. Are the RB files available?"
)
if not rb.commit_slot(rb_client_writer.rb_consumer_id,
rb_current_slot):
raise RuntimeError("Cannot commit rb slot %d." % rb_current_slot)
def test_multi_module_placement(self):
udp_ip = "127.0.0.1"
udp_port = [12001, 12002, 12003, 12004]
bit_depth = 16
udp_receiver_ranks = [0, 1, 2, 3]
n_frames = 10
test_eiger = DetectorDefinition(detector_name="Test Eiger 0.5M",
detector_model=EIGER,
geometry=[1, 1],
bit_depth=bit_depth)
test_control_client = MockControlClient()
ringbuffer_master = MockRingBufferMaster()
ringbuffer_master.create_buffer()
ringbuffer_client = MockRingBufferClient(
process_id=4,
follower_ids=udp_receiver_ranks,
detector_def=test_eiger,
as_reader=True)
ringbuffer_client.init_buffer()
def start_receiver(process_id, process_udp_port):
ringbuffer_client_udp = MockRingBufferClient(
process_id=process_id,
follower_ids=[4],
detector_def=test_eiger,
as_reader=False)
ringbuffer_client.init_buffer()
start_udp_receiver(udp_ip=udp_ip,
udp_port=process_udp_port,
detector_def=test_eiger,
submodule_index=process_id,
ringbuffer=ringbuffer_client_udp,
control_client=test_control_client)
self.receive_process.append(
Process(target=start_receiver, args=(0, udp_port[0])))
self.receive_process[-1].start()
self.receive_process.append(
Process(target=start_receiver, args=(1, udp_port[1])))
self.receive_process[-1].start()
self.receive_process.append(
Process(target=start_receiver, args=(2, udp_port[2])))
self.receive_process[-1].start()
self.receive_process.append(
Process(target=start_receiver, args=(3, udp_port[3])))
self.receive_process[-1].start()
rb_current_slot = rb.claim_next_slot(ringbuffer_client.rb_consumer_id)
self.assertEqual(rb_current_slot, -1,
"You should not be able to get this RB slice yet.")
sleep(0.2)
# Receiver 0
generate_udp_stream(udp_ip,
udp_port[0],
message_generator=generate_submodule_eiger_packets(
bit_depth, n_frames, 0))
rb_current_slot = rb.claim_next_slot(ringbuffer_client.rb_consumer_id)
self.assertEqual(rb_current_slot, -1,
"You should not be able to get this RB slice yet.")
# Receiver 1
generate_udp_stream(udp_ip,
udp_port[1],
message_generator=generate_submodule_eiger_packets(
bit_depth, n_frames, 1))
rb_current_slot = rb.claim_next_slot(ringbuffer_client.rb_consumer_id)
self.assertEqual(rb_current_slot, -1,
"You should not be able to get this RB slice yet.")
# Receiver 2
generate_udp_stream(udp_ip,
udp_port[2],
message_generator=generate_submodule_eiger_packets(
bit_depth, n_frames, 2))
rb_current_slot = rb.claim_next_slot(ringbuffer_client.rb_consumer_id)
self.assertEqual(rb_current_slot, -1,
"You should not be able to get this RB slice yet.")
# Receiver 3
generate_udp_stream(udp_ip,
udp_port[3],
message_generator=generate_submodule_eiger_packets(
bit_depth, n_frames, 3))
sleep(0.2)
rb_current_slot = rb.claim_next_slot(ringbuffer_client.rb_consumer_id)
self.assertNotEqual(rb_current_slot, -1,
"No frames found in ringbuffer.")
self.assertListEqual(list(rb.get_header_info(0).committed_slots[:2]),
[n_frames, n_frames])
for i in range(n_frames):
self.assertEqual(i, rb_current_slot)
metadata_pointer = rb.get_buffer_slot(
ringbuffer_client.rb_hbuffer_id, rb_current_slot)
metadata = get_frame_metadata(metadata_pointer, len(udp_port))
self.assertListEqual(metadata["framenums"],
[i + 1] * len(udp_port))
self.assertListEqual(metadata["missing_packets_1"],
[0] * len(udp_port))
self.assertListEqual(metadata["missing_packets_2"],
[0] * len(udp_port))
self.assertListEqual(metadata["pulse_ids"], [0] * len(udp_port))
self.assertListEqual(metadata["daq_recs"], [0] * len(udp_port))
self.assertListEqual(metadata["module_number"],
list(range(len(udp_port))))
self.assertListEqual(metadata["module_enabled"],
[1] * len(udp_port))
self.assertEqual(metadata["frame"], i + 1)
self.assertEqual(metadata["daq_rec"], 0)
self.assertEqual(metadata["pulse_id"], 0)
self.assertEqual(metadata["is_good_frame"], 1)
self.assertListEqual(metadata["pulse_id_diff"],
[0] * len(udp_port))
self.assertListEqual(metadata["framenum_diff"],
[0] * len(udp_port))
data_pointer = rb.get_buffer_slot(
ringbuffer_client.rb_raw_dbuffer_id, rb_current_slot)
data_shape = [test_eiger.n_submodules_total
] + test_eiger.detector_model.submodule_size
data = get_frame_data(data_pointer, data_shape)
n_lines_per_packet = get_n_lines_per_packet(test_eiger)
n_packets_per_frame = get_n_packets_per_frame(test_eiger)
for submodule_index in range(test_eiger.n_submodules_total):
for packet_index in range(n_packets_per_frame):
packet_line_offset = n_lines_per_packet * packet_index
submodule_data = data[submodule_index][
packet_line_offset:packet_line_offset +
n_lines_per_packet]
submodule_indexes = submodule_data // 1000
self.assertEqual(submodule_indexes.min(), submodule_index)
self.assertEqual(submodule_indexes.max(), submodule_index)
packet_indexes = (submodule_data % 1000) // 10
self.assertEqual(packet_indexes.min(), packet_index)
self.assertEqual(packet_indexes.max(), packet_index)
frame_indexes = submodule_data % 10
self.assertEqual(frame_indexes.min(), i)
self.assertEqual(frame_indexes.max(), i)
self.assertTrue(
rb.commit_slot(ringbuffer_client.rb_consumer_id,
rb_current_slot))
rb_current_slot = rb.claim_next_slot(
ringbuffer_client.rb_consumer_id)
self.assertEqual(-1,
rb.claim_next_slot(ringbuffer_client.rb_consumer_id))