def test_DataAnalyzerProcess__processes_change_magnetometer_config_command(
four_board_analyzer_process_beta_2_mode, ):
da_process = four_board_analyzer_process_beta_2_mode["da_process"]
from_main_queue = four_board_analyzer_process_beta_2_mode[
"from_main_queue"]
test_num_wells = 24
expected_wells = [5, 6, 15, 16]
test_config_dict = create_magnetometer_config_dict(test_num_wells)
for well_idx in expected_wells:
module_id = SERIAL_COMM_WELL_IDX_TO_MODULE_ID[well_idx]
test_config_dict[module_id][3] = True
expected_sampling_period = 15000
set_sampling_period_command = {
"communication_type": "acquisition_manager",
"command": "change_magnetometer_config",
"magnetometer_config": test_config_dict,
"sampling_period": expected_sampling_period,
}
put_object_into_queue_and_raise_error_if_eventually_still_empty(
set_sampling_period_command, from_main_queue)
invoke_process_run_and_check_errors(da_process)
assert da_process.get_active_wells() == expected_wells
expected_buffer_size = MIN_NUM_SECONDS_NEEDED_FOR_ANALYSIS * int(
1e6 / expected_sampling_period)
assert da_process.get_buffer_size() == expected_buffer_size
def test_DataAnalyzerProcess_beta_2_performance__fill_data_analysis_buffer(
runnable_four_board_analyzer_process, ):
# 11 seconds of data (100 Hz) coming in from File Writer to going through to Main
#
# initial pulse3D import: 1.662150824
# pulse3D 0.23.3: 1.680566285
p, board_queues, comm_from_main_queue, comm_to_main_queue, _ = runnable_four_board_analyzer_process
p._beta_2_mode = True
p.change_magnetometer_config(
{
"magnetometer_config": DEFAULT_MAGNETOMETER_CONFIG,
"sampling_period": DEFAULT_SAMPLING_PERIOD
}, )
put_object_into_queue_and_raise_error_if_eventually_still_empty(
dict(START_MANAGED_ACQUISITION_COMMUNICATION),
comm_from_main_queue,
)
invoke_process_run_and_check_errors(p, perform_setup_before_loop=True)
num_seconds = MIN_NUM_SECONDS_NEEDED_FOR_ANALYSIS + 1
fill_da_input_data_queue(board_queues[0][0], num_seconds)
start = time.perf_counter_ns()
invoke_process_run_and_check_errors(p, num_iterations=num_seconds)
dur_seconds = (time.perf_counter_ns() - start) / 10**9
# prevent BrokenPipeErrors
drain_queue(board_queues[0][1])
drain_queue(comm_to_main_queue)
# print(f"Duration (seconds): {dur_seconds}") # Eli (4/8/21): this is commented code that is deliberately kept in the codebase since it is often toggled on/off during optimization
assert dur_seconds < 10
def test_DataAnalyzerProcess__correctly_loads_construct_sensor_data_to_buffer_when_not_empty(
test_well_index, test_construct_data, test_description,
four_board_analyzer_process):
p, board_queues, comm_from_main_queue, _, _ = four_board_analyzer_process
p.init_streams()
incoming_data = board_queues[0][0]
put_object_into_queue_and_raise_error_if_eventually_still_empty(
get_mutable_copy_of_START_MANAGED_ACQUISITION_COMMUNICATION(),
comm_from_main_queue,
)
invoke_process_run_and_check_errors(p)
test_construct_dict = {
"is_reference_sensor": False,
"well_index": test_well_index,
"data": test_construct_data,
}
put_object_into_queue_and_raise_error_if_eventually_still_empty(
test_construct_dict, incoming_data)
data_buffer = p._data_buffer # pylint:disable=protected-access
expected_construct_data = [[0, 0], [0, 0]]
data_buffer[test_well_index]["construct_data"] = copy.deepcopy(
expected_construct_data)
invoke_process_run_and_check_errors(p)
expected_construct_data[0].extend(test_construct_data[0])
expected_construct_data[1].extend(test_construct_data[1])
np.testing.assert_equal(data_buffer[test_well_index]["construct_data"],
expected_construct_data)
def test_put_object_into_queue_and_raise_error_if_eventually_still_empty__raises_error_if_queue_not_populated(
mocker,
):
q = Queue()
mocker.patch.object(q, "put", autospec=True)
with pytest.raises(QueueStillEmptyError):
put_object_into_queue_and_raise_error_if_eventually_still_empty("bill", q)
def test_DataAnalyzerProcess_beta_1_performance__single_data_packet_per_well_without_analysis(
runnable_four_board_analyzer_process, ):
# 1 second of data (625 Hz) coming in from File Writer to going through to Main
#
# start: 0.530731389
# added twitch metric analysis: 0.578328276
# initial pulse3D import: 0.533860423
# pulse3D 0.23.3: 0.539447351
p, board_queues, comm_from_main_queue, comm_to_main_queue, _ = runnable_four_board_analyzer_process
put_object_into_queue_and_raise_error_if_eventually_still_empty(
get_mutable_copy_of_START_MANAGED_ACQUISITION_COMMUNICATION(),
comm_from_main_queue,
)
invoke_process_run_and_check_errors(p, perform_setup_before_loop=True)
num_seconds = 1
fill_da_input_data_queue(board_queues[0][0], num_seconds)
start = time.perf_counter_ns()
invoke_process_run_and_check_errors(p,
num_iterations=num_seconds * (24 + 6))
dur_seconds = (time.perf_counter_ns() - start) / 10**9
# prevent BrokenPipeErrors
drain_queue(board_queues[0][1])
drain_queue(comm_to_main_queue)
# print(f"Duration (seconds): {dur_seconds}") # Eli (4/8/21): this is commented code that is deliberately kept in the codebase since it is often toggled on/off during optimization
assert dur_seconds < 2
def test_DataAnalyzerProcess_beta_1_performance__first_second_of_data_with_analysis(
runnable_four_board_analyzer_process, ):
# Fill data analysis buffer with 10 seconds of data to start metric analysis,
# Then record duration of sending 1 additional second of data
#
# start: 0.547285524
# initial pulse3D import: 0.535316489
# pulse3D 0.23.3: 0.535428579
p, board_queues, comm_from_main_queue, comm_to_main_queue, _ = runnable_four_board_analyzer_process
put_object_into_queue_and_raise_error_if_eventually_still_empty(
get_mutable_copy_of_START_MANAGED_ACQUISITION_COMMUNICATION(),
comm_from_main_queue,
)
invoke_process_run_and_check_errors(p, perform_setup_before_loop=True)
# load data
num_seconds = MIN_NUM_SECONDS_NEEDED_FOR_ANALYSIS + 1
fill_da_input_data_queue(board_queues[0][0], num_seconds)
invoke_process_run_and_check_errors(
p, num_iterations=MIN_NUM_SECONDS_NEEDED_FOR_ANALYSIS * (24 + 6))
# send additional data and time analysis
start = time.perf_counter_ns()
invoke_process_run_and_check_errors(p, num_iterations=(24 + 6))
dur_seconds = (time.perf_counter_ns() - start) / 10**9
# prevent BrokenPipeErrors
drain_queue(board_queues[0][1])
drain_queue(comm_to_main_queue)
# print(f"Duration (seconds): {dur_seconds}") # Eli (4/8/21): this is commented code that is deliberately kept in the codebase since it is often toggled on/off during optimization
assert dur_seconds < 2
def test_DataAnalyzerProcess_beta_1_performance__fill_data_analysis_buffer(
runnable_four_board_analyzer_process, ):
# 11 seconds of data (625 Hz) coming in from File Writer to going through to Main
#
# mantarray-waveform-analysis v0.3: 4.148136512
# mantarray-waveform-analysis v0.3.1: 3.829136133
# mantarray-waveform-analysis v0.4.0: 3.323093677
# remove concatenate: 2.966678695
# 30 Hz Bessel filter: 2.930061808 # Tanner (9/3/20): not intended to speed anything up, just adding this to show it had it didn't have much affect on performance
# 30 Hz Butterworth filter: 2.935009033 # Tanner (9/10/20): not intended to speed anything up, just adding this to show it had it didn't have much affect on performance
#
# added twitch metric analysis: 3.013469479
# initial pulse3D import: 3.855403546
# pulse3D 0.23.3: 3.890723909
p, board_queues, comm_from_main_queue, comm_to_main_queue, _ = runnable_four_board_analyzer_process
put_object_into_queue_and_raise_error_if_eventually_still_empty(
get_mutable_copy_of_START_MANAGED_ACQUISITION_COMMUNICATION(),
comm_from_main_queue,
)
invoke_process_run_and_check_errors(p, perform_setup_before_loop=True)
num_seconds = MIN_NUM_SECONDS_NEEDED_FOR_ANALYSIS + 1
fill_da_input_data_queue(board_queues[0][0], num_seconds)
start = time.perf_counter_ns()
invoke_process_run_and_check_errors(p,
num_iterations=num_seconds * (24 + 6))
dur_seconds = (time.perf_counter_ns() - start) / 10**9
# prevent BrokenPipeErrors
drain_queue(board_queues[0][1])
drain_queue(comm_to_main_queue)
# print(f"Duration (seconds): {dur_seconds}") # Eli (4/8/21): this is commented code that is deliberately kept in the codebase since it is often toggled on/off during optimization
assert dur_seconds < 10
def test_DataAnalyzerProcess__processes_stop_managed_acquisition_command(
four_board_analyzer_process, ):
p, _, comm_from_main_queue, _, _ = four_board_analyzer_process
put_object_into_queue_and_raise_error_if_eventually_still_empty(
get_mutable_copy_of_START_MANAGED_ACQUISITION_COMMUNICATION(),
comm_from_main_queue,
)
invoke_process_run_and_check_errors(p)
data_buffer = p._data_buffer # pylint:disable=protected-access
for well_idx in range(24):
data_buffer[well_idx]["construct_data"] = [[0, 0, 0], [1, 2, 3]]
data_buffer[well_idx]["ref_data"] = [[0, 0, 0], [4, 5, 6]]
put_object_into_queue_and_raise_error_if_eventually_still_empty(
STOP_MANAGED_ACQUISITION_COMMUNICATION,
comm_from_main_queue,
)
invoke_process_run_and_check_errors(p)
assert p._end_of_data_stream_reached[0] is True # pylint:disable=protected-access
assert data_buffer[0]["construct_data"] is None
assert data_buffer[0]["ref_data"] is None
assert data_buffer[23]["construct_data"] is None
assert data_buffer[23]["ref_data"] is None
def test_put_object_into_queue_and_raise_error_if_eventually_still_empty__passes_timeout_kwarg_to_subfunction(
mocker,
):
expected = 2.2
q = Queue()
spied_not_empty = mocker.spy(queue_utils, "is_queue_eventually_not_empty")
put_object_into_queue_and_raise_error_if_eventually_still_empty("bill", q, timeout_seconds=expected)
spied_not_empty.assert_called_once_with(q, timeout_seconds=expected)
def test_DataAnalyzerProcess__drain_all_queues__drains_all_queues_except_error_queue_and_returns__all_items(
four_board_analyzer_process, ):
expected = [[10, 11], [12, 13], [14, 15], [16, 17]]
expected_error = "error"
expected_from_main = "from_main"
expected_to_main = "to_main"
(
data_analyzer_process,
board_queues,
from_main_queue,
to_main_queue,
error_queue,
) = four_board_analyzer_process
for i, board in enumerate(board_queues):
for j, queue in enumerate(board):
queue_item = expected[i][j]
put_object_into_queue_and_raise_error_if_eventually_still_empty(
queue_item, queue)
from_main_queue.put_nowait(expected_from_main)
to_main_queue.put_nowait(expected_to_main)
put_object_into_queue_and_raise_error_if_eventually_still_empty(
expected_error, error_queue)
confirm_queue_is_eventually_of_size(from_main_queue, 1)
confirm_queue_is_eventually_of_size(to_main_queue, 1)
actual = data_analyzer_process._drain_all_queues() # pylint:disable=protected-access
confirm_queue_is_eventually_of_size(error_queue, 1)
actual_error = error_queue.get(timeout=QUEUE_CHECK_TIMEOUT_SECONDS)
assert actual_error == expected_error
confirm_queue_is_eventually_empty(from_main_queue)
confirm_queue_is_eventually_empty(to_main_queue)
confirm_queue_is_eventually_empty(board_queues[3][0])
confirm_queue_is_eventually_empty(board_queues[2][0])
confirm_queue_is_eventually_empty(board_queues[1][0])
confirm_queue_is_eventually_empty(board_queues[0][1])
confirm_queue_is_eventually_empty(board_queues[0][0])
assert actual["board_0"]["outgoing_data"] == [expected[0][1]]
assert actual["board_3"]["file_writer_to_data_analyzer"] == [
expected[3][0]
]
assert actual["board_2"]["file_writer_to_data_analyzer"] == [
expected[2][0]
]
assert actual["board_1"]["file_writer_to_data_analyzer"] == [
expected[1][0]
]
assert actual["board_0"]["file_writer_to_data_analyzer"] == [
expected[0][0]
]
assert actual["from_main_to_data_analyzer"] == [expected_from_main]
assert actual["from_data_analyzer_to_main"] == [expected_to_main]
def put_object_into_queue_and_raise_error_if_eventually_still_empty(
obj: object,
the_queue: UnionOfThreadingAndMultiprocessingQueue,
timeout_seconds: Union[float, int] = QUEUE_CHECK_TIMEOUT_SECONDS,
sleep_after_put_seconds: Optional[Union[float, int]] = None,
) -> None:
stdlib_utils.put_object_into_queue_and_raise_error_if_eventually_still_empty(
obj, the_queue, timeout_seconds=timeout_seconds)
if sleep_after_put_seconds is not None:
time.sleep(sleep_after_put_seconds)
def set_simulator_idle_ready(simulator_fixture):
simulator = simulator_fixture["simulator"]
testing_queue = simulator_fixture["testing_queue"]
put_object_into_queue_and_raise_error_if_eventually_still_empty(
{
"command": "set_status_code",
"status_code": SERIAL_COMM_IDLE_READY_CODE
},
testing_queue,
)
invoke_process_run_and_check_errors(simulator)
def test_DataAnalyzerProcess__processes_start_managed_acquisition_command__by_draining_outgoing_data_queue(
four_board_analyzer_process, ):
p, board_queues, comm_from_main_queue, _, _ = four_board_analyzer_process
start_command = get_mutable_copy_of_START_MANAGED_ACQUISITION_COMMUNICATION(
)
put_object_into_queue_and_raise_error_if_eventually_still_empty(
start_command, comm_from_main_queue)
put_object_into_queue_and_raise_error_if_eventually_still_empty(
"item", board_queues[0][1])
invoke_process_run_and_check_errors(p)
confirm_queue_is_eventually_empty(board_queues[0][1])
def test_DataAnalyzerProcess__reinits_streams_upon_receiving_stop_managed_acquisition_command(
four_board_analyzer_process, mocker):
p, _, from_main_queue, _, _ = four_board_analyzer_process
spied_init_streams = mocker.spy(p, "init_streams")
put_object_into_queue_and_raise_error_if_eventually_still_empty(
STOP_MANAGED_ACQUISITION_COMMUNICATION,
from_main_queue,
)
invoke_process_run_and_check_errors(p)
spied_init_streams.assert_called_once_with()
def test_DataAnalyzerProcess__raises_error_with_unrecognized_acquisition_manager_command(
four_board_analyzer_process, mocker, patch_print):
p, _, comm_from_main_queue, _, _ = four_board_analyzer_process
expected_command = "fake_command"
start_command = {
"communication_type": "acquisition_manager",
"command": expected_command,
}
put_object_into_queue_and_raise_error_if_eventually_still_empty(
start_command, comm_from_main_queue)
with pytest.raises(UnrecognizedCommandFromMainToDataAnalyzerError,
match=expected_command):
invoke_process_run_and_check_errors(p)
def test_DataAnalyzerProcess__raises_error_if_communication_type_is_invalid(
four_board_analyzer_process, mocker, patch_print):
p, _, comm_from_main_queue, _, _ = four_board_analyzer_process
invalid_command = {
"communication_type": "fake_type",
}
put_object_into_queue_and_raise_error_if_eventually_still_empty(
invalid_command,
comm_from_main_queue,
)
with pytest.raises(UnrecognizedCommandFromMainToDataAnalyzerError,
match="fake_type"):
invoke_process_run_and_check_errors(p)
def test_DataAnalyzerProcess_commands_for_each_run_iteration__checks_for_calibration_update_from_main(
four_board_analyzer_process, ):
calibration_comm = {
"communication_type": "calibration",
"calibration_settings": 1,
}
p, _, comm_from_main_queue, _, _ = four_board_analyzer_process
put_object_into_queue_and_raise_error_if_eventually_still_empty(
calibration_comm,
comm_from_main_queue,
timeout_seconds=QUEUE_CHECK_TIMEOUT_SECONDS,
)
invoke_process_run_and_check_errors(p)
actual = p.get_calibration_settings()
assert actual == calibration_comm["calibration_settings"]
def set_stim_info_and_start_stimulating(simulator_fixture, stim_info):
simulator = simulator_fixture["simulator"]
testing_queue = simulator_fixture["testing_queue"]
put_object_into_queue_and_raise_error_if_eventually_still_empty(
{
"command": "set_stim_info",
"stim_info": stim_info
}, testing_queue)
invoke_process_run_and_check_errors(simulator)
put_object_into_queue_and_raise_error_if_eventually_still_empty(
{
"command": "set_stim_status",
"status": True
}, testing_queue)
invoke_process_run_and_check_errors(simulator)
# remove all bytes sent from initial subprotocol status update
simulator.read_all()
def test_DataAnalyzerProcess__correctly_pairs_descending_order_ref_sensor_data_in_buffer(
four_board_analyzer_process, ):
p, board_queues, comm_from_main_queue, _, _ = four_board_analyzer_process
incoming_data = board_queues[0][0]
put_object_into_queue_and_raise_error_if_eventually_still_empty(
get_mutable_copy_of_START_MANAGED_ACQUISITION_COMMUNICATION(),
comm_from_main_queue,
)
invoke_process_run_and_check_errors(p)
test_ref_data = np.array(
[
[
125, 375, 625, 875, 1125, 1375, 1625, 1875, 2125, 2375, 2625,
2875
],
[11, 23, 35, 47, 59, 71, 83, 95, 107, 119, 131, 143],
],
dtype=np.int32,
)
test_ref_dict = {
"is_reference_sensor": True,
"reference_for_wells": REF_INDEX_TO_24_WELL_INDEX[5],
"data": test_ref_data,
}
put_object_into_queue_and_raise_error_if_eventually_still_empty(
test_ref_dict, incoming_data)
invoke_process_run_and_check_errors(p)
data_buffer = p._data_buffer # pylint:disable=protected-access
expected_ref_data_23 = np.array([[1250, 11250, 21250], [11, 59, 107]],
dtype=np.int32)
np.testing.assert_equal(data_buffer[23]["ref_data"], expected_ref_data_23)
expected_ref_data_22 = np.array([[3750, 13750, 23750], [23, 71, 119]],
dtype=np.int32)
np.testing.assert_equal(data_buffer[22]["ref_data"], expected_ref_data_22)
expected_ref_data_19 = np.array([[6250, 16250, 26250], [35, 83, 131]],
dtype=np.int32)
np.testing.assert_equal(data_buffer[19]["ref_data"], expected_ref_data_19)
expected_ref_data_18 = np.array([[8750, 18750, 28750], [47, 95, 143]],
dtype=np.int32)
np.testing.assert_equal(data_buffer[18]["ref_data"], expected_ref_data_18)
def test_DataAnalyzerProcess__correctly_pairs_ascending_order_ref_sensor_data_in_buffer(
four_board_analyzer_process, ):
p, board_queues, comm_from_main_queue, _, _ = four_board_analyzer_process
incoming_data = board_queues[0][0]
put_object_into_queue_and_raise_error_if_eventually_still_empty(
get_mutable_copy_of_START_MANAGED_ACQUISITION_COMMUNICATION(),
comm_from_main_queue,
)
invoke_process_run_and_check_errors(p)
test_ref_data = np.array(
[
[
125, 375, 625, 875, 1125, 1375, 1625, 1875, 2125, 2375, 2625,
2875
],
[6, 18, 30, 42, 54, 66, 78, 90, 102, 114, 126, 138],
],
dtype=np.int32,
)
test_ref_dict = {
"is_reference_sensor": True,
"reference_for_wells": REF_INDEX_TO_24_WELL_INDEX[0],
"data": test_ref_data,
}
put_object_into_queue_and_raise_error_if_eventually_still_empty(
test_ref_dict, incoming_data)
invoke_process_run_and_check_errors(p)
data_buffer = p._data_buffer # pylint:disable=protected-access
expected_ref_data_0 = np.array([[1250, 11250, 21250], [6, 54, 102]],
dtype=np.int32)
np.testing.assert_equal(data_buffer[0]["ref_data"], expected_ref_data_0)
expected_ref_data_1 = np.array([[3750, 13750, 23750], [18, 66, 114]],
dtype=np.int32)
np.testing.assert_equal(data_buffer[1]["ref_data"], expected_ref_data_1)
expected_ref_data_4 = np.array([[6250, 16250, 26250], [30, 78, 126]],
dtype=np.int32)
np.testing.assert_equal(data_buffer[4]["ref_data"], expected_ref_data_4)
expected_ref_data_5 = np.array([[8750, 18750, 28750], [42, 90, 138]],
dtype=np.int32)
np.testing.assert_equal(data_buffer[5]["ref_data"], expected_ref_data_5)
def test_DataAnalyzerProcess_beta_2_performance__first_second_of_data_with_analysis(
runnable_four_board_analyzer_process, ):
# Fill data analysis buffer with 10 seconds of data to start metric analysis,
# Then record duration of sending 1 additional second of data
#
# initial pulse3D import: 0.334087008
# pulse3D 0.23.3: 0.337370183
p, board_queues, comm_from_main_queue, comm_to_main_queue, _ = runnable_four_board_analyzer_process
p._beta_2_mode = True
p.change_magnetometer_config(
{
"magnetometer_config": DEFAULT_MAGNETOMETER_CONFIG,
"sampling_period": DEFAULT_SAMPLING_PERIOD
}, )
put_object_into_queue_and_raise_error_if_eventually_still_empty(
dict(START_MANAGED_ACQUISITION_COMMUNICATION),
comm_from_main_queue,
)
invoke_process_run_and_check_errors(p, perform_setup_before_loop=True)
# load data
num_seconds = MIN_NUM_SECONDS_NEEDED_FOR_ANALYSIS + 1
fill_da_input_data_queue(board_queues[0][0], num_seconds)
invoke_process_run_and_check_errors(p, num_iterations=num_seconds - 1)
# send additional data and time analysis
start = time.perf_counter_ns()
invoke_process_run_and_check_errors(p)
dur_seconds = (time.perf_counter_ns() - start) / 10**9
# prevent BrokenPipeErrors
drain_queue(board_queues[0][1])
drain_queue(comm_to_main_queue)
# print(f"Duration (seconds): {dur_seconds}") # Eli (4/8/21): this is commented code that is deliberately kept in the codebase since it is often toggled on/off during optimization
assert dur_seconds < 2
def test_DataAnalyzerProcess__formats_and_passes_incoming_stim_packet_through_to_main(
four_board_analyzer_process_beta_2_mode, mocker):
da_process = four_board_analyzer_process_beta_2_mode["da_process"]
incoming_data_queue = four_board_analyzer_process_beta_2_mode[
"board_queues"][0][0]
outgoing_data_queue = four_board_analyzer_process_beta_2_mode[
"board_queues"][0][1]
test_stim_packet = copy.deepcopy(SIMPLE_STIM_DATA_PACKET_FROM_ALL_WELLS)
put_object_into_queue_and_raise_error_if_eventually_still_empty(
copy.deepcopy(test_stim_packet), incoming_data_queue)
invoke_process_run_and_check_errors(da_process)
confirm_queue_is_eventually_of_size(outgoing_data_queue, 1)
outgoing_msg = outgoing_data_queue.get(timeout=QUEUE_CHECK_TIMEOUT_SECONDS)
expected_stim_data = {
well_idx: stim_status_arr.tolist()
for well_idx, stim_status_arr in
test_stim_packet["well_statuses"].items()
}
assert outgoing_msg["data_type"] == "stimulation"
assert outgoing_msg["data_json"] == json.dumps(expected_stim_data)
def test_DataAnalyzerProcess__logs_performance_metrics_after_creating_beta_2_data(
four_board_analyzer_process_beta_2_mode, mocker):
da_process = four_board_analyzer_process_beta_2_mode["da_process"]
to_main_queue = four_board_analyzer_process_beta_2_mode["to_main_queue"]
from_main_queue = four_board_analyzer_process_beta_2_mode[
"from_main_queue"]
board_queues = four_board_analyzer_process_beta_2_mode["board_queues"]
# perform setup so performance logging values are initialized
invoke_process_run_and_check_errors(da_process,
perform_setup_before_loop=True)
# set this to a lower value to speed up the test
da_process._minimum_iteration_duration_seconds /= 10
# mock functions to speed up test
mocker.patch.object(data_analyzer,
"get_force_signal",
autospec=True,
return_value=np.zeros((2, 2)))
mocker.patch.object(data_analyzer,
"peak_detector",
autospec=True,
side_effect=PeakDetectionError())
# set magnetometer configuration
expected_sampling_period_us = 10000
num_data_points_per_second = MICRO_TO_BASE_CONVERSION // expected_sampling_period_us
set_magnetometer_config(
four_board_analyzer_process_beta_2_mode,
{
"magnetometer_config": GENERIC_BOARD_MAGNETOMETER_CONFIGURATION,
"sampling_period": expected_sampling_period_us,
},
)
# start managed acquisition
start_command = get_mutable_copy_of_START_MANAGED_ACQUISITION_COMMUNICATION(
)
put_object_into_queue_and_raise_error_if_eventually_still_empty(
start_command, from_main_queue)
invoke_process_run_and_check_errors(da_process)
# remove command receipt
to_main_queue.get(timeout=QUEUE_CHECK_TIMEOUT_SECONDS)
# create expected durations for metric creation
expected_num_data_packets = MIN_NUM_SECONDS_NEEDED_FOR_ANALYSIS
expected_data_creation_durs = [
random.uniform(30, 70) for _ in range(expected_num_data_packets)
]
mocker.patch.object(
data_analyzer,
"_get_secs_since_data_creation_start",
autospec=True,
side_effect=expected_data_creation_durs,
)
expected_data_analysis_durs = [random.uniform(20, 80) for _ in range(24)]
mocker.patch.object(
data_analyzer,
"_get_secs_since_data_analysis_start",
autospec=True,
side_effect=expected_data_analysis_durs,
)
# create test data packets
for packet_num in range(expected_num_data_packets):
test_packet = copy.deepcopy(
SIMPLE_BETA_2_CONSTRUCT_DATA_FROM_ALL_WELLS)
test_packet["time_indices"] = (np.arange(
num_data_points_per_second * packet_num,
num_data_points_per_second * (packet_num + 1),
dtype=np.int64,
) * expected_sampling_period_us)
put_object_into_queue_and_raise_error_if_eventually_still_empty(
test_packet, board_queues[0][0])
invoke_process_run_and_check_errors(da_process)
confirm_queue_is_eventually_of_size(
to_main_queue, expected_num_data_packets * 2
) # Tanner (1/4/21): a log message is also put into queue after each waveform data dump
actual = drain_queue(to_main_queue)[-1]["message"]
assert actual["communication_type"] == "performance_metrics"
assert actual["data_creation_duration"] == expected_data_creation_durs[-1]
assert actual["data_creation_duration_metrics"] == {
"max":
max(expected_data_creation_durs),
"min":
min(expected_data_creation_durs),
"stdev":
round(stdev(expected_data_creation_durs), 6),
"mean":
round(
sum(expected_data_creation_durs) /
len(expected_data_creation_durs), 6),
}
assert actual["data_analysis_duration_metrics"] == {
"max":
max(expected_data_analysis_durs),
"min":
min(expected_data_analysis_durs),
"stdev":
round(stdev(expected_data_analysis_durs), 6),
"mean":
round(
sum(expected_data_analysis_durs) /
len(expected_data_analysis_durs), 6),
}
# values created in parent class
assert "start_timepoint_of_measurements" not in actual
assert "idle_iteration_time_ns" not in actual
assert "longest_iterations" in actual
assert "percent_use" in actual
assert "percent_use_metrics" in actual
# prevent BrokenPipeErrors
drain_queue(board_queues[0][1])
def test_put_object_into_queue_and_raise_error_if_eventually_still_empty__puts_object_into_queue():
expected = "bob"
q = Queue()
put_object_into_queue_and_raise_error_if_eventually_still_empty(expected, q)
actual = q.get_nowait()
assert actual == expected
def test_DataAnalyzerProcess__sends_outgoing_data_dict_to_main_as_soon_as_it_retrieves_a_data_packet_from_file_writer__and_sends_data_available_message_to_main(
four_board_analyzer_process_beta_2_mode, mocker):
da_process = four_board_analyzer_process_beta_2_mode["da_process"]
from_main_queue = four_board_analyzer_process_beta_2_mode[
"from_main_queue"]
to_main_queue = four_board_analyzer_process_beta_2_mode["to_main_queue"]
incoming_data_queue = four_board_analyzer_process_beta_2_mode[
"board_queues"][0][0]
outgoing_data_queue = four_board_analyzer_process_beta_2_mode[
"board_queues"][0][1]
spied_fix = mocker.spy(data_analyzer, "fix_dropped_samples")
# mock so that well metrics don't populate outgoing data queue
mocker.patch.object(da_process,
"_dump_outgoing_well_metrics",
autospec=True)
# mock so performance log messages don't populate queue to main
mocker.patch.object(da_process,
"_handle_performance_logging",
autospec=True)
da_process.init_streams()
# set config arbitrary sampling period
test_sampling_period = 1000
set_magnetometer_config(
four_board_analyzer_process_beta_2_mode,
{
"magnetometer_config": GENERIC_BOARD_MAGNETOMETER_CONFIGURATION,
"sampling_period": test_sampling_period,
},
)
# start managed_acquisition
put_object_into_queue_and_raise_error_if_eventually_still_empty(
dict(START_MANAGED_ACQUISITION_COMMUNICATION), from_main_queue)
invoke_process_run_and_check_errors(da_process)
confirm_queue_is_eventually_of_size(to_main_queue, 1)
# remove message to main
to_main_queue.get(timeout=QUEUE_CHECK_TIMEOUT_SECONDS)
invoke_process_run_and_check_errors(da_process)
confirm_queue_is_eventually_empty(outgoing_data_queue)
confirm_queue_is_eventually_empty(to_main_queue)
test_data_packet = copy.deepcopy(
SIMPLE_BETA_2_CONSTRUCT_DATA_FROM_ALL_WELLS)
# fix time indices so it doesn't create "ZeroDivisionError: float division"
test_data_packet["time_indices"] = np.arange(
test_data_packet["time_indices"].shape[0])
put_object_into_queue_and_raise_error_if_eventually_still_empty(
copy.deepcopy(test_data_packet), incoming_data_queue)
invoke_process_run_and_check_errors(da_process)
assert spied_fix.call_count == 24
confirm_queue_is_eventually_of_size(outgoing_data_queue, 1)
confirm_queue_is_eventually_of_size(to_main_queue, 1)
# test data dump
waveform_data_points = dict()
filter_coefficients = create_filter(BUTTERWORTH_LOWPASS_30_UUID,
test_sampling_period)
for well_idx in range(24):
default_channel_data = test_data_packet[well_idx][
SERIAL_COMM_DEFAULT_DATA_CHANNEL]
fixed_default_channel_data = fix_dropped_samples(default_channel_data)
flipped_default_channel_data = (
(fixed_default_channel_data.astype(np.int32) -
max(fixed_default_channel_data)) * -1 +
MEMSIC_CENTER_OFFSET).astype(np.uint16)
compressed_data = get_force_signal(
np.array([
test_data_packet["time_indices"], flipped_default_channel_data
], np.int64),
filter_coefficients,
)
waveform_data_points[well_idx] = {
"x_data_points":
compressed_data[0].tolist(),
"y_data_points":
(compressed_data[1] * MICRO_TO_BASE_CONVERSION).tolist(),
}
expected_outgoing_dict = {
"waveform_data": {
"basic_data": {
"waveform_data_points": waveform_data_points
}
},
"earliest_timepoint": test_data_packet["time_indices"][0].item(),
"latest_timepoint": test_data_packet["time_indices"][-1].item(),
"num_data_points": len(test_data_packet["time_indices"]),
}
outgoing_msg = outgoing_data_queue.get(timeout=QUEUE_CHECK_TIMEOUT_SECONDS)
assert outgoing_msg["data_type"] == "waveform_data"
assert outgoing_msg["data_json"] == json.dumps(expected_outgoing_dict)
# test message sent to main
outgoing_msg = to_main_queue.get(timeout=QUEUE_CHECK_TIMEOUT_SECONDS)
expected_msg = {
"communication_type": "data_available",
"timestamp": "2021-06-15 16:39:10.120589",
"num_data_points": len(test_data_packet["time_indices"]),
"earliest_timepoint": test_data_packet["time_indices"][0],
"latest_timepoint": test_data_packet["time_indices"][-1],
}
assert outgoing_msg == expected_msg
def test_DataAnalyzerProcess__does_not_process_data_packets_after_receiving_stop_managed_acquisition_command_until_receiving_first_packet_of_new_stream(
four_board_analyzer_process_beta_2_mode, mocker):
da_process = four_board_analyzer_process_beta_2_mode["da_process"]
from_main_queue = four_board_analyzer_process_beta_2_mode[
"from_main_queue"]
to_main_queue = four_board_analyzer_process_beta_2_mode["to_main_queue"]
incoming_data_queue = four_board_analyzer_process_beta_2_mode[
"board_queues"][0][0]
# mock so these since not using real data
mocked_process_data = mocker.patch.object(da_process,
"_process_beta_2_data",
autospec=True,
return_value={})
invoke_process_run_and_check_errors(da_process,
perform_setup_before_loop=True)
# set config arbitrary sampling period
test_sampling_period = 10000
set_magnetometer_config(
four_board_analyzer_process_beta_2_mode,
{
"magnetometer_config": GENERIC_BOARD_MAGNETOMETER_CONFIGURATION,
"sampling_period": test_sampling_period,
},
)
# start managed_acquisition
put_object_into_queue_and_raise_error_if_eventually_still_empty(
dict(START_MANAGED_ACQUISITION_COMMUNICATION), from_main_queue)
invoke_process_run_and_check_errors(da_process)
# send first packet of first stream and make sure it is processed
test_data_packet = copy.deepcopy(
SIMPLE_BETA_2_CONSTRUCT_DATA_FROM_ALL_WELLS)
test_data_packet["is_first_packet_of_stream"] = True
put_object_into_queue_and_raise_error_if_eventually_still_empty(
test_data_packet, incoming_data_queue)
invoke_process_run_and_check_errors(da_process)
assert mocked_process_data.call_count == 1
# send another packet of first stream and make sure it is processed
test_data_packet = copy.deepcopy(
SIMPLE_BETA_2_CONSTRUCT_DATA_FROM_ALL_WELLS)
test_data_packet["is_first_packet_of_stream"] = False
put_object_into_queue_and_raise_error_if_eventually_still_empty(
test_data_packet, incoming_data_queue)
invoke_process_run_and_check_errors(da_process)
assert mocked_process_data.call_count == 2
# stop managed acquisition and make sure next data packet in the first stream is not processed
put_object_into_queue_and_raise_error_if_eventually_still_empty(
dict(STOP_MANAGED_ACQUISITION_COMMUNICATION), from_main_queue)
invoke_process_run_and_check_errors(da_process)
test_data_packet = copy.deepcopy(
SIMPLE_BETA_2_CONSTRUCT_DATA_FROM_ALL_WELLS)
test_data_packet["is_first_packet_of_stream"] = False
put_object_into_queue_and_raise_error_if_eventually_still_empty(
test_data_packet, incoming_data_queue)
invoke_process_run_and_check_errors(da_process)
assert mocked_process_data.call_count == 2
# start managed acquisition again and make sure next data packet in the first stream is not processed
put_object_into_queue_and_raise_error_if_eventually_still_empty(
dict(START_MANAGED_ACQUISITION_COMMUNICATION), from_main_queue)
invoke_process_run_and_check_errors(da_process)
test_data_packet = copy.deepcopy(
SIMPLE_BETA_2_CONSTRUCT_DATA_FROM_ALL_WELLS)
test_data_packet["is_first_packet_of_stream"] = False
put_object_into_queue_and_raise_error_if_eventually_still_empty(
test_data_packet, incoming_data_queue)
invoke_process_run_and_check_errors(da_process)
assert mocked_process_data.call_count == 2
# send first data packet from second stream and make sure it is processed
test_data_packet = copy.deepcopy(
SIMPLE_BETA_2_CONSTRUCT_DATA_FROM_ALL_WELLS)
test_data_packet["is_first_packet_of_stream"] = True
put_object_into_queue_and_raise_error_if_eventually_still_empty(
test_data_packet, incoming_data_queue)
invoke_process_run_and_check_errors(da_process)
assert mocked_process_data.call_count == 3
# prevent BrokenPipeErrors
drain_queue(to_main_queue)
