def startReading(pathToData, guiQueue, diagnosisQueue): with TdmsFile.open(pathToData) as tdms_file: for group in tdms_file.groups(): all_group_channels = group.channels() sensorNumber = 0 for channel in all_group_channels: logging.info("Reading sensor: {}".format( sensorDictionary[sensorNumber])) for chunk in channel.data_chunks(): channel_chunk_data = chunk[:] if guiQueue is not None: with GUILOCK: guiQueue.put(channel_chunk_data) if diagnosisQueue is not None: with DIAGNOSIS_LOCK: diagnosisQueue.put(channel_chunk_data) time.sleep(0.9) if keyboard.is_pressed("q"): sys.exit() with CURRENT_SENSOR_LOCK: global CURRENT_SENSOR if CURRENT_SENSOR != 2: CURRENT_SENSOR += 1 sensorNumber += 1 else: return
def _get_tdms_file_property(self, log_file_base_path: str, log_file_name: str, property_name: str) -> Optional[str]: log_file_path = Path(log_file_base_path) / log_file_name with TdmsFile.open(str(log_file_path)) as tdms_file: real_property_name = f"Test_properties~{property_name}" return tdms_file.properties.get(real_property_name)
def on_test_file(test_file, lazy_load, func): if lazy_load: with test_file.get_tempfile() as temp_file: with TdmsFile.open(temp_file) as tdms_file: return func(tdms_file) else: return func(test_file.load())
def test_reading_subset_of_data(offset, length): channel_data = np.arange(0, 100, 1, dtype=np.int32) # Split data into different sized segments segment_data = [ channel_data[0:10], channel_data[10:20], channel_data[20:60], channel_data[60:80], channel_data[80:90], channel_data[90:100], ] hex_segment_data = [ "".join(hexlify_value('<i', x) for x in data) for data in segment_data ] test_file = GeneratedFile() test_file.add_segment( ("kTocMetaData", "kTocRawData", "kTocNewObjList"), segment_objects_metadata(channel_metadata("/'group'/'channel1'", 3, 5), ), hex_segment_data[0]) for hex_data in hex_segment_data[1:]: test_file.add_segment(("kTocRawData", ), "", hex_data) with test_file.get_tempfile() as temp_file: with TdmsFile.open(temp_file.file) as tdms_file: channel_subset = tdms_file['group']['channel1'].read_data( offset, length) expected_data = channel_data[offset:offset + length] assert len(channel_subset) == len(expected_data) np.testing.assert_equal(channel_subset, expected_data)
def load_tdms_video(path, meta_path, frame=None): """ Calculate the 3 dimensions for a given video from TDMS metadata and reshape the video to these dimensions. Parameters ---------- path : pathlib.Path File path to TDMS video file. meta_path : pathlib.Path File path to TDMS file containing metadata about the video. frame : int, optional Read this one single frame rather than them all. """ fps, height, duration = _parse_tdms_metadata(meta_path) if frame is None: video = read_tdms(path) width = int(video.size / (duration * height)) return video.values.reshape(duration, height, width), fps with TdmsFile.open(path) as tdms_file: group = tdms_file.groups()[0] channel = group.channels()[0] width = int(len(channel) / (duration * height)) length = width * height start = frame * length video = channel[start:start + length] return video.reshape(height, width), fps
def test_stream_scaled_data_chunks(benchmark): """ Benchmark streaming channel data when the data is scaled """ properties = { "NI_Number_Of_Scales": (3, "01 00 00 00"), "NI_Scale[0]_Scale_Type": (0x20, hexlify_value("<I", len("Linear")) + string_hexlify("Linear")), "NI_Scale[0]_Linear_Slope": (10, hexlify_value("<d", 2.0)), "NI_Scale[0]_Linear_Y_Intercept": (10, hexlify_value("<d", 10.0)) } test_file = GeneratedFile() data_array = np.arange(0, 1000, dtype=np.dtype('int32')) data = data_array.tobytes() test_file.add_segment( ("kTocMetaData", "kTocRawData", "kTocNewObjList"), segment_objects_metadata( channel_metadata("/'group'/'channel1'", TDS_TYPE_INT32, 100, properties), ), data, binary_data=True) for _ in range(0, 9): test_file.add_segment(("kTocRawData", ), "", data, binary_data=True) with TdmsFile.open(test_file.get_bytes_io_file()) as tdms_file: channel = tdms_file['group']['channel1'] channel_data = benchmark(stream_chunks, channel) channel_data = np.concatenate(channel_data) expected_data = np.tile(10.0 + 2.0 * data_array, 10) np.testing.assert_equal(channel_data, expected_data)
def opened_tdms_file(): """ Allow re-use of an opened TDMS file """ test_file, expected_data = scenarios.chunked_segment().values with test_file.get_tempfile() as temp_file: with TdmsFile.open(temp_file.file) as tdms_file: yield tdms_file, expected_data
def test_indexing_timestamp_channel_with_integer(index): """ Test indexing into a timestamp data channel with an integer index """ test_file, expected_data = scenarios.timestamp_data().values with test_file.get_tempfile() as temp_file: with TdmsFile.open(temp_file.file) as tdms_file: for ((group, channel), expected_channel_data) in expected_data.items(): channel_object = tdms_file[group][channel] assert channel_object[index] == expected_channel_data[index]
def test_lazily_read_channel_data(test_file, expected_data): """Test reading channel data lazily""" with test_file.get_tempfile() as temp_file: with TdmsFile.open(temp_file.file) as tdms_file: for ((group, channel), expected_data) in expected_data.items(): actual_data = tdms_file[group][channel].read_data() assert actual_data.dtype == expected_data.dtype compare_arrays(actual_data, expected_data)
def test_reading_subset_of_data_for_scenario(test_file, expected_data, offset, length): """Test reading a subset of a channel's data """ assume(any(offset <= len(d) for d in expected_data.values())) with test_file.get_tempfile() as temp_file: with TdmsFile.open(temp_file.file) as tdms_file: for ((group, channel), expected_data) in expected_data.items(): actual_data = tdms_file[group][channel].read_data(offset, length) compare_arrays(actual_data, expected_data[offset:offset + length])
def test_slice_contiguous_data_channel(benchmark): """ Benchmark reading a slice of data from a contiguous data file """ with TdmsFile.open(get_contiguous_file().get_bytes_io_file()) as tdms_file: channel = tdms_file['group']['channel3'] channel_data = benchmark(get_slice, channel, 5555, 6555) expected_data = np.repeat([3], 1000) np.testing.assert_equal(channel_data, expected_data)
def test_indexing_channel_with_ellipsis(): """ Test indexing into a channel with ellipsis returns all data """ test_file, expected_data = scenarios.chunked_segment().values with test_file.get_tempfile() as temp_file: with TdmsFile.open(temp_file.file) as tdms_file: for ((group, channel), expected_channel_data) in expected_data.items(): channel_object = tdms_file[group][channel] compare_arrays(channel_object[...], expected_channel_data)
def import_data(file): """Read tdms file""" df = None channel_list = [] group_list = [] with TdmsFile.open(file) as f: print("opening", file) for group in f.groups(): group_name = group.name group_list.append(group_name) for channel in group.channels(): channel_name = channel.name channel_list.append(channel_name) #channel_name = channel.name #Access dictionary of properties: #properties = channel.properties #Access numpy array of data for channel: #data = channel[:] # select group selected_group = group_list[0] time = channel.time_track() # convert tim = numpy array to pandas df time_df = pd.DataFrame(time, columns = ['Time']) print(time_df[:10]) #data_df = f["analog"].as_dataframe() data_df = f[selected_group].as_dataframe() print(data_df[:10]) # merge two pandas df merged_df = pd.concat([time_df, data_df], axis=1).reindex(time_df.index) print(merged_df.head()) print(merged_df.tail()) print(merged_df.shape) #time = channel.time_track() #print(time) #print(f.groups()) #print(f.channels()) #channel = tdms_file[group_name][channel_name] #channel_data = channel[:] # tdms_file = TdmsFile.read(file) # # group_list = [] # channel_list = [] return merged_df
def test_stream_contiguous_data_channel(benchmark): """ Benchmark streaming channel data from a contiguous data file """ with TdmsFile.open(get_contiguous_file().get_bytes_io_file()) as tdms_file: channel = tdms_file['group']['channel3'] channel_data = benchmark(stream_chunks, channel) channel_data = np.concatenate(channel_data) expected_data = np.repeat([3], 10000) np.testing.assert_equal(channel_data, expected_data)
def test_read_interleaved_data_channel(benchmark): """ Benchmark reading a single channel's data from an interleaved data file """ with TdmsFile.open( get_interleaved_file().get_bytes_io_file()) as tdms_file: channel = tdms_file['group']['channel3'] channel_data = benchmark(read_channel_data, channel) expected_data = np.repeat([3], 10000) np.testing.assert_equal(channel_data, expected_data)
def test_index_contiguous_data_channel(benchmark): """ Benchmark reading a data from a contiguous data file using integer indices """ with TdmsFile.open(get_contiguous_file().get_bytes_io_file()) as tdms_file: channel = tdms_file['group']['channel3'] channel_data = np.zeros(10000, dtype=channel.dtype) benchmark(index_values, channel, channel_data) expected_data = np.repeat([3], 10000) np.testing.assert_equal(channel_data, expected_data)
def test_indexing_channel_with_invalid_integer_raises_error(index): """ Test indexing into a channel with an invalid integer index """ test_file, expected_data = scenarios.chunked_segment().values with test_file.get_tempfile() as temp_file: with TdmsFile.open(temp_file.file) as tdms_file: for ((group, channel), expected_channel_data) in expected_data.items(): channel_object = tdms_file[group][channel] with pytest.raises(IndexError): _ = channel_object[index]
def test_lazily_read_raw_channel_data_slice(): """Test reading raw channel data lazily""" test_file, expected_data = scenarios.single_segment_with_one_channel().values with test_file.get_tempfile() as temp_file: with TdmsFile.open(temp_file.file) as tdms_file: for ((group, channel), expected_data) in expected_data.items(): actual_data = tdms_file[group][channel].read_data(offset=1, length=2, scaled=False) assert actual_data.dtype == expected_data.dtype compare_arrays(actual_data, expected_data[1:3])
def test_multiple_close_after_open(): test_file, _ = scenarios.single_segment_with_one_channel().values temp_file = test_file.get_tempfile(delete=False) try: temp_file.file.close() with TdmsFile.open(temp_file.name) as tdms_data: tdms_data.close() tdms_data.close() finally: os.remove(temp_file.name)
def test_invalid_length_in_read_data_throws(): """ Exception is thrown when reading a subset of data with an invalid length """ test_file, expected_data = scenarios.single_segment_with_one_channel().values group, channel = list(expected_data.keys())[0] with test_file.get_tempfile() as temp_file: with TdmsFile.open(temp_file.file) as tdms_file: with pytest.raises(ValueError) as exc_info: tdms_file[group][channel].read_data(0, -5) assert "length must be non-negative" in str(exc_info.value)
def test_lazily_read_channel_data_with_channel_data_method(): """Test reading channel data lazily using the channel_data method of TdmsFile """ test_file, expected_data = scenarios.single_segment_with_two_channels( ).values with test_file.get_tempfile() as temp_file: with TdmsFile.open(temp_file.file) as tdms_file: for ((group, channel), expected_data) in expected_data.items(): actual_data = tdms_file.channel_data(group, channel) assert actual_data.dtype == expected_data.dtype np.testing.assert_almost_equal(actual_data, expected_data)
def test_stream_channel_data_chunks(test_file, expected_data): """Test streaming chunks of data for a single channel from a TDMS file """ with test_file.get_tempfile() as temp_file: with TdmsFile.open(temp_file.file) as tdms_file: for ((group, channel), expected_channel_data) in expected_data.items(): actual_data = [] for chunk in tdms_file[group][channel].data_chunks(): assert chunk.offset == len(actual_data) actual_data.extend(chunk[:]) compare_arrays(actual_data, expected_channel_data)
def test_read_data_after_close_throws(): """ Trying to read after opening and closing without reading data should throw """ test_file, expected_data = scenarios.single_segment_with_one_channel().values group, channel = list(expected_data.keys())[0] with test_file.get_tempfile() as temp_file: with TdmsFile.open(temp_file.file) as tdms_file: pass with pytest.raises(RuntimeError) as exc_info: tdms_file[group][channel].read_data() assert "Cannot read data after the underlying TDMS reader is closed" in str(exc_info.value)
def test_indexing_channel_with_invalid_type_raises_error(index): """ Test indexing into a channel with an invalid index type """ test_file, expected_data = scenarios.chunked_segment().values with test_file.get_tempfile() as temp_file: with TdmsFile.open(temp_file.file) as tdms_file: for ((group, channel), expected_channel_data) in expected_data.items(): channel_object = tdms_file[group][channel] with pytest.raises(TypeError) as exc_info: _ = channel_object[index] assert "Invalid index type" in str(exc_info.value)
def test_indexing_channel_with_zero_step_raises_error(): """ Test indexing into a channel with a slice with zero step size raises an error """ test_file, expected_data = scenarios.chunked_segment().values with test_file.get_tempfile() as temp_file: with TdmsFile.open(temp_file.file) as tdms_file: for ((group, channel), expected_channel_data) in expected_data.items(): channel_object = tdms_file[group][channel] with pytest.raises(ValueError) as exc_info: _ = channel_object[::0] assert str(exc_info.value) == "Step size cannot be zero"
def _verify_tdms_file_has_note( log_file_base_path: str, log_file_name: str, expected_note_contents: str ) -> None: log_file_path = Path(log_file_base_path) / log_file_name with TdmsFile.open(str(log_file_path)) as tdms_file: user_notes_group = tdms_file["Test Information"] user_notes_channel = user_notes_group["User Notes"] assert user_notes_channel is not None user_notes_time_channel = user_notes_group["User Notes_time"] assert user_notes_time_channel is not None assert 1 == len(user_notes_channel) assert expected_note_contents == user_notes_channel[0]
def test_iterate_channel_data_in_open_mode(): """Test iterating over channel data after opening a file without reading data """ test_file, expected_data = scenarios.chunked_segment().values with test_file.get_tempfile() as temp_file: with TdmsFile.open(temp_file.file) as tdms_file: for ((group, channel), expected_channel_data) in expected_data.items(): actual_data = [] for value in tdms_file[group][channel]: actual_data.append(value) compare_arrays(actual_data, expected_channel_data)
def test_indexing_channel_with_integer_and_caching(): """ Test indexing into a channel with an integer index, reusing the same file to test caching """ test_file, expected_data = scenarios.chunked_segment().values with test_file.get_tempfile() as temp_file: with TdmsFile.open(temp_file.file) as tdms_file: for ((group, channel), expected_channel_data) in expected_data.items(): channel_object = tdms_file[group][channel] values = [] for i in range(len(channel_object)): values.append(channel_object[i]) compare_arrays(values, expected_channel_data)
def test_indexing_scaled_channel_with_integer(): """ Test indexing into a channel with an integer index when the channel is scaled """ test_file, expected_data = scenarios.scaled_data().values with test_file.get_tempfile() as temp_file: with TdmsFile.open(temp_file.file) as tdms_file: for ((group, channel), expected_channel_data) in expected_data.items(): channel_object = tdms_file[group][channel] values = [] for i in range(len(channel_object)): values.append(channel_object[i]) compare_arrays(values, expected_channel_data)
def test_streaming_to_hdf(tmp_path): """ Test conversion of channel data to HDF when streaming data from disk """ test_file, expected_data = scenarios.chunked_segment().values with test_file.get_tempfile() as temp_file: with TdmsFile.open(temp_file.file) as tdms_file: h5_path = tmp_path / 'h5_streaming_data_test.h5' h5 = tdms_file.as_hdf(h5_path) for ((group, channel), expected_data) in expected_data.items(): h5_channel = h5[group][channel] np.testing.assert_almost_equal(h5_channel[...], expected_data) h5.close()