def interleaved_timestamp_and_numpy_data(): """Test reading timestamp data interleaved with a standard numpy data type """ times = [ np.datetime64('2012-08-23T00:00:00.123', 'us'), np.datetime64('2012-08-23T01:02:03.456', 'us'), ] metadata = ( # Number of objects "02 00 00 00" # Length of the object path "16 00 00 00") metadata += string_hexlify("/'Group'/'TimeChannel'") metadata += ( # Length of index information "14 00 00 00" # Raw data data type "44 00 00 00" # Dimension "01 00 00 00" # Number of raw data values "02 00 00 00" "00 00 00 00" # Number of properties (0) "00 00 00 00") metadata += ("15 00 00 00") metadata += string_hexlify("/'Group'/'IntChannel'") metadata += ( # Length of index information "14 00 00 00" # Raw data data type "03 00 00 00" # Dimension "01 00 00 00" # Number of raw data values "02 00 00 00" "00 00 00 00" # Number of properties (0) "00 00 00 00") data = (timestamp_data_chunk([times[0]]) + "01 00 00 00" + timestamp_data_chunk([times[1]]) + "02 00 00 00") test_file = GeneratedFile() toc = ("kTocMetaData", "kTocRawData", "kTocNewObjList", "kTocInterleavedData") test_file.add_segment(toc, metadata, data) expected_data = { ('Group', 'TimeChannel'): np.array([times[0], times[1]]), ('Group', 'IntChannel'): np.array([1, 2], dtype=np.dtype('int32')), } return test_file, expected_data
def interleaved_timestamp_data(): """Test reading interleaved timestamp data """ times = [ np.datetime64('2012-08-23T00:00:00.123', 'us'), np.datetime64('2012-08-23T01:02:03.456', 'us'), np.datetime64('2012-08-23T12:00:00.0', 'us'), np.datetime64('2012-08-23T12:02:03.9999', 'us'), ] metadata = ( # Number of objects "02 00 00 00" # Length of the object path "17 00 00 00") metadata += string_hexlify("/'Group'/'TimeChannel1'") metadata += ( # Length of index information "14 00 00 00" # Raw data data type "44 00 00 00" # Dimension "01 00 00 00" # Number of raw data values "02 00 00 00" "00 00 00 00" # Number of properties (0) "00 00 00 00") metadata += ("17 00 00 00") metadata += string_hexlify("/'Group'/'TimeChannel2'") metadata += ( # Length of index information "14 00 00 00" # Raw data data type "44 00 00 00" # Dimension "01 00 00 00" # Number of raw data values "02 00 00 00" "00 00 00 00" # Number of properties (0) "00 00 00 00") test_file = GeneratedFile() toc = ("kTocMetaData", "kTocRawData", "kTocNewObjList", "kTocInterleavedData") test_file.add_segment(toc, metadata, timestamp_data_chunk(times)) expected_data = { ('Group', 'TimeChannel1'): np.array([times[0], times[2]]), ('Group', 'TimeChannel2'): np.array([times[1], times[3]]), } return test_file, expected_data
def test_unicode_string_data(tmp_path): """ Test HDF5 conversion for string datatype with non-ASCII data """ strings = ["Hello, \u4E16\u754C", "\U0001F600"] sizes = [len(s.encode('utf-8')) for s in strings] test_file = GeneratedFile() toc = ("kTocMetaData", "kTocRawData", "kTocNewObjList") metadata = ( # Number of objects "01 00 00 00" # Length of the object path "11 00 00 00") metadata += string_hexlify("/'Group'/'String'") metadata += ( # Length of index information "1C 00 00 00" # Raw data data type "20 00 00 00" # Dimension "01 00 00 00" # Number of raw data values "02 00 00 00" "00 00 00 00" + # Number of bytes in data, including index hexlify_value('q', sum(sizes) + 4 * len(sizes)) + # Number of properties (0) "00 00 00 00") data = "" offset = 0 for size in sizes: # Index gives end positions of strings: offset += size data += hexlify_value('i', offset) for string in strings: data += string_hexlify(string) test_file.add_segment(toc, metadata, data) tdms_data = test_file.load() data = tdms_data["Group"]["String"].data assert len(data) == len(strings) for expected, read in zip(strings, data): assert expected == read h5_path = tmp_path / 'h5_unicode_strings_test.h5' h5 = tdms_data.as_hdf(h5_path) h5_strings = h5['Group']['String'] assert h5_strings.dtype.kind == 'O' assert h5_strings.shape[0] == len(strings) for expected, read in zip(strings, h5_strings[...]): assert expected == read h5.close()
def test_as_hdf_string(tmp_path): """ Test HDF5 conversion for string datatype """ strings = ["abc123", "?<>~`!@#$%^&*()-=_+,.;'[]:{}|"] test_file = GeneratedFile() toc = ("kTocMetaData", "kTocRawData", "kTocNewObjList") metadata = ( # Number of objects "01 00 00 00" # Length of the object path "11 00 00 00") metadata += string_hexlify("/'Group'/'String'") metadata += ( # Length of index information "1C 00 00 00" # Raw data data type "20 00 00 00" # Dimension "01 00 00 00" # Number of raw data values "02 00 00 00" "00 00 00 00" # Number of bytes in data "2B 00 00 00" "00 00 00 00" # Number of properties (0) "00 00 00 00") data = ( "06 00 00 00" # index to after first string "24 00 00 00" # index to after second string ) for string in strings: data += string_hexlify(string) test_file.add_segment(toc, metadata, data) tdms_data = test_file.load() data = tdms_data["Group"]["String"].data assert len(data) == len(strings) for expected, read in zip(strings, data): assert expected == read h5_path = tmp_path / 'h5_strings_test.h5' h5 = tdms_data.as_hdf(h5_path) h5_strings = h5['Group']['String'] assert h5_strings.dtype.kind == 'O' assert h5_strings.shape[0] == len(strings) for expected, read in zip(strings, h5_strings[...]): assert expected == read h5.close()
def daqmx_channel_metadata( channel_name, num_values, raw_data_widths, scaler_metadata, properties=None, digital_line_scaler=False): path = "/'Group'/'" + channel_name + "'" return ( # Length of the object path hexlify_value("<I", len(path)) + # Object path string_hexlify(path) + # Raw data index (DAQmx) ("6A 12 00 00" if digital_line_scaler else "69 12 00 00") + # Data type (DAQmx) "FF FF FF FF" # Array dimension "01 00 00 00" + # Number of values (chunk size) hexlify_value("<Q", num_values) + # Scaler metadata hexlify_value("<I", len(scaler_metadata)) + "".join(scaler_metadata) + # Raw data width vector size hexlify_value("<I", len(raw_data_widths)) + # Raw data width values "".join(hexlify_value("<I", v) for v in raw_data_widths) + hex_properties(properties))
def test_stream_scaled_data_chunks(benchmark): """ Benchmark streaming channel data when the data is scaled """ properties = { "NI_Number_Of_Scales": (3, "01 00 00 00"), "NI_Scale[0]_Scale_Type": (0x20, hexlify_value("<I", len("Linear")) + string_hexlify("Linear")), "NI_Scale[0]_Linear_Slope": (10, hexlify_value("<d", 2.0)), "NI_Scale[0]_Linear_Y_Intercept": (10, hexlify_value("<d", 10.0)) } test_file = GeneratedFile() data_array = np.arange(0, 1000, dtype=np.dtype('int32')) data = data_array.tobytes() test_file.add_segment( ("kTocMetaData", "kTocRawData", "kTocNewObjList"), segment_objects_metadata( channel_metadata("/'group'/'channel1'", TDS_TYPE_INT32, 100, properties), ), data, binary_data=True) for _ in range(0, 9): test_file.add_segment(("kTocRawData", ), "", data, binary_data=True) with TdmsFile.open(test_file.get_bytes_io_file()) as tdms_file: channel = tdms_file['group']['channel1'] channel_data = benchmark(stream_chunks, channel) channel_data = np.concatenate(channel_data) expected_data = np.tile(10.0 + 2.0 * data_array, 10) np.testing.assert_equal(channel_data, expected_data)
def test_string_data(): """Test reading a file with string data""" strings = ["abcdefg", "qwertyuiop"] test_file = GeneratedFile() toc = ("kTocMetaData", "kTocRawData", "kTocNewObjList") metadata = ( # Number of objects "01 00 00 00" # Length of the object path "18 00 00 00") metadata += string_hexlify("/'Group'/'StringChannel'") metadata += ( # Length of index information "1C 00 00 00" # Raw data data type "20 00 00 00" # Dimension "01 00 00 00" # Number of raw data values "02 00 00 00" "00 00 00 00" # Number of bytes in data "19 00 00 00" "00 00 00 00" # Number of properties (0) "00 00 00 00") data = ( "07 00 00 00" # index to after first string "11 00 00 00" # index to after second string ) for string in strings: data += string_hexlify(string) test_file.add_segment(toc, metadata, data) tdms_data = test_file.load() channel = tdms_data["Group"]["StringChannel"] assert len(channel.data) == len(strings) assert channel.data.dtype == channel.dtype for expected, read in zip(strings, channel.data): assert expected == read
def scaled_data(): properties = { "NI_Number_Of_Scales": (3, "01 00 00 00"), "NI_Scale[0]_Scale_Type": (0x20, hexlify_value("<I", len("Linear")) + string_hexlify("Linear")), "NI_Scale[0]_Linear_Slope": (10, hexlify_value("<d", 2.0)), "NI_Scale[0]_Linear_Y_Intercept": (10, hexlify_value("<d", 10.0)) } test_file = GeneratedFile() test_file.add_segment( ("kTocMetaData", "kTocRawData", "kTocNewObjList"), segment_objects_metadata( channel_metadata("/'group'/'channel1'", TDS_TYPE_INT32, 2, properties), ), "01 00 00 00" "02 00 00 00" "03 00 00 00" "04 00 00 00") expected_data = { ('group', 'channel1'): np.array([12, 14, 16, 18], dtype=np.float64), } return test_file, expected_data
def timed_segment(): """TDMS segment with one group and two channels, each with time properties""" toc = ("kTocMetaData", "kTocRawData", "kTocNewObjList") metadata = ( # Number of objects "03 00 00 00" # Length of the first object path "08 00 00 00" # Object path (/'Group') "2F 27 47 72" "6F 75 70 27" # Raw data index "FF FF FF FF" # Num properties "02 00 00 00" # Name length "04 00 00 00" # Property name (prop) "70 72 6F 70" # Property data type (string) "20 00 00 00" # Length of string value "05 00 00 00" # Value "76 61 6C 75 65" # Length of second property name "03 00 00 00" # Property name (num) "6E 75 6D" # Data type of property "03 00 00 00" # Value "0A 00 00 00" # Length of the second object path "13 00 00 00" # Second object path (/'Group'/'Channel1') "2F 27 47 72" "6F 75 70 27" "2F 27 43 68" "61 6E 6E 65" "6C 31 27" # Length of index information "14 00 00 00" # Raw data data type "03 00 00 00" # Dimension "01 00 00 00" # Number of raw data values "02 00 00 00" "00 00 00 00" # Number of properties "03 00 00 00" # Set time properties for the first channel "0F 00 00 00" + string_hexlify('wf_start_offset') + "0A 00 00 00" + hexlify_value("<d", 2.0) + "0C 00 00 00" + string_hexlify('wf_increment') + "0A 00 00 00" + hexlify_value("<d", 0.1) + "0D 00 00 00" + string_hexlify('wf_start_time') + "44 00 00 00" + hexlify_value("<Q", 0) + hexlify_value("<q", 3524551547) + # Length of the third object path "13 00 00 00" # Third object path (/'Group'/'Channel2') "2F 27 47 72" "6F 75 70 27" "2F 27 43 68" "61 6E 6E 65" "6C 32 27" # Length of index information "14 00 00 00" # Raw data data type "03 00 00 00" # Dimension "01 00 00 00" # Number of data values "02 00 00 00" "00 00 00 00" # Number of properties "03 00 00 00" # Set time properties for the second channel "0F 00 00 00" + string_hexlify('wf_start_offset') + "0A 00 00 00" + hexlify_value("<d", 2.0) + "0C 00 00 00" + string_hexlify('wf_increment') + "0A 00 00 00" + hexlify_value("<d", 0.1) + "0D 00 00 00" + string_hexlify('wf_start_time') + "44 00 00 00" + hexlify_value("<Q", 0) + hexlify_value("<q", 3524551547)) data = ( # Data for segment "01 00 00 00" "02 00 00 00" "03 00 00 00" "04 00 00 00") return toc, metadata, data