def daqmx_channel_metadata( channel_name, num_values, raw_data_widths, scaler_metadata, properties=None, digital_line_scaler=False): path = "/'Group'/'" + channel_name + "'" return ( # Length of the object path hexlify_value("<I", len(path)) + # Object path string_hexlify(path) + # Raw data index (DAQmx) ("6A 12 00 00" if digital_line_scaler else "69 12 00 00") + # Data type (DAQmx) "FF FF FF FF" # Array dimension "01 00 00 00" + # Number of values (chunk size) hexlify_value("<Q", num_values) + # Scaler metadata hexlify_value("<I", len(scaler_metadata)) + "".join(scaler_metadata) + # Raw data width vector size hexlify_value("<I", len(raw_data_widths)) + # Raw data width values "".join(hexlify_value("<I", v) for v in raw_data_widths) + hex_properties(properties))
def float_data(): """ Test reading a file with float valued data """ single_arr = np.array([0.123, 0.234, 0.345, 0.456], dtype=np.float32) double_arr = np.array([0.987, 0.876, 0.765, 0.654], dtype=np.double) data = "" for num in single_arr[0:2]: data += hexlify_value("<f", num) for num in double_arr[0:2]: data += hexlify_value("<d", num) for num in single_arr[2:4]: data += hexlify_value("<f", num) for num in double_arr[2:4]: data += hexlify_value("<d", num) test_file = GeneratedFile() test_file.add_segment( ("kTocMetaData", "kTocRawData", "kTocNewObjList"), segment_objects_metadata( channel_metadata("/'group'/'single_channel'", TDS_TYPE_FLOAT32, 2), channel_metadata("/'group'/'double_channel'", TDS_TYPE_FLOAT64, 2), ), data) expected_data = { ('group', 'single_channel'): single_arr, ('group', 'double_channel'): double_arr, } return test_file, expected_data
def test_stream_scaled_data_chunks(benchmark): """ Benchmark streaming channel data when the data is scaled """ properties = { "NI_Number_Of_Scales": (3, "01 00 00 00"), "NI_Scale[0]_Scale_Type": (0x20, hexlify_value("<I", len("Linear")) + string_hexlify("Linear")), "NI_Scale[0]_Linear_Slope": (10, hexlify_value("<d", 2.0)), "NI_Scale[0]_Linear_Y_Intercept": (10, hexlify_value("<d", 10.0)) } test_file = GeneratedFile() data_array = np.arange(0, 1000, dtype=np.dtype('int32')) data = data_array.tobytes() test_file.add_segment( ("kTocMetaData", "kTocRawData", "kTocNewObjList"), segment_objects_metadata( channel_metadata("/'group'/'channel1'", TDS_TYPE_INT32, 100, properties), ), data, binary_data=True) for _ in range(0, 9): test_file.add_segment(("kTocRawData", ), "", data, binary_data=True) with TdmsFile.open(test_file.get_bytes_io_file()) as tdms_file: channel = tdms_file['group']['channel1'] channel_data = benchmark(stream_chunks, channel) channel_data = np.concatenate(channel_data) expected_data = np.tile(10.0 + 2.0 * data_array, 10) np.testing.assert_equal(channel_data, expected_data)
def float_data_with_unit(): """ Test reading a file with float valued data with units These are the same as normal floating point data but have a 'unit_string' property """ single_arr = np.array([0.123, 0.234, 0.345, 0.456], dtype=np.float32) double_arr = np.array([0.987, 0.876, 0.765, 0.654], dtype=np.double) data = "" for num in single_arr[0:2]: data += hexlify_value("<f", num) for num in double_arr[0:2]: data += hexlify_value("<d", num) for num in single_arr[2:4]: data += hexlify_value("<f", num) for num in double_arr[2:4]: data += hexlify_value("<d", num) test_file = GeneratedFile() test_file.add_segment(("kTocMetaData", "kTocRawData", "kTocNewObjList"), segment_objects_metadata( channel_metadata("/'group'/'single_channel'", TDS_TYPE_FLOAT32_WITH_UNIT, 2), channel_metadata("/'group'/'double_channel'", TDS_TYPE_FLOAT64_WITH_UNIT, 2), ), data) expected_data = { ('group', 'single_channel'): single_arr, ('group', 'double_channel'): double_arr, } return test_file, expected_data
def complex_data(): """ Test reading a file with complex valued data """ complex_single_arr = np.array([1 + 2j, 3 + 4j], dtype=np.complex64) complex_double_arr = np.array([5 + 6j, 7 + 8j], dtype=np.complex128) data = "" for num in complex_single_arr: data += hexlify_value("<f", num.real) data += hexlify_value("<f", num.imag) for num in complex_double_arr: data += hexlify_value("<d", num.real) data += hexlify_value("<d", num.imag) test_file = GeneratedFile() test_file.add_segment( ("kTocMetaData", "kTocRawData", "kTocNewObjList"), segment_objects_metadata( channel_metadata("/'group'/'complex_single_channel'", TDS_TYPE_COMPLEX64, 2), channel_metadata("/'group'/'complex_double_channel'", TDS_TYPE_COMPLEX128, 2), ), data) expected_data = { ('group', 'complex_single_channel'): complex_single_arr, ('group', 'complex_double_channel'): complex_double_arr, } return test_file, expected_data
def digital_scaler_metadata(scale_id, type_id, bit_offset, raw_buffer_index=0): return ( # DAQmx data type (type ids don't match TDMS types) hexlify_value("<I", type_id) + # Raw buffer index hexlify_value("<I", raw_buffer_index) + # Raw byte offset hexlify_value("<I", bit_offset) + # Sample format bitmap (don't know what this is for...) "00" + # Scale ID hexlify_value("<I", scale_id))
def test_unicode_string_data(tmp_path): """ Test HDF5 conversion for string datatype with non-ASCII data """ strings = ["Hello, \u4E16\u754C", "\U0001F600"] sizes = [len(s.encode('utf-8')) for s in strings] test_file = GeneratedFile() toc = ("kTocMetaData", "kTocRawData", "kTocNewObjList") metadata = ( # Number of objects "01 00 00 00" # Length of the object path "11 00 00 00") metadata += string_hexlify("/'Group'/'String'") metadata += ( # Length of index information "1C 00 00 00" # Raw data data type "20 00 00 00" # Dimension "01 00 00 00" # Number of raw data values "02 00 00 00" "00 00 00 00" + # Number of bytes in data, including index hexlify_value('q', sum(sizes) + 4 * len(sizes)) + # Number of properties (0) "00 00 00 00") data = "" offset = 0 for size in sizes: # Index gives end positions of strings: offset += size data += hexlify_value('i', offset) for string in strings: data += string_hexlify(string) test_file.add_segment(toc, metadata, data) tdms_data = test_file.load() data = tdms_data["Group"]["String"].data assert len(data) == len(strings) for expected, read in zip(strings, data): assert expected == read h5_path = tmp_path / 'h5_unicode_strings_test.h5' h5 = tdms_data.as_hdf(h5_path) h5_strings = h5['Group']['String'] assert h5_strings.dtype.kind == 'O' assert h5_strings.shape[0] == len(strings) for expected, read in zip(strings, h5_strings[...]): assert expected == read h5.close()
def test_read_raw_timestamp_data(): """ Test reading timestamp data as a raw TDMS timestamps """ test_file = GeneratedFile() seconds = 3672033330 second_fractions = 1234567890 * 10**10 test_file.add_segment( ("kTocMetaData", "kTocRawData", "kTocNewObjList"), segment_objects_metadata( channel_metadata("/'group'/'channel1'", 0x44, 4), ), hexlify_value("<Q", 0) + hexlify_value("<q", seconds) + hexlify_value("<Q", second_fractions) + hexlify_value("<q", seconds) + hexlify_value("<Q", 0) + hexlify_value("<q", seconds + 1) + hexlify_value("<Q", second_fractions) + hexlify_value("<q", seconds + 1)) expected_seconds = np.array([seconds, seconds, seconds + 1, seconds + 1], np.dtype('int64')) expected_second_fractions = np.array( [0, second_fractions, 0, second_fractions], np.dtype('uint64')) with test_file.get_tempfile() as temp_file: tdms_data = TdmsFile.read(temp_file.file, raw_timestamps=True) data = tdms_data['group']['channel1'][:] assert isinstance(data, TimestampArray) np.testing.assert_equal(data.seconds, expected_seconds) np.testing.assert_equal(data.second_fractions, expected_second_fractions)
def test_floating_point_data_types(tmp_path): """ Test conversion of f32 and f64 types to HDF """ test_file = GeneratedFile() test_file.add_segment( ("kTocMetaData", "kTocRawData", "kTocNewObjList"), segment_objects_metadata( channel_metadata("/'group'/'f32'", 9, 4), channel_metadata("/'group'/'f64'", 10, 4), ), hexlify_value('<f', 1) + hexlify_value('<f', 2) + hexlify_value('<f', 3) + hexlify_value('<f', 4) + hexlify_value('<d', 1) + hexlify_value('<d', 2) + hexlify_value('<d', 3) + hexlify_value('<d', 4) ) tdms_data = test_file.load() h5_path = tmp_path / 'h5_data_test.h5' h5 = tdms_data.as_hdf(h5_path) for chan, expected_dtype in [ ('f32', np.dtype('float32')), ('f64', np.dtype('float64'))]: h5_channel = h5['group'][chan] assert h5_channel.dtype == expected_dtype np.testing.assert_almost_equal(h5_channel[...], [1.0, 2.0, 3.0, 4.0]) h5.close()
def test_reading_subset_of_data(offset, length): channel_data = np.arange(0, 100, 1, dtype=np.int32) # Split data into different sized segments segment_data = [ channel_data[0:10], channel_data[10:20], channel_data[20:60], channel_data[60:80], channel_data[80:90], channel_data[90:100], ] hex_segment_data = [ "".join(hexlify_value('<i', x) for x in data) for data in segment_data ] test_file = GeneratedFile() test_file.add_segment( ("kTocMetaData", "kTocRawData", "kTocNewObjList"), segment_objects_metadata(channel_metadata("/'group'/'channel1'", 3, 5), ), hex_segment_data[0]) for hex_data in hex_segment_data[1:]: test_file.add_segment(("kTocRawData", ), "", hex_data) with test_file.get_tempfile() as temp_file: with TdmsFile.open(temp_file.file) as tdms_file: channel_subset = tdms_file['group']['channel1'].read_data( offset, length) expected_data = channel_data[offset:offset + length] assert len(channel_subset) == len(expected_data) np.testing.assert_equal(channel_subset, expected_data)
def daqmx_scaler_metadata(scale_id, type_id, byte_offset, raw_buffer_index=0, digital_line_scaler=False): return ( # DAQmx data type (type ids don't match TDMS types) hexlify_value("<I", type_id) + # Raw buffer index hexlify_value("<I", raw_buffer_index) + # Raw byte offset hexlify_value("<I", byte_offset) + # Sample format bitmap (don't know what this is for...) ("00" if digital_line_scaler else "00 00 00 00") + # Scale ID hexlify_value("<I", scale_id))
def test_read_raw_timestamp_properties(): """ Test reading timestamp properties as a raw TDMS timestamp """ test_file = GeneratedFile() second_fractions = 1234567890 * 10**10 properties = { "wf_start_time": (0x44, hexlify_value("<Q", second_fractions) + hexlify_value("<q", 3524551547)) } test_file.add_segment(("kTocMetaData", "kTocRawData", "kTocNewObjList"), segment_objects_metadata( channel_metadata("/'group'/'channel1'", 3, 2, properties), ), "01 00 00 00" "02 00 00 00") with test_file.get_tempfile() as temp_file: tdms_data = TdmsFile.read(temp_file.file, raw_timestamps=True) start_time = tdms_data['group']['channel1'].properties['wf_start_time'] assert start_time.seconds == 3524551547 assert start_time.second_fractions == second_fractions
def timestamp_data_chunk(times): epoch = np.datetime64('1904-01-01T00:00:00') def total_seconds(td): return int(td / np.timedelta64(1, 's')) def microseconds(dt): diff = dt - epoch secs = total_seconds(diff) remainder = diff - np.timedelta64(secs, 's') return int(remainder / np.timedelta64(1, 'us')) seconds = [total_seconds(t - epoch) for t in times] fractions = [int(float(microseconds(t)) * 2**58 / 5**6) for t in times] data = "" for f, s in zip(fractions, seconds): data += hexlify_value("<Q", f) data += hexlify_value("<q", s) return data
def test_timestamp_property(tmp_path): """ Test a timestamp property is converted to an attribute in an HDF file HDF doesn't support timestamps natively, so these are converted to strings """ test_file = GeneratedFile() properties = { "wf_start_time": (0x44, hexlify_value("<Q", 0) + hexlify_value("<q", 3524551547)) } test_file.add_segment( ("kTocMetaData", "kTocRawData", "kTocNewObjList"), segment_objects_metadata( channel_metadata("/'group'/'channel1'", 3, 2, properties), ), "01 00 00 00" "02 00 00 00" ) tdms_data = test_file.load() h5_path = tmp_path / 'h5_properties_test.h5' h5 = tdms_data.as_hdf(h5_path) assert h5['group']['channel1'].attrs['wf_start_time'] == b'2015-09-08T10:05:47.000000Z'
def scaled_data(): properties = { "NI_Number_Of_Scales": (3, "01 00 00 00"), "NI_Scale[0]_Scale_Type": (0x20, hexlify_value("<I", len("Linear")) + string_hexlify("Linear")), "NI_Scale[0]_Linear_Slope": (10, hexlify_value("<d", 2.0)), "NI_Scale[0]_Linear_Y_Intercept": (10, hexlify_value("<d", 10.0)) } test_file = GeneratedFile() test_file.add_segment( ("kTocMetaData", "kTocRawData", "kTocNewObjList"), segment_objects_metadata( channel_metadata("/'group'/'channel1'", TDS_TYPE_INT32, 2, properties), ), "01 00 00 00" "02 00 00 00" "03 00 00 00" "04 00 00 00") expected_data = { ('group', 'channel1'): np.array([12, 14, 16, 18], dtype=np.float64), } return test_file, expected_data
def combine_metadata(*args): num_objects_hex = hexlify_value("<I", len(args)) return num_objects_hex + "".join(args)
def timed_segment(): """TDMS segment with one group and two channels, each with time properties""" toc = ("kTocMetaData", "kTocRawData", "kTocNewObjList") metadata = ( # Number of objects "03 00 00 00" # Length of the first object path "08 00 00 00" # Object path (/'Group') "2F 27 47 72" "6F 75 70 27" # Raw data index "FF FF FF FF" # Num properties "02 00 00 00" # Name length "04 00 00 00" # Property name (prop) "70 72 6F 70" # Property data type (string) "20 00 00 00" # Length of string value "05 00 00 00" # Value "76 61 6C 75 65" # Length of second property name "03 00 00 00" # Property name (num) "6E 75 6D" # Data type of property "03 00 00 00" # Value "0A 00 00 00" # Length of the second object path "13 00 00 00" # Second object path (/'Group'/'Channel1') "2F 27 47 72" "6F 75 70 27" "2F 27 43 68" "61 6E 6E 65" "6C 31 27" # Length of index information "14 00 00 00" # Raw data data type "03 00 00 00" # Dimension "01 00 00 00" # Number of raw data values "02 00 00 00" "00 00 00 00" # Number of properties "03 00 00 00" # Set time properties for the first channel "0F 00 00 00" + string_hexlify('wf_start_offset') + "0A 00 00 00" + hexlify_value("<d", 2.0) + "0C 00 00 00" + string_hexlify('wf_increment') + "0A 00 00 00" + hexlify_value("<d", 0.1) + "0D 00 00 00" + string_hexlify('wf_start_time') + "44 00 00 00" + hexlify_value("<Q", 0) + hexlify_value("<q", 3524551547) + # Length of the third object path "13 00 00 00" # Third object path (/'Group'/'Channel2') "2F 27 47 72" "6F 75 70 27" "2F 27 43 68" "61 6E 6E 65" "6C 32 27" # Length of index information "14 00 00 00" # Raw data data type "03 00 00 00" # Dimension "01 00 00 00" # Number of data values "02 00 00 00" "00 00 00 00" # Number of properties "03 00 00 00" # Set time properties for the second channel "0F 00 00 00" + string_hexlify('wf_start_offset') + "0A 00 00 00" + hexlify_value("<d", 2.0) + "0C 00 00 00" + string_hexlify('wf_increment') + "0A 00 00 00" + hexlify_value("<d", 0.1) + "0D 00 00 00" + string_hexlify('wf_start_time') + "44 00 00 00" + hexlify_value("<Q", 0) + hexlify_value("<q", 3524551547)) data = ( # Data for segment "01 00 00 00" "02 00 00 00" "03 00 00 00" "04 00 00 00") return toc, metadata, data