def daqmx_channel_metadata(
channel_name, num_values,
raw_data_widths, scaler_metadata, properties=None, digital_line_scaler=False):
path = "/'Group'/'" + channel_name + "'"
return (
# Length of the object path
hexlify_value("<I", len(path)) +
# Object path
string_hexlify(path) +
# Raw data index (DAQmx)
("6A 12 00 00" if digital_line_scaler else "69 12 00 00") +
# Data type (DAQmx)
"FF FF FF FF"
# Array dimension
"01 00 00 00" +
# Number of values (chunk size)
hexlify_value("<Q", num_values) +
# Scaler metadata
hexlify_value("<I", len(scaler_metadata)) +
"".join(scaler_metadata) +
# Raw data width vector size
hexlify_value("<I", len(raw_data_widths)) +
# Raw data width values
"".join(hexlify_value("<I", v) for v in raw_data_widths) +
hex_properties(properties))
def float_data():
""" Test reading a file with float valued data
"""
single_arr = np.array([0.123, 0.234, 0.345, 0.456], dtype=np.float32)
double_arr = np.array([0.987, 0.876, 0.765, 0.654], dtype=np.double)
data = ""
for num in single_arr[0:2]:
data += hexlify_value("<f", num)
for num in double_arr[0:2]:
data += hexlify_value("<d", num)
for num in single_arr[2:4]:
data += hexlify_value("<f", num)
for num in double_arr[2:4]:
data += hexlify_value("<d", num)
test_file = GeneratedFile()
test_file.add_segment(
("kTocMetaData", "kTocRawData", "kTocNewObjList"),
segment_objects_metadata(
channel_metadata("/'group'/'single_channel'", TDS_TYPE_FLOAT32, 2),
channel_metadata("/'group'/'double_channel'", TDS_TYPE_FLOAT64, 2),
), data)
expected_data = {
('group', 'single_channel'): single_arr,
('group', 'double_channel'): double_arr,
}
return test_file, expected_data
def test_stream_scaled_data_chunks(benchmark):
""" Benchmark streaming channel data when the data is scaled
"""
properties = {
"NI_Number_Of_Scales": (3, "01 00 00 00"),
"NI_Scale[0]_Scale_Type":
(0x20, hexlify_value("<I", len("Linear")) + string_hexlify("Linear")),
"NI_Scale[0]_Linear_Slope": (10, hexlify_value("<d", 2.0)),
"NI_Scale[0]_Linear_Y_Intercept": (10, hexlify_value("<d", 10.0))
}
test_file = GeneratedFile()
data_array = np.arange(0, 1000, dtype=np.dtype('int32'))
data = data_array.tobytes()
test_file.add_segment(
("kTocMetaData", "kTocRawData", "kTocNewObjList"),
segment_objects_metadata(
channel_metadata("/'group'/'channel1'", TDS_TYPE_INT32, 100,
properties), ),
data,
binary_data=True)
for _ in range(0, 9):
test_file.add_segment(("kTocRawData", ), "", data, binary_data=True)
with TdmsFile.open(test_file.get_bytes_io_file()) as tdms_file:
channel = tdms_file['group']['channel1']
channel_data = benchmark(stream_chunks, channel)
channel_data = np.concatenate(channel_data)
expected_data = np.tile(10.0 + 2.0 * data_array, 10)
np.testing.assert_equal(channel_data, expected_data)
def float_data_with_unit():
""" Test reading a file with float valued data with units
These are the same as normal floating point data but have a 'unit_string' property
"""
single_arr = np.array([0.123, 0.234, 0.345, 0.456], dtype=np.float32)
double_arr = np.array([0.987, 0.876, 0.765, 0.654], dtype=np.double)
data = ""
for num in single_arr[0:2]:
data += hexlify_value("<f", num)
for num in double_arr[0:2]:
data += hexlify_value("<d", num)
for num in single_arr[2:4]:
data += hexlify_value("<f", num)
for num in double_arr[2:4]:
data += hexlify_value("<d", num)
test_file = GeneratedFile()
test_file.add_segment(("kTocMetaData", "kTocRawData", "kTocNewObjList"),
segment_objects_metadata(
channel_metadata("/'group'/'single_channel'",
TDS_TYPE_FLOAT32_WITH_UNIT, 2),
channel_metadata("/'group'/'double_channel'",
TDS_TYPE_FLOAT64_WITH_UNIT, 2),
), data)
expected_data = {
('group', 'single_channel'): single_arr,
('group', 'double_channel'): double_arr,
}
return test_file, expected_data
def complex_data():
""" Test reading a file with complex valued data
"""
complex_single_arr = np.array([1 + 2j, 3 + 4j], dtype=np.complex64)
complex_double_arr = np.array([5 + 6j, 7 + 8j], dtype=np.complex128)
data = ""
for num in complex_single_arr:
data += hexlify_value("<f", num.real)
data += hexlify_value("<f", num.imag)
for num in complex_double_arr:
data += hexlify_value("<d", num.real)
data += hexlify_value("<d", num.imag)
test_file = GeneratedFile()
test_file.add_segment(
("kTocMetaData", "kTocRawData", "kTocNewObjList"),
segment_objects_metadata(
channel_metadata("/'group'/'complex_single_channel'",
TDS_TYPE_COMPLEX64, 2),
channel_metadata("/'group'/'complex_double_channel'",
TDS_TYPE_COMPLEX128, 2),
), data)
expected_data = {
('group', 'complex_single_channel'): complex_single_arr,
('group', 'complex_double_channel'): complex_double_arr,
}
return test_file, expected_data
def digital_scaler_metadata(scale_id, type_id, bit_offset, raw_buffer_index=0):
return (
# DAQmx data type (type ids don't match TDMS types)
hexlify_value("<I", type_id) +
# Raw buffer index
hexlify_value("<I", raw_buffer_index) +
# Raw byte offset
hexlify_value("<I", bit_offset) +
# Sample format bitmap (don't know what this is for...)
"00" +
# Scale ID
hexlify_value("<I", scale_id))
def test_unicode_string_data(tmp_path):
""" Test HDF5 conversion for string datatype with non-ASCII data
"""
strings = ["Hello, \u4E16\u754C", "\U0001F600"]
sizes = [len(s.encode('utf-8')) for s in strings]
test_file = GeneratedFile()
toc = ("kTocMetaData", "kTocRawData", "kTocNewObjList")
metadata = (
# Number of objects
"01 00 00 00"
# Length of the object path
"11 00 00 00")
metadata += string_hexlify("/'Group'/'String'")
metadata += (
# Length of index information
"1C 00 00 00"
# Raw data data type
"20 00 00 00"
# Dimension
"01 00 00 00"
# Number of raw data values
"02 00 00 00"
"00 00 00 00" +
# Number of bytes in data, including index
hexlify_value('q', sum(sizes) + 4 * len(sizes)) +
# Number of properties (0)
"00 00 00 00")
data = ""
offset = 0
for size in sizes:
# Index gives end positions of strings:
offset += size
data += hexlify_value('i', offset)
for string in strings:
data += string_hexlify(string)
test_file.add_segment(toc, metadata, data)
tdms_data = test_file.load()
data = tdms_data["Group"]["String"].data
assert len(data) == len(strings)
for expected, read in zip(strings, data):
assert expected == read
h5_path = tmp_path / 'h5_unicode_strings_test.h5'
h5 = tdms_data.as_hdf(h5_path)
h5_strings = h5['Group']['String']
assert h5_strings.dtype.kind == 'O'
assert h5_strings.shape[0] == len(strings)
for expected, read in zip(strings, h5_strings[...]):
assert expected == read
h5.close()
def test_read_raw_timestamp_data():
""" Test reading timestamp data as a raw TDMS timestamps
"""
test_file = GeneratedFile()
seconds = 3672033330
second_fractions = 1234567890 * 10**10
test_file.add_segment(
("kTocMetaData", "kTocRawData", "kTocNewObjList"),
segment_objects_metadata(
channel_metadata("/'group'/'channel1'", 0x44, 4), ),
hexlify_value("<Q", 0) + hexlify_value("<q", seconds) +
hexlify_value("<Q", second_fractions) + hexlify_value("<q", seconds) +
hexlify_value("<Q", 0) + hexlify_value("<q", seconds + 1) +
hexlify_value("<Q", second_fractions) +
hexlify_value("<q", seconds + 1))
expected_seconds = np.array([seconds, seconds, seconds + 1, seconds + 1],
np.dtype('int64'))
expected_second_fractions = np.array(
[0, second_fractions, 0, second_fractions], np.dtype('uint64'))
with test_file.get_tempfile() as temp_file:
tdms_data = TdmsFile.read(temp_file.file, raw_timestamps=True)
data = tdms_data['group']['channel1'][:]
assert isinstance(data, TimestampArray)
np.testing.assert_equal(data.seconds, expected_seconds)
np.testing.assert_equal(data.second_fractions,
expected_second_fractions)
def test_floating_point_data_types(tmp_path):
""" Test conversion of f32 and f64 types to HDF
"""
test_file = GeneratedFile()
test_file.add_segment(
("kTocMetaData", "kTocRawData", "kTocNewObjList"),
segment_objects_metadata(
channel_metadata("/'group'/'f32'", 9, 4),
channel_metadata("/'group'/'f64'", 10, 4),
),
hexlify_value('<f', 1) +
hexlify_value('<f', 2) +
hexlify_value('<f', 3) +
hexlify_value('<f', 4) +
hexlify_value('<d', 1) +
hexlify_value('<d', 2) +
hexlify_value('<d', 3) +
hexlify_value('<d', 4)
)
tdms_data = test_file.load()
h5_path = tmp_path / 'h5_data_test.h5'
h5 = tdms_data.as_hdf(h5_path)
for chan, expected_dtype in [
('f32', np.dtype('float32')),
('f64', np.dtype('float64'))]:
h5_channel = h5['group'][chan]
assert h5_channel.dtype == expected_dtype
np.testing.assert_almost_equal(h5_channel[...], [1.0, 2.0, 3.0, 4.0])
h5.close()
def test_reading_subset_of_data(offset, length):
channel_data = np.arange(0, 100, 1, dtype=np.int32)
# Split data into different sized segments
segment_data = [
channel_data[0:10],
channel_data[10:20],
channel_data[20:60],
channel_data[60:80],
channel_data[80:90],
channel_data[90:100],
]
hex_segment_data = [
"".join(hexlify_value('<i', x) for x in data) for data in segment_data
]
test_file = GeneratedFile()
test_file.add_segment(
("kTocMetaData", "kTocRawData", "kTocNewObjList"),
segment_objects_metadata(channel_metadata("/'group'/'channel1'", 3,
5), ), hex_segment_data[0])
for hex_data in hex_segment_data[1:]:
test_file.add_segment(("kTocRawData", ), "", hex_data)
with test_file.get_tempfile() as temp_file:
with TdmsFile.open(temp_file.file) as tdms_file:
channel_subset = tdms_file['group']['channel1'].read_data(
offset, length)
expected_data = channel_data[offset:offset + length]
assert len(channel_subset) == len(expected_data)
np.testing.assert_equal(channel_subset, expected_data)
def daqmx_scaler_metadata(scale_id,
type_id,
byte_offset,
raw_buffer_index=0,
digital_line_scaler=False):
return (
# DAQmx data type (type ids don't match TDMS types)
hexlify_value("<I", type_id) +
# Raw buffer index
hexlify_value("<I", raw_buffer_index) +
# Raw byte offset
hexlify_value("<I", byte_offset) +
# Sample format bitmap (don't know what this is for...)
("00" if digital_line_scaler else "00 00 00 00") +
# Scale ID
hexlify_value("<I", scale_id))
def test_read_raw_timestamp_properties():
""" Test reading timestamp properties as a raw TDMS timestamp
"""
test_file = GeneratedFile()
second_fractions = 1234567890 * 10**10
properties = {
"wf_start_time": (0x44, hexlify_value("<Q", second_fractions) +
hexlify_value("<q", 3524551547))
}
test_file.add_segment(("kTocMetaData", "kTocRawData", "kTocNewObjList"),
segment_objects_metadata(
channel_metadata("/'group'/'channel1'", 3, 2,
properties), ), "01 00 00 00"
"02 00 00 00")
with test_file.get_tempfile() as temp_file:
tdms_data = TdmsFile.read(temp_file.file, raw_timestamps=True)
start_time = tdms_data['group']['channel1'].properties['wf_start_time']
assert start_time.seconds == 3524551547
assert start_time.second_fractions == second_fractions
def timestamp_data_chunk(times):
epoch = np.datetime64('1904-01-01T00:00:00')
def total_seconds(td):
return int(td / np.timedelta64(1, 's'))
def microseconds(dt):
diff = dt - epoch
secs = total_seconds(diff)
remainder = diff - np.timedelta64(secs, 's')
return int(remainder / np.timedelta64(1, 'us'))
seconds = [total_seconds(t - epoch) for t in times]
fractions = [int(float(microseconds(t)) * 2**58 / 5**6) for t in times]
data = ""
for f, s in zip(fractions, seconds):
data += hexlify_value("<Q", f)
data += hexlify_value("<q", s)
return data
def test_timestamp_property(tmp_path):
""" Test a timestamp property is converted to an attribute in an HDF file
HDF doesn't support timestamps natively, so these are converted to strings
"""
test_file = GeneratedFile()
properties = {
"wf_start_time": (0x44, hexlify_value("<Q", 0) + hexlify_value("<q", 3524551547))
}
test_file.add_segment(
("kTocMetaData", "kTocRawData", "kTocNewObjList"),
segment_objects_metadata(
channel_metadata("/'group'/'channel1'", 3, 2, properties),
),
"01 00 00 00" "02 00 00 00"
)
tdms_data = test_file.load()
h5_path = tmp_path / 'h5_properties_test.h5'
h5 = tdms_data.as_hdf(h5_path)
assert h5['group']['channel1'].attrs['wf_start_time'] == b'2015-09-08T10:05:47.000000Z'
def scaled_data():
properties = {
"NI_Number_Of_Scales": (3, "01 00 00 00"),
"NI_Scale[0]_Scale_Type":
(0x20, hexlify_value("<I", len("Linear")) + string_hexlify("Linear")),
"NI_Scale[0]_Linear_Slope": (10, hexlify_value("<d", 2.0)),
"NI_Scale[0]_Linear_Y_Intercept": (10, hexlify_value("<d", 10.0))
}
test_file = GeneratedFile()
test_file.add_segment(
("kTocMetaData", "kTocRawData", "kTocNewObjList"),
segment_objects_metadata(
channel_metadata("/'group'/'channel1'", TDS_TYPE_INT32, 2,
properties), ), "01 00 00 00"
"02 00 00 00"
"03 00 00 00"
"04 00 00 00")
expected_data = {
('group', 'channel1'): np.array([12, 14, 16, 18], dtype=np.float64),
}
return test_file, expected_data
def combine_metadata(*args):
num_objects_hex = hexlify_value("<I", len(args))
return num_objects_hex + "".join(args)
def timed_segment():
"""TDMS segment with one group and two channels,
each with time properties"""
toc = ("kTocMetaData", "kTocRawData", "kTocNewObjList")
metadata = (
# Number of objects
"03 00 00 00"
# Length of the first object path
"08 00 00 00"
# Object path (/'Group')
"2F 27 47 72"
"6F 75 70 27"
# Raw data index
"FF FF FF FF"
# Num properties
"02 00 00 00"
# Name length
"04 00 00 00"
# Property name (prop)
"70 72 6F 70"
# Property data type (string)
"20 00 00 00"
# Length of string value
"05 00 00 00"
# Value
"76 61 6C 75 65"
# Length of second property name
"03 00 00 00"
# Property name (num)
"6E 75 6D"
# Data type of property
"03 00 00 00"
# Value
"0A 00 00 00"
# Length of the second object path
"13 00 00 00"
# Second object path (/'Group'/'Channel1')
"2F 27 47 72"
"6F 75 70 27"
"2F 27 43 68"
"61 6E 6E 65"
"6C 31 27"
# Length of index information
"14 00 00 00"
# Raw data data type
"03 00 00 00"
# Dimension
"01 00 00 00"
# Number of raw data values
"02 00 00 00"
"00 00 00 00"
# Number of properties
"03 00 00 00"
# Set time properties for the first channel
"0F 00 00 00" + string_hexlify('wf_start_offset') + "0A 00 00 00" +
hexlify_value("<d", 2.0) + "0C 00 00 00" +
string_hexlify('wf_increment') + "0A 00 00 00" +
hexlify_value("<d", 0.1) + "0D 00 00 00" +
string_hexlify('wf_start_time') + "44 00 00 00" +
hexlify_value("<Q", 0) + hexlify_value("<q", 3524551547) +
# Length of the third object path
"13 00 00 00"
# Third object path (/'Group'/'Channel2')
"2F 27 47 72"
"6F 75 70 27"
"2F 27 43 68"
"61 6E 6E 65"
"6C 32 27"
# Length of index information
"14 00 00 00"
# Raw data data type
"03 00 00 00"
# Dimension
"01 00 00 00"
# Number of data values
"02 00 00 00"
"00 00 00 00"
# Number of properties
"03 00 00 00"
# Set time properties for the second channel
"0F 00 00 00" + string_hexlify('wf_start_offset') + "0A 00 00 00" +
hexlify_value("<d", 2.0) + "0C 00 00 00" +
string_hexlify('wf_increment') + "0A 00 00 00" +
hexlify_value("<d", 0.1) + "0D 00 00 00" +
string_hexlify('wf_start_time') + "44 00 00 00" +
hexlify_value("<Q", 0) + hexlify_value("<q", 3524551547))
data = (
# Data for segment
"01 00 00 00"
"02 00 00 00"
"03 00 00 00"
"04 00 00 00")
return toc, metadata, data
