def interleaved_timestamp_and_numpy_data():
"""Test reading timestamp data interleaved with a standard numpy data type
"""
times = [
np.datetime64('2012-08-23T00:00:00.123', 'us'),
np.datetime64('2012-08-23T01:02:03.456', 'us'),
]
metadata = (
# Number of objects
"02 00 00 00"
# Length of the object path
"16 00 00 00")
metadata += string_hexlify("/'Group'/'TimeChannel'")
metadata += (
# Length of index information
"14 00 00 00"
# Raw data data type
"44 00 00 00"
# Dimension
"01 00 00 00"
# Number of raw data values
"02 00 00 00"
"00 00 00 00"
# Number of properties (0)
"00 00 00 00")
metadata += ("15 00 00 00")
metadata += string_hexlify("/'Group'/'IntChannel'")
metadata += (
# Length of index information
"14 00 00 00"
# Raw data data type
"03 00 00 00"
# Dimension
"01 00 00 00"
# Number of raw data values
"02 00 00 00"
"00 00 00 00"
# Number of properties (0)
"00 00 00 00")
data = (timestamp_data_chunk([times[0]]) + "01 00 00 00" +
timestamp_data_chunk([times[1]]) + "02 00 00 00")
test_file = GeneratedFile()
toc = ("kTocMetaData", "kTocRawData", "kTocNewObjList",
"kTocInterleavedData")
test_file.add_segment(toc, metadata, data)
expected_data = {
('Group', 'TimeChannel'): np.array([times[0], times[1]]),
('Group', 'IntChannel'): np.array([1, 2], dtype=np.dtype('int32')),
}
return test_file, expected_data
def interleaved_timestamp_data():
"""Test reading interleaved timestamp data
"""
times = [
np.datetime64('2012-08-23T00:00:00.123', 'us'),
np.datetime64('2012-08-23T01:02:03.456', 'us'),
np.datetime64('2012-08-23T12:00:00.0', 'us'),
np.datetime64('2012-08-23T12:02:03.9999', 'us'),
]
metadata = (
# Number of objects
"02 00 00 00"
# Length of the object path
"17 00 00 00")
metadata += string_hexlify("/'Group'/'TimeChannel1'")
metadata += (
# Length of index information
"14 00 00 00"
# Raw data data type
"44 00 00 00"
# Dimension
"01 00 00 00"
# Number of raw data values
"02 00 00 00"
"00 00 00 00"
# Number of properties (0)
"00 00 00 00")
metadata += ("17 00 00 00")
metadata += string_hexlify("/'Group'/'TimeChannel2'")
metadata += (
# Length of index information
"14 00 00 00"
# Raw data data type
"44 00 00 00"
# Dimension
"01 00 00 00"
# Number of raw data values
"02 00 00 00"
"00 00 00 00"
# Number of properties (0)
"00 00 00 00")
test_file = GeneratedFile()
toc = ("kTocMetaData", "kTocRawData", "kTocNewObjList",
"kTocInterleavedData")
test_file.add_segment(toc, metadata, timestamp_data_chunk(times))
expected_data = {
('Group', 'TimeChannel1'): np.array([times[0], times[2]]),
('Group', 'TimeChannel2'): np.array([times[1], times[3]]),
}
return test_file, expected_data
def test_unicode_string_data(tmp_path):
""" Test HDF5 conversion for string datatype with non-ASCII data
"""
strings = ["Hello, \u4E16\u754C", "\U0001F600"]
sizes = [len(s.encode('utf-8')) for s in strings]
test_file = GeneratedFile()
toc = ("kTocMetaData", "kTocRawData", "kTocNewObjList")
metadata = (
# Number of objects
"01 00 00 00"
# Length of the object path
"11 00 00 00")
metadata += string_hexlify("/'Group'/'String'")
metadata += (
# Length of index information
"1C 00 00 00"
# Raw data data type
"20 00 00 00"
# Dimension
"01 00 00 00"
# Number of raw data values
"02 00 00 00"
"00 00 00 00" +
# Number of bytes in data, including index
hexlify_value('q', sum(sizes) + 4 * len(sizes)) +
# Number of properties (0)
"00 00 00 00")
data = ""
offset = 0
for size in sizes:
# Index gives end positions of strings:
offset += size
data += hexlify_value('i', offset)
for string in strings:
data += string_hexlify(string)
test_file.add_segment(toc, metadata, data)
tdms_data = test_file.load()
data = tdms_data["Group"]["String"].data
assert len(data) == len(strings)
for expected, read in zip(strings, data):
assert expected == read
h5_path = tmp_path / 'h5_unicode_strings_test.h5'
h5 = tdms_data.as_hdf(h5_path)
h5_strings = h5['Group']['String']
assert h5_strings.dtype.kind == 'O'
assert h5_strings.shape[0] == len(strings)
for expected, read in zip(strings, h5_strings[...]):
assert expected == read
h5.close()
def test_as_hdf_string(tmp_path):
""" Test HDF5 conversion for string datatype
"""
strings = ["abc123", "?<>~`!@#$%^&*()-=_+,.;'[]:{}|"]
test_file = GeneratedFile()
toc = ("kTocMetaData", "kTocRawData", "kTocNewObjList")
metadata = (
# Number of objects
"01 00 00 00"
# Length of the object path
"11 00 00 00")
metadata += string_hexlify("/'Group'/'String'")
metadata += (
# Length of index information
"1C 00 00 00"
# Raw data data type
"20 00 00 00"
# Dimension
"01 00 00 00"
# Number of raw data values
"02 00 00 00"
"00 00 00 00"
# Number of bytes in data
"2B 00 00 00"
"00 00 00 00"
# Number of properties (0)
"00 00 00 00")
data = (
"06 00 00 00" # index to after first string
"24 00 00 00" # index to after second string
)
for string in strings:
data += string_hexlify(string)
test_file.add_segment(toc, metadata, data)
tdms_data = test_file.load()
data = tdms_data["Group"]["String"].data
assert len(data) == len(strings)
for expected, read in zip(strings, data):
assert expected == read
h5_path = tmp_path / 'h5_strings_test.h5'
h5 = tdms_data.as_hdf(h5_path)
h5_strings = h5['Group']['String']
assert h5_strings.dtype.kind == 'O'
assert h5_strings.shape[0] == len(strings)
for expected, read in zip(strings, h5_strings[...]):
assert expected == read
h5.close()
def daqmx_channel_metadata(
channel_name, num_values,
raw_data_widths, scaler_metadata, properties=None, digital_line_scaler=False):
path = "/'Group'/'" + channel_name + "'"
return (
# Length of the object path
hexlify_value("<I", len(path)) +
# Object path
string_hexlify(path) +
# Raw data index (DAQmx)
("6A 12 00 00" if digital_line_scaler else "69 12 00 00") +
# Data type (DAQmx)
"FF FF FF FF"
# Array dimension
"01 00 00 00" +
# Number of values (chunk size)
hexlify_value("<Q", num_values) +
# Scaler metadata
hexlify_value("<I", len(scaler_metadata)) +
"".join(scaler_metadata) +
# Raw data width vector size
hexlify_value("<I", len(raw_data_widths)) +
# Raw data width values
"".join(hexlify_value("<I", v) for v in raw_data_widths) +
hex_properties(properties))
def test_stream_scaled_data_chunks(benchmark):
""" Benchmark streaming channel data when the data is scaled
"""
properties = {
"NI_Number_Of_Scales": (3, "01 00 00 00"),
"NI_Scale[0]_Scale_Type":
(0x20, hexlify_value("<I", len("Linear")) + string_hexlify("Linear")),
"NI_Scale[0]_Linear_Slope": (10, hexlify_value("<d", 2.0)),
"NI_Scale[0]_Linear_Y_Intercept": (10, hexlify_value("<d", 10.0))
}
test_file = GeneratedFile()
data_array = np.arange(0, 1000, dtype=np.dtype('int32'))
data = data_array.tobytes()
test_file.add_segment(
("kTocMetaData", "kTocRawData", "kTocNewObjList"),
segment_objects_metadata(
channel_metadata("/'group'/'channel1'", TDS_TYPE_INT32, 100,
properties), ),
data,
binary_data=True)
for _ in range(0, 9):
test_file.add_segment(("kTocRawData", ), "", data, binary_data=True)
with TdmsFile.open(test_file.get_bytes_io_file()) as tdms_file:
channel = tdms_file['group']['channel1']
channel_data = benchmark(stream_chunks, channel)
channel_data = np.concatenate(channel_data)
expected_data = np.tile(10.0 + 2.0 * data_array, 10)
np.testing.assert_equal(channel_data, expected_data)
def test_string_data():
"""Test reading a file with string data"""
strings = ["abcdefg", "qwertyuiop"]
test_file = GeneratedFile()
toc = ("kTocMetaData", "kTocRawData", "kTocNewObjList")
metadata = (
# Number of objects
"01 00 00 00"
# Length of the object path
"18 00 00 00")
metadata += string_hexlify("/'Group'/'StringChannel'")
metadata += (
# Length of index information
"1C 00 00 00"
# Raw data data type
"20 00 00 00"
# Dimension
"01 00 00 00"
# Number of raw data values
"02 00 00 00"
"00 00 00 00"
# Number of bytes in data
"19 00 00 00"
"00 00 00 00"
# Number of properties (0)
"00 00 00 00")
data = (
"07 00 00 00" # index to after first string
"11 00 00 00" # index to after second string
)
for string in strings:
data += string_hexlify(string)
test_file.add_segment(toc, metadata, data)
tdms_data = test_file.load()
channel = tdms_data["Group"]["StringChannel"]
assert len(channel.data) == len(strings)
assert channel.data.dtype == channel.dtype
for expected, read in zip(strings, channel.data):
assert expected == read
def scaled_data():
properties = {
"NI_Number_Of_Scales": (3, "01 00 00 00"),
"NI_Scale[0]_Scale_Type":
(0x20, hexlify_value("<I", len("Linear")) + string_hexlify("Linear")),
"NI_Scale[0]_Linear_Slope": (10, hexlify_value("<d", 2.0)),
"NI_Scale[0]_Linear_Y_Intercept": (10, hexlify_value("<d", 10.0))
}
test_file = GeneratedFile()
test_file.add_segment(
("kTocMetaData", "kTocRawData", "kTocNewObjList"),
segment_objects_metadata(
channel_metadata("/'group'/'channel1'", TDS_TYPE_INT32, 2,
properties), ), "01 00 00 00"
"02 00 00 00"
"03 00 00 00"
"04 00 00 00")
expected_data = {
('group', 'channel1'): np.array([12, 14, 16, 18], dtype=np.float64),
}
return test_file, expected_data
def timed_segment():
"""TDMS segment with one group and two channels,
each with time properties"""
toc = ("kTocMetaData", "kTocRawData", "kTocNewObjList")
metadata = (
# Number of objects
"03 00 00 00"
# Length of the first object path
"08 00 00 00"
# Object path (/'Group')
"2F 27 47 72"
"6F 75 70 27"
# Raw data index
"FF FF FF FF"
# Num properties
"02 00 00 00"
# Name length
"04 00 00 00"
# Property name (prop)
"70 72 6F 70"
# Property data type (string)
"20 00 00 00"
# Length of string value
"05 00 00 00"
# Value
"76 61 6C 75 65"
# Length of second property name
"03 00 00 00"
# Property name (num)
"6E 75 6D"
# Data type of property
"03 00 00 00"
# Value
"0A 00 00 00"
# Length of the second object path
"13 00 00 00"
# Second object path (/'Group'/'Channel1')
"2F 27 47 72"
"6F 75 70 27"
"2F 27 43 68"
"61 6E 6E 65"
"6C 31 27"
# Length of index information
"14 00 00 00"
# Raw data data type
"03 00 00 00"
# Dimension
"01 00 00 00"
# Number of raw data values
"02 00 00 00"
"00 00 00 00"
# Number of properties
"03 00 00 00"
# Set time properties for the first channel
"0F 00 00 00" + string_hexlify('wf_start_offset') + "0A 00 00 00" +
hexlify_value("<d", 2.0) + "0C 00 00 00" +
string_hexlify('wf_increment') + "0A 00 00 00" +
hexlify_value("<d", 0.1) + "0D 00 00 00" +
string_hexlify('wf_start_time') + "44 00 00 00" +
hexlify_value("<Q", 0) + hexlify_value("<q", 3524551547) +
# Length of the third object path
"13 00 00 00"
# Third object path (/'Group'/'Channel2')
"2F 27 47 72"
"6F 75 70 27"
"2F 27 43 68"
"61 6E 6E 65"
"6C 32 27"
# Length of index information
"14 00 00 00"
# Raw data data type
"03 00 00 00"
# Dimension
"01 00 00 00"
# Number of data values
"02 00 00 00"
"00 00 00 00"
# Number of properties
"03 00 00 00"
# Set time properties for the second channel
"0F 00 00 00" + string_hexlify('wf_start_offset') + "0A 00 00 00" +
hexlify_value("<d", 2.0) + "0C 00 00 00" +
string_hexlify('wf_increment') + "0A 00 00 00" +
hexlify_value("<d", 0.1) + "0D 00 00 00" +
string_hexlify('wf_start_time') + "44 00 00 00" +
hexlify_value("<Q", 0) + hexlify_value("<q", 3524551547))
data = (
# Data for segment
"01 00 00 00"
"02 00 00 00"
"03 00 00 00"
"04 00 00 00")
return toc, metadata, data