def test_can_read_tdms_file_after_writing():
a_input = np.linspace(0.0, 1.0, 100, dtype='float32')
b_input = np.linspace(0.0, 100.0, 100, dtype='float64')
a_segment = ChannelObject("group", "a", a_input)
b_segment = ChannelObject("group", "b", b_input)
output_file = BytesIO()
with TdmsWriter(output_file) as tdms_writer:
tdms_writer.write_segment([a_segment, b_segment])
output_file.seek(0)
tdms_file = TdmsFile(output_file)
a_output = tdms_file["group"]["a"]
b_output = tdms_file["group"]["b"]
assert a_output.data_type == types.SingleFloat
assert b_output.data_type == types.DoubleFloat
assert a_output.dtype == np.dtype('float32')
assert b_output.dtype == np.dtype('float64')
assert len(a_output) == len(a_input)
assert len(b_output) == len(b_input)
assert (a_output[:] == a_input).all()
assert (b_output[:] == b_input).all()
def test_can_write_complex():
input_complex64_data = np.array([1 + 2j, 3 + 4j], np.complex64)
input_complex128_data = np.array([5 + 6j, 7 + 8j], np.complex128)
complex64_segment = ChannelObject("group", "complex64_data",
input_complex64_data)
complex128_segment = ChannelObject("group", "complex128_data",
input_complex128_data)
output_file = BytesIO()
with TdmsWriter(output_file) as tdms_writer:
tdms_writer.write_segment([complex64_segment])
tdms_writer.write_segment([complex128_segment])
output_file.seek(0)
tdms_file = TdmsFile(output_file)
output_data = tdms_file["group"]["complex64_data"]
assert output_data.data_type == types.ComplexSingleFloat
assert output_data.dtype == np.complex64
assert len(output_data) == 2
assert output_data[0] == input_complex64_data[0]
assert output_data[1] == input_complex64_data[1]
output_data = tdms_file["group"]["complex128_data"]
assert output_data.data_type == types.ComplexDoubleFloat
assert output_data.dtype == np.complex128
assert len(output_data) == 2
assert output_data[0] == input_complex128_data[0]
assert output_data[1] == input_complex128_data[1]
def test_can_write_complex(self):
input_complex64_data = np.array([1 + 2j, 3 + 4j], np.complex64)
input_complex128_data = np.array([5 + 6j, 7 + 8j], np.complex128)
complex64_segment = ChannelObject("group", "complex64_data",
input_complex64_data)
complex128_segment = ChannelObject("group", "complex128_data",
input_complex128_data)
output_file = BytesIO()
with TdmsWriter(output_file) as tdms_writer:
tdms_writer.write_segment([complex64_segment])
tdms_writer.write_segment([complex128_segment])
output_file.seek(0)
tdms_file = TdmsFile(output_file)
output_data = tdms_file.object("group", "complex64_data").data
self.assertEqual(output_data.dtype, np.complex64)
self.assertEqual(len(output_data), 2)
self.assertEqual(output_data[0], input_complex64_data[0])
self.assertEqual(output_data[1], input_complex64_data[1])
output_data = tdms_file.object("group", "complex128_data").data
self.assertEqual(output_data.dtype, np.complex128)
self.assertEqual(len(output_data), 2)
self.assertEqual(output_data[0], input_complex128_data[0])
self.assertEqual(output_data[1], input_complex128_data[1])
def test_can_append_to_file_using_path(self):
input_1 = np.linspace(0.0, 1.0, 10)
input_2 = np.linspace(1.0, 2.0, 10)
segment_1 = ChannelObject("group", "a", input_1)
segment_2 = ChannelObject("group", "a", input_2)
tempdir = tempfile.mkdtemp()
temppath = "%s/test_file.tdms" % tempdir
try:
with TdmsWriter(temppath) as tdms_writer:
tdms_writer.write_segment([segment_1])
with TdmsWriter(temppath, 'a') as tdms_writer:
tdms_writer.write_segment([segment_2])
tdms_file = TdmsFile(temppath)
output = tdms_file.object("group", "a").data
self.assertEqual(len(output), 20)
np.testing.assert_almost_equal(output,
np.concatenate([input_1, input_2]))
finally:
if os.path.exists(temppath):
os.remove(temppath)
os.rmdir(tempdir)
def SPEtoTDMS_seq(spefilepath,meastype):
"""convert a sequential SPE file (image or spectral) into a Tdms file. """
if isinstance(spefilepath,bytes): #The labview addin passes a bytes instead of string.
spefilepath = spefilepath.decode("utf-8")
folder = os.path.splitext(os.path.dirname(spefilepath))[0]
base = os.path.splitext(os.path.basename(spefilepath))[0]
tdmsfilepath = os.path.join(folder,base + ".tdms")
spe_file = sl.load_from_files([spefilepath])
frames = spe_file.data
gatedelays = _get_gatedelays(spe_file)
root_object = RootObject(properties={})
common_group_object = GroupObject("Common", properties={})
with TdmsWriter(tdmsfilepath) as tdms_writer:
channel_object = ChannelObject("Common", "Gate Delays" ,gatedelays, properties={})
tdms_writer.write_segment([root_object,common_group_object,channel_object])
if(meastype == 0):
wavelength = spe_file.wavelength
with TdmsWriter(tdmsfilepath, mode = 'a') as tdms_writer:
channel_object = ChannelObject("Common", "Wavelength" ,wavelength, properties={})
tdms_writer.write_segment([root_object,common_group_object,channel_object])
write_spectra(tdmsfilepath, root_object, frames,wavelength )
if(meastype == 1):
write_image(tdmsfilepath, root_object, frames )
def write_array_data(self,
filepath,
groupName,
rawData,
tags={},
sampleInfo={
"batchId": None,
"batchName": '',
"sampleId": None
}):
nextIndexOfChannel = 0
hasGroup = False
root_object = RootObject(properties=sampleInfo)
group_object = GroupObject(groupName,
properties={'data_form': 'array'})
channel_object = ChannelObject(groupName,
'm0',
rawData,
properties=tags)
if os.path.exists(filepath):
tdms_file = TdmsFile.read(filepath)
original_groups = tdms_file.groups()
original_channels = [
chan for group in original_groups for chan in group.channels()
]
try:
hasGroup = tdms_file[groupName] is not None
except KeyError:
hasGroup = False
print(f'has group? {hasGroup}')
if hasGroup:
channels = tdms_file[groupName].channels()
print(channels)
nextIndexOfChannel = len(channels)
channelName = f'm{nextIndexOfChannel}'
channel_object = ChannelObject(groupName,
channelName,
rawData,
properties=tags)
with TdmsWriter(filepath, mode='a') as tdms_writer:
# root_object = RootObject(tdms_file.properties)
# channels_to_copy = [chan for chan in original_channels]
# channels_to_copy.append(channel_object)
# tdms_writer.write_segment([root_object] + original_groups + channels_to_copy)
# Write first segment
tdms_writer.write_segment(
[root_object, group_object, channel_object])
else:
with TdmsWriter(filepath) as tdms_writer:
# Write first segment
tdms_writer.write_segment(
[root_object, group_object, channel_object])
def cut_log_spectra(fileinpaths, times, fileoutpaths_list, **kwargs):
for i, fileinpath in enumerate(fileinpaths):
fileoutpaths = fileoutpaths_list[i]
tdmsfile = TF(fileinpath)
for j, t in enumerate(times):
fileoutpath = fileoutpaths[j]
direc = os.path.split(fileoutpath)[0]
if not os.path.exists(direc):
os.makedirs(direc)
root_object = RootObject(properties={})
try:
with TdmsWriter(fileoutpath, mode='w') as tdms_writer:
timedata = [
dt64(y)
for y in tdmsfile.channel_data('Global', 'Time')
]
idx1, idx2 = _get_indextime(timedata, t[0], t[1])
if idx1 == idx2:
pass
else:
for group in tdmsfile.groups():
group_object = GroupObject(group, properties={})
if group == "Global":
for channel in tdmsfile.group_channels(group):
if channel.channel == 'Wavelength':
channel_object = ChannelObject(
channel.group, channel.channel,
channel.data)
else:
channel_object = ChannelObject(
channel.group, channel.channel,
channel.data[idx1:idx2])
tdms_writer.write_segment([
root_object, group_object,
channel_object
])
else:
for channel_object in tdmsfile.group_channels(
group)[idx1:idx2]:
tdms_writer.write_segment([
root_object, group_object,
channel_object
])
except ValueError as error:
print(error)
print('removing the file at: \n', fileoutpath)
os.remove(fileoutpath)
def _cut_channel(channel, idx1, idx2, waveform):
"""
Cut an individual channel based on input times.
If no time data is passed the channel is assumed to be a waveform and time_track is used to get a numpy array of the times
"""
if (idx1 == idx2): #times are not within file
raise ValueError(
'times not in channel') #.tdms_file.object().properties['name']
props = channel.properties.copy(
) #watch out, dicts must be explicity copied to not alter the time track
# print(type(props))
if (waveform):
start = props['wf_start_time']
offset = datetime.timedelta(milliseconds=props['wf_increment'] * 1000 *
idx1)
props['wf_start_time'] = start + offset
data = channel.data[idx1:idx2]
if channel.data_type == nptdms.types.DoubleFloat:
props = {**props, **_calc_stats(data)}
return ChannelObject(channel.group,
channel.channel,
data,
properties=props)
def test_can_write_timestamp_data(self):
tzinfo = None
if pytz:
tzinfo = pytz.utc
input_data = [
datetime(2017, 7, 9, 12, 35, 0, 0, tzinfo),
datetime(2017, 7, 9, 12, 36, 0, 0, tzinfo),
datetime(2017, 7, 9, 12, 37, 0, 0, tzinfo),
]
segment = ChannelObject("group", "timedata", input_data)
output_file = BytesIO()
with TdmsWriter(output_file) as tdms_writer:
tdms_writer.write_segment([segment])
output_file.seek(0)
tdms_file = TdmsFile(output_file)
output_data = tdms_file.object("group", "timedata").data
self.assertEqual(len(output_data), 3)
self.assertEqual(output_data[0], input_data[0])
self.assertEqual(output_data[1], input_data[1])
self.assertEqual(output_data[2], input_data[2])
def _cut_channel(channel, time1, time2, timedata=None):
"""
Cut an individual channel based on input times.
If no time data is passed the channel is assumed to be a waveform and time_track is used to get a numpy array of the times
"""
waveform = False
if (timedata == None
): #if no timedata is passed, assume channel is a waveform
timedata = channel.time_track(absolute_time=True)
waveform = True
idx1, idx2 = _get_indextime(timedata, time1, time2)
if (idx1 == idx2): #times are not within file
raise ValueError(
'times not in channel') #.tdms_file.object().properties['name']
props = channel.properties.copy(
) #watch out, dicts must be explicity copied to not alter the time track
# print(type(props))
if (waveform):
start = props['wf_start_time']
offset = datetime.timedelta(milliseconds=props['wf_increment'] * 1000 *
idx1)
props['wf_start_time'] = start + offset
data = channel.data[idx1:idx2]
props = {**props, **_calc_stats(data)}
return ChannelObject(channel.group,
channel.channel,
data,
properties=props)
def test_can_write_to_file_using_open_file(self):
input_1 = np.linspace(0.0, 1.0, 10)
segment = ChannelObject("group", "a", input_1)
with tempfile.NamedTemporaryFile(delete=True) as output_file:
with TdmsWriter(output_file.file) as tdms_writer:
tdms_writer.write_segment([segment])
def test_can_write_timestamp_data_with_datetimes():
input_data = [
datetime(2017, 7, 9, 12, 35, 0),
datetime(2017, 7, 9, 12, 36, 0),
datetime(2017, 7, 9, 12, 37, 0)
]
expected_data = np.array(
['2017-07-09T12:35:00', '2017-07-09T12:36:00', '2017-07-09T12:37:00'],
dtype='datetime64')
segment = ChannelObject("group", "timedata", input_data)
output_file = BytesIO()
with TdmsWriter(output_file) as tdms_writer:
tdms_writer.write_segment([segment])
output_file.seek(0)
tdms_file = TdmsFile(output_file)
output_data = tdms_file["group"]["timedata"].data
assert len(output_data) == 3
assert output_data[0] == expected_data[0]
assert output_data[1] == expected_data[1]
assert output_data[2] == expected_data[2]
def save_signals(sensor_data, data_sps, act_cycle):
"""
the function saves sensordata into a tdms file.
the file is stored into the variable save_path, filename is the actual cycle nr.
"""
wf_increment = 1.0 / np.mean(data_sps)
wf_start_offset = 0.0
try:
with TdmsWriter(save_path + "/" + str(act_cycle) +
".tdms") as tdms_writer:
for i in range(0, len(sensor_names)):
channel = ChannelObject('open',
sensor_names[i],
sensor_data[i, :],
properties={
"wf_increment": wf_increment,
"wf_start_offset": wf_start_offset
})
tdms_writer.write_segment([channel])
print("cycle saved")
except:
print("saving not sucessfull")
return
def test_can_write_numpy_timestamp_data_with_dates(self):
tzinfo = None
if pytz:
tzinfo = pytz.utc
input_data = np.array([
'2017-07-09',
'2017-07-09',
'2017-07-09'], dtype='datetime64')
segment = ChannelObject("group", "timedata", input_data)
output_file = BytesIO()
with TdmsWriter(output_file) as tdms_writer:
tdms_writer.write_segment([segment])
output_file.seek(0)
tdms_file = TdmsFile(output_file)
output_data = tdms_file.object("group", "timedata").data
self.assertEqual(len(output_data), 3)
self.assertEqual(
output_data[0], datetime(2017, 7, 9, 0, 0, 0, 0, tzinfo))
self.assertEqual(
output_data[1], datetime(2017, 7, 9, 0, 0, 0, 0, tzinfo))
self.assertEqual(
output_data[2], datetime(2017, 7, 9, 0, 0, 0, 0, tzinfo))
def test_can_write_timestamp_data_with_datetimes(self):
input_data = [
datetime(2017, 7, 9, 12, 35, 0),
datetime(2017, 7, 9, 12, 36, 0),
datetime(2017, 7, 9, 12, 37, 0)
]
expected_data = np.array([
'2017-07-09T12:35:00', '2017-07-09T12:36:00', '2017-07-09T12:37:00'
],
dtype='datetime64')
segment = ChannelObject("group", "timedata", input_data)
output_file = BytesIO()
with TdmsWriter(output_file) as tdms_writer:
tdms_writer.write_segment([segment])
output_file.seek(0)
tdms_file = TdmsFile(output_file)
output_data = tdms_file.object("group", "timedata").data
self.assertEqual(len(output_data), 3)
self.assertEqual(output_data[0], expected_data[0])
self.assertEqual(output_data[1], expected_data[1])
self.assertEqual(output_data[2], expected_data[2])
def _write_dataframe(tdms_writer, dataframe, name):
"""Write a dataframe to a tdms group."""
root_object = RootObject(properties={})
i = 0
for column in dataframe.iteritems():
column = column[1].as_matrix()
channel_object = ChannelObject(name, name + "_" + str(i), column)
tdms_writer.write_segment([root_object, channel_object])
i = i + 1
def test_can_write_multiple_segments(self):
input_1 = np.linspace(0.0, 1.0, 10)
input_2 = np.linspace(2.0, 3.0, 10)
segment_1 = ChannelObject("group", "a", input_1)
segment_2 = ChannelObject("group", "a", input_2)
output_file = BytesIO()
with TdmsWriter(output_file) as tdms_writer:
tdms_writer.write_segment([segment_1])
tdms_writer.write_segment([segment_2])
output_file.seek(0)
tdms_file = TdmsFile(output_file)
output_data = tdms_file.object("group", "a").data
expected_data = np.append(input_1, input_2)
self.assertEqual(len(output_data), len(expected_data))
self.assertTrue((output_data == expected_data).all())
def write_spectra(tdmsfilepath, root_object, frames, wavelength ):
"""writes a series of spectra to a tmds file. """
framenum = 0
rawdata_group_object = GroupObject("Raw Data", properties={})
with TdmsWriter(tdmsfilepath, mode = 'a') as tdms_writer:
for frame in frames:
channel_object = ChannelObject("Raw Data", "Frame" + str(framenum), frame[0][0], properties={})
tdms_writer.write_segment([root_object,rawdata_group_object,channel_object])
framenum = framenum +1
def test_can_write_multiple_segments():
input_1 = np.linspace(0.0, 1.0, 10)
input_2 = np.linspace(2.0, 3.0, 10)
segment_1 = ChannelObject("group", "a", input_1)
segment_2 = ChannelObject("group", "a", input_2)
output_file = BytesIO()
with TdmsWriter(output_file) as tdms_writer:
tdms_writer.write_segment([segment_1])
tdms_writer.write_segment([segment_2])
output_file.seek(0)
tdms_file = TdmsFile(output_file)
output_data = tdms_file["group"]["a"].data
expected_data = np.append(input_1, input_2)
assert len(output_data) == len(expected_data)
assert (output_data == expected_data).all()
def test_can_read_tdms_file_after_writing():
a_input = np.linspace(0.0, 1.0, 100)
b_input = np.linspace(0.0, 100.0, 100)
a_segment = ChannelObject("group", "a", a_input)
b_segment = ChannelObject("group", "b", b_input)
output_file = BytesIO()
with TdmsWriter(output_file) as tdms_writer:
tdms_writer.write_segment([a_segment, b_segment])
output_file.seek(0)
tdms_file = TdmsFile(output_file)
a_output = tdms_file["group"]["a"].data
b_output = tdms_file["group"]["b"].data
assert len(a_output) == len(a_input)
assert len(b_output) == len(b_input)
assert (a_output == a_input).all()
assert (b_output == b_input).all()
def test_can_read_tdms_file_after_writing(self):
a_input = np.linspace(0.0, 1.0, 100)
b_input = np.linspace(0.0, 100.0, 100)
a_segment = ChannelObject("group", "a", a_input)
b_segment = ChannelObject("group", "b", b_input)
output_file = BytesIO()
with TdmsWriter(output_file) as tdms_writer:
tdms_writer.write_segment([a_segment, b_segment])
output_file.seek(0)
tdms_file = TdmsFile(output_file)
a_output = tdms_file.object("group", "a").data
b_output = tdms_file.object("group", "b").data
self.assertEqual(len(a_output), len(a_input))
self.assertEqual(len(b_output), len(b_input))
self.assertTrue((a_output == a_input).all())
self.assertTrue((b_output == b_input).all())
def save_array_as_tdms(file_path, array, group, channels):
# save vertical 2d array each column to a different channel
# e.g. rgb point cloud shaped (1000, 3)
group_obj = GroupObject(group)
print(channels)
print(group)
# print(array.shape)
channels_obj = [ChannelObject(group, channel, array[:, i])
for i, channel in enumerate(channels)]
with TdmsWriter(file_path) as tdms_writer:
tdms_writer.write_segment([group_obj, *channels_obj])
def write_image(tdmsfilepath, root_object, frames ):
"""writes a series of images to a tdms file. """
framenum = 0
with TdmsWriter(tdmsfilepath, mode = 'a') as tdms_writer:
for frame in frames:
rawdata_group_object = GroupObject("Frame" + str(framenum), properties={})
linenum = 0
for line in frame[0]:
channel_object = ChannelObject("Frame" + str(framenum), "line" + str(linenum) , line, properties={})
tdms_writer.write_segment([root_object,rawdata_group_object,channel_object])
linenum = linenum +1
framenum = framenum +1
def saveToDatabase(data, tdms_writer=None):
currentTime = datetime.now().strftime("%H:%M:%S")
logging.info(f"Saving data at time: {currentTime}")
if SAVE_TO_JSON:
outputdata = {"time": currentTime, "data": data.tolist()}
with open(DATABASE_PATH + ".js", "a") as f:
f.write(json.dumps(outputdata) + ",\n")
if tdms_writer is not None:
channel = ChannelObject("Undefined", "Channel1", data)
tdms_writer.write_segment([channel])
def test_can_write_to_file_using_path(self):
input_1 = np.linspace(0.0, 1.0, 10)
segment = ChannelObject("group", "a", input_1)
tempdir = tempfile.mkdtemp()
temppath = "%s/test_file.tdms" % tempdir
try:
with TdmsWriter(temppath) as tdms_writer:
tdms_writer.write_segment([segment])
finally:
if os.path.exists(temppath):
os.remove(temppath)
os.rmdir(tempdir)
def parse_lasertiming(fileinpaths, **kwargs):
"""Takes in a series of SPE sequential images and outputs a tdms file of the maximum of each image."""
intensities, timestamps, gatedelays = _lasertiming(fileinpaths)
folder = os.path.split(fileinpaths[0])[0]
fileoutpath = os.path.join(folder, 'PIMax_Timing_Parsed.tdms')
with TdmsWriter(fileoutpath, mode='w') as tdms_writer:
root_object = RootObject(properties={})
channel_object = ChannelObject('Gate Delays', 'Gate Delays',
gatedelays)
tdms_writer.write_segment([root_object, channel_object])
_write_dataframe(tdms_writer, intensities, "MaxIntensities")
_write_dataframe(tdms_writer, timestamps, "Timestamps")
def write_waveform_data(self,
filepath,
groupName,
time_data=[],
y_data=[],
tags={},
sampleInfo={
"batchId": None,
"batchName": '',
"sampleId": None
}):
nextIndexOfChannel = 0
if os.path.exists(filepath):
self.open(filepath)
nextIndexOfChannel = self.how_many_channels_in_group_name(
groupName) + 1
self.close()
time_channelName = f'm{nextIndexOfChannel}_t'
value_channelName = f'm{nextIndexOfChannel}_y'
root_object = RootObject(properties=sampleInfo)
group_object = GroupObject(groupName,
properties={'data_form': 'waveform'})
with TdmsWriter(filepath) as tdms_writer:
# Write time data segment
time_channel_object = ChannelObject(groupName,
time_channelName,
time_data,
properties={})
tdms_writer.write_segment(
[root_object, group_object, time_channel_object])
# Write value data segment
value_channel_object = ChannelObject(groupName,
value_channelName,
y_data,
properties=tags)
tdms_writer.write_segment(
[root_object, group_object, value_channel_object])
def write_tdms(data, keys, groupname, pathname):
"""
Write data to TDMS file
Args:
data: numpy array of data, which will be saved as tdms file
keys: list of channel names for each column; len(data) == len(keys) should be true
groupname: all channels will be placed under this group within the tdms file structure
pathname: path of the tdms file, which will be generated
"""
with TdmsWriter(pathname) as tdms_writer:
for idx, __ in enumerate(data):
channel = ChannelObject(groupname, keys[idx], np.array(data[idx]))
tdms_writer.write_segment([channel])
def test_can_write_string_data():
input_data = ["hello world", u"\u3053\u3093\u306b\u3061\u306f\u4e16\u754c"]
segment = ChannelObject("group", "string_data", input_data)
output_file = BytesIO()
with TdmsWriter(output_file) as tdms_writer:
tdms_writer.write_segment([segment])
output_file.seek(0)
tdms_file = TdmsFile(output_file)
output_data = tdms_file["group"]["string_data"].data
assert len(output_data) == 2
assert output_data[0] == input_data[0]
assert output_data[1] == input_data[1]
def _cut_datetime_channel(channel, time1, time2):
"""
cut an array of datetimes. This is a temporary function
used for powermeter for now, which logs times from labview (Time_LV), which is an array of datetime objects. In the future this, cutting of np64 time arrays, waveforms, and numeric channels should all be combined.
"""
timedata = channel.data
idx1, idx2 = _get_indextime(timedata, time1, time2)
if (idx1 == idx2): #times are not within file
raise ValueError(
'times not in channel') #.tdms_file.object().properties['name']
props = channel.properties.copy()
return ChannelObject(channel.group,
channel.channel,
timedata[idx1:idx2],
properties=props)