def test_can_append_to_file_using_path(self): input_1 = np.linspace(0.0, 1.0, 10) input_2 = np.linspace(1.0, 2.0, 10) segment_1 = ChannelObject("group", "a", input_1) segment_2 = ChannelObject("group", "a", input_2) tempdir = tempfile.mkdtemp() temppath = "%s/test_file.tdms" % tempdir try: with TdmsWriter(temppath) as tdms_writer: tdms_writer.write_segment([segment_1]) with TdmsWriter(temppath, 'a') as tdms_writer: tdms_writer.write_segment([segment_2]) tdms_file = TdmsFile(temppath) output = tdms_file.object("group", "a").data self.assertEqual(len(output), 20) np.testing.assert_almost_equal(output, np.concatenate([input_1, input_2])) finally: if os.path.exists(temppath): os.remove(temppath) os.rmdir(tempdir)
def SPEtoTDMS_seq(spefilepath,meastype): """convert a sequential SPE file (image or spectral) into a Tdms file. """ if isinstance(spefilepath,bytes): #The labview addin passes a bytes instead of string. spefilepath = spefilepath.decode("utf-8") folder = os.path.splitext(os.path.dirname(spefilepath))[0] base = os.path.splitext(os.path.basename(spefilepath))[0] tdmsfilepath = os.path.join(folder,base + ".tdms") spe_file = sl.load_from_files([spefilepath]) frames = spe_file.data gatedelays = _get_gatedelays(spe_file) root_object = RootObject(properties={}) common_group_object = GroupObject("Common", properties={}) with TdmsWriter(tdmsfilepath) as tdms_writer: channel_object = ChannelObject("Common", "Gate Delays" ,gatedelays, properties={}) tdms_writer.write_segment([root_object,common_group_object,channel_object]) if(meastype == 0): wavelength = spe_file.wavelength with TdmsWriter(tdmsfilepath, mode = 'a') as tdms_writer: channel_object = ChannelObject("Common", "Wavelength" ,wavelength, properties={}) tdms_writer.write_segment([root_object,common_group_object,channel_object]) write_spectra(tdmsfilepath, root_object, frames,wavelength ) if(meastype == 1): write_image(tdmsfilepath, root_object, frames )
def write_array_data(self, filepath, groupName, rawData, tags={}, sampleInfo={ "batchId": None, "batchName": '', "sampleId": None }): nextIndexOfChannel = 0 hasGroup = False root_object = RootObject(properties=sampleInfo) group_object = GroupObject(groupName, properties={'data_form': 'array'}) channel_object = ChannelObject(groupName, 'm0', rawData, properties=tags) if os.path.exists(filepath): tdms_file = TdmsFile.read(filepath) original_groups = tdms_file.groups() original_channels = [ chan for group in original_groups for chan in group.channels() ] try: hasGroup = tdms_file[groupName] is not None except KeyError: hasGroup = False print(f'has group? {hasGroup}') if hasGroup: channels = tdms_file[groupName].channels() print(channels) nextIndexOfChannel = len(channels) channelName = f'm{nextIndexOfChannel}' channel_object = ChannelObject(groupName, channelName, rawData, properties=tags) with TdmsWriter(filepath, mode='a') as tdms_writer: # root_object = RootObject(tdms_file.properties) # channels_to_copy = [chan for chan in original_channels] # channels_to_copy.append(channel_object) # tdms_writer.write_segment([root_object] + original_groups + channels_to_copy) # Write first segment tdms_writer.write_segment( [root_object, group_object, channel_object]) else: with TdmsWriter(filepath) as tdms_writer: # Write first segment tdms_writer.write_segment( [root_object, group_object, channel_object])
def test_can_read_tdms_file_properties_after_writing(self): test_time = datetime.utcnow() if pytz: test_time = test_time.replace(tzinfo=pytz.utc) a_segment = RootObject(properties={ "prop1": "foo", "prop2": 3, }) b_segment = GroupObject("group_name", properties={ "prop3": 1.2345, "prop4": test_time, }) output_file = BytesIO() with TdmsWriter(output_file) as tdms_writer: tdms_writer.write_segment([a_segment, b_segment]) output_file.seek(0) tdms_file = TdmsFile(output_file) a_output = tdms_file.object() b_output = tdms_file.object("group_name") self.assertTrue("prop1" in a_output.properties, msg="prop1 not found") self.assertTrue("prop2" in a_output.properties, msg="prop2 not found") self.assertTrue("prop3" in b_output.properties, msg="prop3 not found") self.assertTrue("prop4" in b_output.properties, msg="prop4 not found") self.assertEqual(a_output.properties["prop1"], "foo") self.assertEqual(a_output.properties["prop2"], 3) self.assertEqual(b_output.properties["prop3"], 1.2345) self.assertEqual(b_output.properties["prop4"], test_time)
def save_signals(sensor_data, data_sps, act_cycle): """ the function saves sensordata into a tdms file. the file is stored into the variable save_path, filename is the actual cycle nr. """ wf_increment = 1.0 / np.mean(data_sps) wf_start_offset = 0.0 try: with TdmsWriter(save_path + "/" + str(act_cycle) + ".tdms") as tdms_writer: for i in range(0, len(sensor_names)): channel = ChannelObject('open', sensor_names[i], sensor_data[i, :], properties={ "wf_increment": wf_increment, "wf_start_offset": wf_start_offset }) tdms_writer.write_segment([channel]) print("cycle saved") except: print("saving not sucessfull") return
def test_can_write_to_file_using_open_file(self): input_1 = np.linspace(0.0, 1.0, 10) segment = ChannelObject("group", "a", input_1) with tempfile.NamedTemporaryFile(delete=True) as output_file: with TdmsWriter(output_file.file) as tdms_writer: tdms_writer.write_segment([segment])
def test_can_write_complex(self): input_complex64_data = np.array([1 + 2j, 3 + 4j], np.complex64) input_complex128_data = np.array([5 + 6j, 7 + 8j], np.complex128) complex64_segment = ChannelObject("group", "complex64_data", input_complex64_data) complex128_segment = ChannelObject("group", "complex128_data", input_complex128_data) output_file = BytesIO() with TdmsWriter(output_file) as tdms_writer: tdms_writer.write_segment([complex64_segment]) tdms_writer.write_segment([complex128_segment]) output_file.seek(0) tdms_file = TdmsFile(output_file) output_data = tdms_file.object("group", "complex64_data").data self.assertEqual(output_data.dtype, np.complex64) self.assertEqual(len(output_data), 2) self.assertEqual(output_data[0], input_complex64_data[0]) self.assertEqual(output_data[1], input_complex64_data[1]) output_data = tdms_file.object("group", "complex128_data").data self.assertEqual(output_data.dtype, np.complex128) self.assertEqual(len(output_data), 2) self.assertEqual(output_data[0], input_complex128_data[0]) self.assertEqual(output_data[1], input_complex128_data[1])
def test_can_write_timestamp_data(self): tzinfo = None if pytz: tzinfo = pytz.utc input_data = [ datetime(2017, 7, 9, 12, 35, 0, 0, tzinfo), datetime(2017, 7, 9, 12, 36, 0, 0, tzinfo), datetime(2017, 7, 9, 12, 37, 0, 0, tzinfo), ] segment = ChannelObject("group", "timedata", input_data) output_file = BytesIO() with TdmsWriter(output_file) as tdms_writer: tdms_writer.write_segment([segment]) output_file.seek(0) tdms_file = TdmsFile(output_file) output_data = tdms_file.object("group", "timedata").data self.assertEqual(len(output_data), 3) self.assertEqual(output_data[0], input_data[0]) self.assertEqual(output_data[1], input_data[1]) self.assertEqual(output_data[2], input_data[2])
def test_can_write_complex(): input_complex64_data = np.array([1 + 2j, 3 + 4j], np.complex64) input_complex128_data = np.array([5 + 6j, 7 + 8j], np.complex128) complex64_segment = ChannelObject("group", "complex64_data", input_complex64_data) complex128_segment = ChannelObject("group", "complex128_data", input_complex128_data) output_file = BytesIO() with TdmsWriter(output_file) as tdms_writer: tdms_writer.write_segment([complex64_segment]) tdms_writer.write_segment([complex128_segment]) output_file.seek(0) tdms_file = TdmsFile(output_file) output_data = tdms_file["group"]["complex64_data"] assert output_data.data_type == types.ComplexSingleFloat assert output_data.dtype == np.complex64 assert len(output_data) == 2 assert output_data[0] == input_complex64_data[0] assert output_data[1] == input_complex64_data[1] output_data = tdms_file["group"]["complex128_data"] assert output_data.data_type == types.ComplexDoubleFloat assert output_data.dtype == np.complex128 assert len(output_data) == 2 assert output_data[0] == input_complex128_data[0] assert output_data[1] == input_complex128_data[1]
def test_can_write_timestamp_data_with_datetimes(self): input_data = [ datetime(2017, 7, 9, 12, 35, 0), datetime(2017, 7, 9, 12, 36, 0), datetime(2017, 7, 9, 12, 37, 0) ] expected_data = np.array([ '2017-07-09T12:35:00', '2017-07-09T12:36:00', '2017-07-09T12:37:00' ], dtype='datetime64') segment = ChannelObject("group", "timedata", input_data) output_file = BytesIO() with TdmsWriter(output_file) as tdms_writer: tdms_writer.write_segment([segment]) output_file.seek(0) tdms_file = TdmsFile(output_file) output_data = tdms_file.object("group", "timedata").data self.assertEqual(len(output_data), 3) self.assertEqual(output_data[0], expected_data[0]) self.assertEqual(output_data[1], expected_data[1]) self.assertEqual(output_data[2], expected_data[2])
def test_can_write_numpy_timestamp_data_with_dates(self): tzinfo = None if pytz: tzinfo = pytz.utc input_data = np.array([ '2017-07-09', '2017-07-09', '2017-07-09'], dtype='datetime64') segment = ChannelObject("group", "timedata", input_data) output_file = BytesIO() with TdmsWriter(output_file) as tdms_writer: tdms_writer.write_segment([segment]) output_file.seek(0) tdms_file = TdmsFile(output_file) output_data = tdms_file.object("group", "timedata").data self.assertEqual(len(output_data), 3) self.assertEqual( output_data[0], datetime(2017, 7, 9, 0, 0, 0, 0, tzinfo)) self.assertEqual( output_data[1], datetime(2017, 7, 9, 0, 0, 0, 0, tzinfo)) self.assertEqual( output_data[2], datetime(2017, 7, 9, 0, 0, 0, 0, tzinfo))
def test_can_read_tdms_file_properties_after_writing(): test_time = np.datetime64('2019-11-19T15:30:00') a_segment = RootObject(properties={ "prop1": "foo", "prop2": 3, }) b_segment = GroupObject("group_name", properties={ "prop3": 1.2345, "prop4": test_time, }) output_file = BytesIO() with TdmsWriter(output_file) as tdms_writer: tdms_writer.write_segment([a_segment, b_segment]) output_file.seek(0) tdms_file = TdmsFile(output_file) file_properties = tdms_file.properties b_output = tdms_file["group_name"] assert "prop1" in file_properties, "prop1 not found" assert "prop2" in file_properties, "prop2 not found" assert "prop3" in b_output.properties, "prop3 not found" assert "prop4" in b_output.properties, "prop4 not found" assert file_properties["prop1"] == "foo" assert file_properties["prop2"] == 3 assert b_output.properties["prop3"] == 1.2345 assert b_output.properties["prop4"] == test_time
def test_can_write_timestamp_data_with_datetimes(): input_data = [ datetime(2017, 7, 9, 12, 35, 0), datetime(2017, 7, 9, 12, 36, 0), datetime(2017, 7, 9, 12, 37, 0) ] expected_data = np.array( ['2017-07-09T12:35:00', '2017-07-09T12:36:00', '2017-07-09T12:37:00'], dtype='datetime64') segment = ChannelObject("group", "timedata", input_data) output_file = BytesIO() with TdmsWriter(output_file) as tdms_writer: tdms_writer.write_segment([segment]) output_file.seek(0) tdms_file = TdmsFile(output_file) output_data = tdms_file["group"]["timedata"].data assert len(output_data) == 3 assert output_data[0] == expected_data[0] assert output_data[1] == expected_data[1] assert output_data[2] == expected_data[2]
def test_can_read_tdms_file_after_writing(): a_input = np.linspace(0.0, 1.0, 100, dtype='float32') b_input = np.linspace(0.0, 100.0, 100, dtype='float64') a_segment = ChannelObject("group", "a", a_input) b_segment = ChannelObject("group", "b", b_input) output_file = BytesIO() with TdmsWriter(output_file) as tdms_writer: tdms_writer.write_segment([a_segment, b_segment]) output_file.seek(0) tdms_file = TdmsFile(output_file) a_output = tdms_file["group"]["a"] b_output = tdms_file["group"]["b"] assert a_output.data_type == types.SingleFloat assert b_output.data_type == types.DoubleFloat assert a_output.dtype == np.dtype('float32') assert b_output.dtype == np.dtype('float64') assert len(a_output) == len(a_input) assert len(b_output) == len(b_input) assert (a_output[:] == a_input).all() assert (b_output[:] == b_input).all()
def cut_log_spectra(fileinpaths, times, fileoutpaths_list, **kwargs): for i, fileinpath in enumerate(fileinpaths): fileoutpaths = fileoutpaths_list[i] tdmsfile = TF(fileinpath) for j, t in enumerate(times): fileoutpath = fileoutpaths[j] direc = os.path.split(fileoutpath)[0] if not os.path.exists(direc): os.makedirs(direc) root_object = RootObject(properties={}) try: with TdmsWriter(fileoutpath, mode='w') as tdmswriter: for group in tdmsfile.groups().remove('Global'): idx1, idx2 = _get_indextime(timedate, t[0], t[1]) for channel in file.group_channels(group)[idx1, idx2]: tdms_writer.write_segment([root_object, channel]) for channel in tdmsfile.group_channels('Global'): if channel.channel == 'Wavelength': channel_object = channel else: channel_object = _cut_channel(channel, time[0], time[1], timedata=None) tdms_writer.write_segment( [root_object, channel_object]) except ValueError as error: print(error) print('removing the file at: \n', fileoutpath) os.remove(fileoutpath)
def write_spectra(tdmsfilepath, root_object, frames, wavelength ): """writes a series of spectra to a tmds file. """ framenum = 0 rawdata_group_object = GroupObject("Raw Data", properties={}) with TdmsWriter(tdmsfilepath, mode = 'a') as tdms_writer: for frame in frames: channel_object = ChannelObject("Raw Data", "Frame" + str(framenum), frame[0][0], properties={}) tdms_writer.write_segment([root_object,rawdata_group_object,channel_object]) framenum = framenum +1
def save_array_as_tdms(file_path, array, group, channels): # save vertical 2d array each column to a different channel # e.g. rgb point cloud shaped (1000, 3) group_obj = GroupObject(group) print(channels) print(group) # print(array.shape) channels_obj = [ChannelObject(group, channel, array[:, i]) for i, channel in enumerate(channels)] with TdmsWriter(file_path) as tdms_writer: tdms_writer.write_segment([group_obj, *channels_obj])
def copy_tdms(self, filePath, targetPath): original_file = TdmsFile(filePath) original_groups = original_file.groups() original_channels = [ chan for group in original_groups for chan in group.channels() ] with TdmsWriter(targetPath) as copied_file: root_object = RootObject(original_file.properties) channels_to_copy = [chan for chan in original_channels] copied_file.write_segment([root_object] + original_groups + channels_to_copy)
def test_can_write_tdms_objects_read_from_file(self): group_segment = GroupObject("group", properties={"prop1": "bar"}) input_data = np.linspace(0.0, 1.0, 10) channel_segment = ChannelObject("group", "a", input_data, properties={ "prop1": "foo", "prop2": 3, }) tempdir = tempfile.mkdtemp() temppath = "%s/test_file.tdms" % tempdir try: with TdmsWriter(temppath) as tdms_writer: tdms_writer.write_segment([group_segment, channel_segment]) tdms_file = TdmsFile(temppath) read_group = tdms_file.object("group") read_channel = tdms_file.object("group", "a") with TdmsWriter(temppath) as tdms_writer: tdms_writer.write_segment([read_group, read_channel]) tdms_file = TdmsFile(temppath) read_group = tdms_file.object("group") read_channel = tdms_file.object("group", "a") self.assertFalse(read_group.has_data) self.assertEqual(read_group.properties["prop1"], "bar") self.assertEqual(len(read_channel.data), 10) np.testing.assert_almost_equal(read_channel.data, input_data) self.assertEqual(read_channel.properties["prop1"], "foo") self.assertEqual(read_channel.properties["prop2"], 3) finally: if os.path.exists(temppath): os.remove(temppath) os.rmdir(tempdir)
def test_can_write_to_file_using_path(self): input_1 = np.linspace(0.0, 1.0, 10) segment = ChannelObject("group", "a", input_1) tempdir = tempfile.mkdtemp() temppath = "%s/test_file.tdms" % tempdir try: with TdmsWriter(temppath) as tdms_writer: tdms_writer.write_segment([segment]) finally: if os.path.exists(temppath): os.remove(temppath) os.rmdir(tempdir)
def write_image(tdmsfilepath, root_object, frames ): """writes a series of images to a tdms file. """ framenum = 0 with TdmsWriter(tdmsfilepath, mode = 'a') as tdms_writer: for frame in frames: rawdata_group_object = GroupObject("Frame" + str(framenum), properties={}) linenum = 0 for line in frame[0]: channel_object = ChannelObject("Frame" + str(framenum), "line" + str(linenum) , line, properties={}) tdms_writer.write_segment([root_object,rawdata_group_object,channel_object]) linenum = linenum +1 framenum = framenum +1
def parse_lasertiming(fileinpaths, **kwargs): """Takes in a series of SPE sequential images and outputs a tdms file of the maximum of each image.""" intensities, timestamps, gatedelays = _lasertiming(fileinpaths) folder = os.path.split(fileinpaths[0])[0] fileoutpath = os.path.join(folder, 'PIMax_Timing_Parsed.tdms') with TdmsWriter(fileoutpath, mode='w') as tdms_writer: root_object = RootObject(properties={}) channel_object = ChannelObject('Gate Delays', 'Gate Delays', gatedelays) tdms_writer.write_segment([root_object, channel_object]) _write_dataframe(tdms_writer, intensities, "MaxIntensities") _write_dataframe(tdms_writer, timestamps, "Timestamps")
def cut_log_spectra(fileinpaths, times, fileoutpaths_list, **kwargs): for i, fileinpath in enumerate(fileinpaths): fileoutpaths = fileoutpaths_list[i] tdmsfile = TF(fileinpath) for j, t in enumerate(times): fileoutpath = fileoutpaths[j] direc = os.path.split(fileoutpath)[0] if not os.path.exists(direc): os.makedirs(direc) root_object = RootObject(properties={}) try: with TdmsWriter(fileoutpath, mode='w') as tdms_writer: timedata = [ dt64(y) for y in tdmsfile.channel_data('Global', 'Time') ] idx1, idx2 = _get_indextime(timedata, t[0], t[1]) if idx1 == idx2: pass else: for group in tdmsfile.groups(): group_object = GroupObject(group, properties={}) if group == "Global": for channel in tdmsfile.group_channels(group): if channel.channel == 'Wavelength': channel_object = ChannelObject( channel.group, channel.channel, channel.data) else: channel_object = ChannelObject( channel.group, channel.channel, channel.data[idx1:idx2]) tdms_writer.write_segment([ root_object, group_object, channel_object ]) else: for channel_object in tdmsfile.group_channels( group)[idx1:idx2]: tdms_writer.write_segment([ root_object, group_object, channel_object ]) except ValueError as error: print(error) print('removing the file at: \n', fileoutpath) os.remove(fileoutpath)
def write_tdms(data, keys, groupname, pathname): """ Write data to TDMS file Args: data: numpy array of data, which will be saved as tdms file keys: list of channel names for each column; len(data) == len(keys) should be true groupname: all channels will be placed under this group within the tdms file structure pathname: path of the tdms file, which will be generated """ with TdmsWriter(pathname) as tdms_writer: for idx, __ in enumerate(data): channel = ChannelObject(groupname, keys[idx], np.array(data[idx])) tdms_writer.write_segment([channel])
def write_chunks_to_file( tdms_writer: nptdms.TdmsWriter, index_ranges: List[Tuple[int, int]], group, channel, segment_size: int, ): """Writes correct data slice per slice to disk. Arguments: tdms_writer: Tdms handle for the new file index_ranges: Chunk Indices that point to valid data slices group: TDMS Group inside the old tdms file channel: TDMS Channel inside group segment_size: Sets size of each segment written to a TDMS file """ clean_data = [] clean_data_nbytes = 0 for (offset, length) in tqdm.tqdm(index_ranges): data = channel.read_data(offset=offset, length=length) clean_data.append(data) clean_data_nbytes += data.nbytes # When segment_size is reached, a new segment is written to file if clean_data_nbytes > segment_size * 1_000_000_000: new_channel = nptdms.ChannelObject(group.name, channel.name, np.concatenate(clean_data)) tdms_writer.write_segment([new_channel]) clean_data = [] clean_data_nbytes = 0 # The remaining chunks are written to file as a last smaller segment if clean_data: new_channel = nptdms.ChannelObject(group.name, channel.name, np.concatenate(clean_data)) tdms_writer.write_segment([new_channel])
def savingThread(que, guiQueue): t = threading.current_thread() name = str(t.getName()) logging.info(f"GUI Thread: {name} started!") with TdmsWriter(DATABASE_PATH + ".tdms") as writer: while not keyboard.is_pressed("q"): with LOCK: if not que.empty(): temp = que.get() saveToDatabase(data=temp, tdms_writer=writer) guiQueue.put(temp) time.sleep(0.2) logging.info(f"GUI Thread: {name} ended!")
def test_can_write_string_data(): input_data = ["hello world", u"\u3053\u3093\u306b\u3061\u306f\u4e16\u754c"] segment = ChannelObject("group", "string_data", input_data) output_file = BytesIO() with TdmsWriter(output_file) as tdms_writer: tdms_writer.write_segment([segment]) output_file.seek(0) tdms_file = TdmsFile(output_file) output_data = tdms_file["group"]["string_data"].data assert len(output_data) == 2 assert output_data[0] == input_data[0] assert output_data[1] == input_data[1]
def test_can_write_floats_from_list(): input_data = [1.0, 2.0, 3.0] segment = ChannelObject("group", "data", input_data) output_file = BytesIO() with TdmsWriter(output_file) as tdms_writer: tdms_writer.write_segment([segment]) output_file.seek(0) tdms_file = TdmsFile(output_file) output_data = tdms_file["group"]["data"].data assert output_data.dtype == np.float64 assert len(output_data) == 3 assert output_data[0] == input_data[0] assert output_data[1] == input_data[1] assert output_data[2] == input_data[2]
def test_can_write_string_data(self): input_data = [ "hello world", u"\u3053\u3093\u306b\u3061\u306f\u4e16\u754c" ] segment = ChannelObject("group", "string_data", input_data) output_file = BytesIO() with TdmsWriter(output_file) as tdms_writer: tdms_writer.write_segment([segment]) output_file.seek(0) tdms_file = TdmsFile(output_file) output_data = tdms_file.object("group", "string_data").data self.assertEqual(len(output_data), 2) self.assertEqual(output_data[0], input_data[0]) self.assertEqual(output_data[1], input_data[1])
def test_can_write_floats_from_list(self): input_data = [1.0, 2.0, 3.0] segment = ChannelObject("group", "data", input_data) output_file = BytesIO() with TdmsWriter(output_file) as tdms_writer: tdms_writer.write_segment([segment]) output_file.seek(0) tdms_file = TdmsFile(output_file) output_data = tdms_file.object("group", "data").data self.assertEqual(output_data.dtype, np.float64) self.assertEqual(len(output_data), 3) self.assertEqual(output_data[0], input_data[0]) self.assertEqual(output_data[1], input_data[1]) self.assertEqual(output_data[2], input_data[2])