def get_period(files): """ A function to get the stimulation onset and offset times (for the whole stimulation block), in ms, relative to the start of the data file. Args: -files: list of tdms files that contain the stim waveform data Returns: -start, stop: times, in ms """ global stim_chan ##make sure that the files are in the correct order files = order_files(files) start = None stop = None files = iter(files) ##this will churn through the files until a start value is found while start == None: path = next(files) print("Loading {}".format(path)) tdms_file = nptdms.TdmsFile(path) ##here are a couple diffent possibilities for how things could be named try: channel_object = tdms_file.object('Group Name', stim_chan) except KeyError: try: channel_object = tdms_file.object('ephys', stim_chan) except KeyError: channel_object = tdms_file.object('Untitled', stim_chan) start, end = find_stim(channel_object) if start != None: print("Found the start") stop = end print("Moving on to the end detection") ##now we'll keep going until there aren't any stim pulses detected ##that all of the stim data is in the first file while end != None: ##if end is not None, we must have found a continuation of the stim block in the previous file, ##so update 'stop' to reflect this stop = end path = next(files) print("Loading {}".format(path)) tdms_file = nptdms.TdmsFile(path) ##here are a couple diffent possibilities for how things could be named try: channel_object = tdms_file.object('Group Name', stim_chan) except KeyError: try: channel_object = tdms_file.object('ephys', stim_chan) except KeyError: channel_object = tdms_file.object('Untitled', stim_chan) ##here we want to ignore any new values of start, because the start value we found in ##an earlier file should be the true start ignore, end = find_stim(channel_object) return start, stop
def load_ephys(path, resample=False, load_time=True, index=0): """ A function to load data from ephys files, and resample them to a lower rate if necessary Args: -path: full path to the datafile -resample: if False, loads the full dataset. If a number is given, it will resample the data to roughly that sample rate (in Hz) -load_time: if True, loads the duration of the recording in seconds -index: useful for ordering after asynchronous multiprocessing Returns: -data: dictionary with labeled data arrays """ ##load the file tdms_file = nptdms.TdmsFile(path) ##figure out which channels here are ephys channels ephys_chans = get_ephys_chans(tdms_file) data = {} for chan in ephys_chans: try: channel_object = tdms_file.object('Group Name', chan) except KeyError: ##case where the group name is different channel_object = tdms_file.object("ephys", chan) if resample: chan_data = downsample(channel_object, resample) else: chan_data = channel_object.data data[chan] = chan_data if load_time: data['time'] = get_duration_seconds(channel_object) return data, index
def load_signal_from_tdms(file_path, param) -> np.ndarray: tdms_file = nptdms.TdmsFile(file_path) channel = tdms_file.object(param['group_name'], param['channel_name']) data = channel.data time = channel.time_track() file_name = file_path[::-1][:file_path[::-1].find('/')][::-1] t_n = time[-1] N = data.size dt = t_n / N fs = int(round(np.reciprocal(dt), 0)) dt = np.reciprocal(float(fs)) data_param = {'t_n': t_n, 'N': N, 'dt': dt, 'fs': fs, 'f_name': file_name} if param["plot"]["do"]: plt.plot(time, data) plt.xlabel(param["plot"]["x_label"], fontsize=16) plt.ylabel(param["plot"]["y_label"], fontsize=16) plt.title(param["plot"]["title"], fontsize=16) plt.show() return data, time, data_param
def order_stim_arrays(file_dict): """ Takes in a list of stimulation files, and returns ordered lists of stim monitor data arrays, except they are now ordered according to which one happened first (according to the datetime stamp). Args: -files_dict: dictionary of stim amplitude:file path pairs Returns: -timestamps: list of datetime.datetime timestamps -amps: list of stim amplitudes, in mA -stim_data: list of raw stim data arrays """ ##first thing to do is to split out the pieces we need into separate lists stim_amps = list(file_dict.keys()) files = list(file_dict.values()) ##convert the amps to float values in mA stim_amps = [utils.standardize_amps(x) for x in stim_amps] ##now we want to laod the data, and get the timestamps timestamps = [] stim_data = [] for f in files: tdms_file = nptdms.TdmsFile(f) ##now find the address of the stim array data group, channel = utils.search_stim(tdms_file) data = tdms_file.object(group, channel) ##grab the timestamp in datetime.datetime format timestamps.append(get_tstart(data)) ##now extract the sim data stim_data.append(data.data) ##now order things according to the datetime stamp idx = list(np.argsort(timestamps)) return [timestamps[i] for i in idx], [stim_amps[i] for i in idx ], [stim_data[i] for i in idx]
def test_processes_example_file_correct(tmpdir): meta = source.MetaData( chunk_size=6, recurrence_size=2, recurrence_distance=3, consistency_sample_size=10, ) output_filename = pathlib.Path(tmpdir) / "output.tdms" fix.export_correct_data( tdms_path="tests/assets/example_file.tdms", meta=meta, export_path=output_filename, ) tdms_operator = nptdms.TdmsFile(output_filename) df_result = tdms_operator.as_dataframe() assert len(df_result) == 15 assert np.array_equal( df_result.columns.values, np.array([ "/'Untitled'/'D'", "/'Untitled'/'C'", "/'Untitled'/'B'", "/'Untitled'/'A'", ]), ) assert np.array_equal(df_result["/'Untitled'/'A'"].values, np.arange(1, 16))
def load_stim(path, offset=0): """ A function to load a stim channel from a TDMS file, and extract the times when stimulation is "on" Args: -path: full path to the datafile -offset: the number of samples to offset the start,stop sample values by (in case we are concatenating multiple files) Returns: -start: start times of stim wf -stop: end times of stim wf -z: binary stim on/off array for full sample rate -fs: sample rate for this dataset """ global stim_chan ##load the file tdms_file = nptdms.TdmsFile(path) try: channel_object = tdms_file.object('Group Name', stim_chan) except KeyError: try: channel_object = tdms_file.object('ephys', stim_chan) except KeyError: channel_object = tdms_file.object('Untitled', stim_chan) raw = channel_object.data fs = 1 / channel_object.properties['wf_increment'] ##process the stim output (guessing on parameters here) start, stop, z = get_stim_times(raw, 0.1, -0.1, 25) start = start + offset stop = stop + offset return start, stop, z, fs
def load_bp(path, resample=False, load_time=True, index=0): """ A function to load data from blood pressure monitors, and resample them to a lower rate if necessary Args: -path: full path to the datafile -resample: if False, loads the full dataset. If a number is given, it will resample the data to roughly that sample rate (in Hz) -load_time: if True, loads the duration of the recording in seconds Returns: -data: dictionary with labeled data arrays """ global bp_chans ##load the file tdms_file = nptdms.TdmsFile(path) data = {} for chan in bp_chans: channel_object = tdms_file.object('Group Name', chan) if resample: chan_data = downsample(channel_object, resample) else: chan_data = channel_object.data data[chan] = chan_data if load_time: data['time'] = get_duration_seconds(channel_object) return data, index
def tdms2df(filepath): filename = os.path.split(filepath)[1] ext = os.path.splitext(filename)[1] if ext != ".tdms": print("File was not a tdms file") return None tdmsfile = nptdms.TdmsFile(filepath) df = tdmsfile.as_dataframe() #test if a waveform channel channel1 = tdmsfile.group_channels(tdmsfile.groups()[0])[0] waveform = True try: channel1.time_track() except KeyError: waveform = False #find the longest waveform if waveform: longestchannel = None length = 0 for group in tdmsfile.groups(): for channel in tdmsfile.group_channels(group): newlength = len(channel.data) if newlength > length: length = newlength longestchannel = channel timedata = longestchannel.time_track(absolute_time = True) df = df.set_index(timedata) return df
def load_physio(path, resample=False, load_time=True): """ A function to load a TDMS dataset aquired from the SomnoSuite serial pipe. Args: -path: full path to the datafile -resample: if a number, resamples the data to 'resample' Hz -load_time: if True, includes the duration of the recording in seconds Returns: -data: dictionary of data arrays arranged by channel names """ global serial_chans ##load our file tdms_file = nptdms.TdmsFile(path) ##load the data into arrays and put into a dictionary data = {} for chan in list(serial_chans.keys()): channel_object = tdms_file.object('Untitled', chan) if resample: chan_data = downsample(channel_object, resample) else: chan_data = channel_object.data data[serial_chans[chan]] = chan_data if load_time: data['time'] = get_duration_seconds( channel_object ) / 10.0 ##not sure why I need to use this scale factor here, but LabView seems to be saving the wf increment at the wrong value (1 instead of 10?) return data
def read_data(self): tdms_file = nptdms.TdmsFile(self.name) # Reads a tdms file. root_object = tdms_file.object() # tdms file information for name, value in root_object.properties.items(): print("{0}: {1}".format(name, value)) group_name = "Trap" channels = tdms_file.group_channels(group_name) self.ch_num = len(channels) self.ch_name = [str(channels[i].channel) for i in range(len(channels))] self.dt = channels[0].properties[u'wf_increment'] # Sampling time self.fs = int(1.0 / self.dt) # Sampling frequency self.N = len(channels[0].time_track()) # Number of time points print("Channel number: %d" % self.ch_num) print("Channel name: %s" % self.ch_name) print("Sampling rate: %d Hz" % self.fs) print("Data size: %d sec \n" % int(self.N * self.dt)) PZT_nm2V = [5000, 5000, 3000] # PZT Volt to nm conversion factor self.QPD_nm2V = QPD_nm2V # QPD sensitivity (nm/V) at V_sum = 8 V. self.stiffness_pN2nm = stiffness_pN2nm # Stiffness [pN/nm] # Read data self.t = channels[0].time_track() self.QPDy = (channels[1].data - np.median( reject_outliers(channels[1].data))) * self.QPD_nm2V[1] self.PZTy = -(channels[4].data - np.median(channels[4].data)) * PZT_nm2V[1] self.QPDs = (channels[2].data) self.QPDy = self.QPDy / self.QPDs self.T = int(self.fs / f_drive) # Oscillation period in number
def read_tdms_file(file_name, channel_list): """ return as Pandas DataFrame data of channels specified in <channel_list> from <file_name> :param file_name: string, the full name of a .tdms file :param channel_list: list of strings, list of channel names, must be contained in the tdms file :return: pandas DataFrame, each column corresponding to one element of <channel_list> """ tdms_file = nptdms.TdmsFile(file_name) df = tdms_file.as_dataframe(time_index=True, absolute_time=True) # 重命名列,原列名称</'未命名'/'NLHQ-X-03-S05'>,新列名称<NLHQ-X-03-S05> df.rename(columns=lambda name: name.split('/')[-1].strip('\''), inplace=True) return df[channel_list]
def copy_tdms(nwb, in_path, out_path, nrois): num_all_rois = nwb['/processing/Acquired_ROIs/roi_spec'].shape[0] print('Copying {} of {} ROIs from {} to {}'.format( nrois, num_all_rois, in_path, out_path)) in_tdms = nptdms.TdmsFile(in_path) group_name = 'Functional Imaging Data' with nptdms.TdmsWriter(out_path) as out_tdms: root, group = in_tdms.object(), in_tdms.object(group_name) out_tdms.write_segment([root, group]) for ch, channel in {'0': 'Red', '1': 'Green'}.items(): ch_name = 'Channel {} Data'.format(ch) ch_obj = in_tdms.object(group_name, ch_name) shape = (cycles_per_trial(nwb), num_all_rois, -1) ch_data = ch_obj.data.reshape(shape) subset = ch_data[:, :nrois, :].reshape(-1) new_obj = nptdms.ChannelObject(group_name, ch_name, subset, properties={}) out_tdms.write_segment([new_obj])
def _convert_file(tdms_file_path: Path, hdf_dir: Path) -> None: """ converts the tdms file of tdms_file_path to an hdf file by use of the nptdms converter :param tdms_file_path: file path of the tdms file :param hdf_dir: file path of the hdf file (dir instead of file name is given because easier handling with mp) """ t_0 = time() with nptdms.TdmsFile(tdms_file_path) as tdms_file: logger.debug("reading tdms file %40s took: %10.10s sec", tdms_file_path.stem, time() - t_0) hdf_file_path = hdf_dir / tdms_file_path.with_suffix(".hdf").name t_0 = time() tdms_file.as_hdf(hdf_file_path, mode="w", group="/") logger.debug("tdms2hdf + writing %40s took: %10.10s sec", tdms_file_path.stem, time() - t_0)
def _load_raw_stim(path, group_name='Group Name'): """ A function to load a stim channel from a TDMS file, and extract the raw values of stim and the sampling rate (usually 25kHz). Args: - path: full path to the datafile - group_name: Name of the group storing the stim data in the tdms file Returns: -raw: raw stim. -fs: sample rate for this dataset. """ global stim_chan ##load the file tdms_file = nptdms.TdmsFile(path) # channel_object = tdms_file.object('Group Name', stim_chan) channel_object = tdms_file.object(group_name, stim_chan) raw = channel_object.data fs = 1/channel_object.properties['wf_increment'] return raw, fs
def loadAndCalibrateSignals(self, path_to_signals): '''load the signals, calibrate, and correct them. Args: path_to_signals (string): path to the signals.tdms file. ''' tdms_file = nptdms.TdmsFile("signals.tdms") self.xchannel = tdms_file.object('position_data', 'xwaveFSD') self.ychannel = tdms_file.object('position_data', 'ywaveFSD') self.zchannel = tdms_file.object('position_data', 'zwaveFSD') power = tdms_file.object('position_data', 'powerFSD') self.powerArray = xr.DataArray(np.array(power.data), dims='time', coords={'time': power.time_track()}, name='power') self.data_calibrated = xr.Dataset({'x': self._calibrateAndCorrect_Channel(self.xchannel, 0), \ 'y': self._calibrateAndCorrect_Channel(self.ychannel, 1), \ 'z': self._calibrateAndCorrect_Channel(self.zchannel, 2),})
def check_correct_tdms_file(tmp_dir: pathlib.Path) -> None: """Checks if the tdms_file exists and if it contains the correct data. """ # New TdmsFile exists with the correct naming path_to_corrected_tdms = tmp_dir / FILENAME_CORRECTED assert path_to_corrected_tdms.exists() # The corrected TdmsFile contains the desired data tdms_operator = nptdms.TdmsFile(path_to_corrected_tdms) df_result = tdms_operator.as_dataframe() assert len(df_result) == 15 assert np.array_equal( df_result.columns.values, np.array([ "/'Untitled'/'D'", "/'Untitled'/'C'", "/'Untitled'/'B'", "/'Untitled'/'A'", ]), ) assert np.array_equal(df_result["/'Untitled'/'A'"].values, np.arange(1, 16))
def get_event_count_cache(fname): """Get the number of events from a tdms or avi file Parameters ---------- fname: str Path to an experimental data file (tdms or avi) Returns ------- event_count: int The number of events in the data set Notes ----- The values for a file name are cached on disk using the file name and the first 100kB of the file as a key. """ fname = pathlib.Path(fname).resolve() ext = fname.suffix # Generate key with fname.open(mode="rb") as fd: data = fd.read(100 * 1024) fhash = hashlib.md5(data + fname.as_uri()).hexdigest() cfgec = settings.SettingsFileCache(name="shapeout_tdms_event_counts.txt") try: event_count = cfgec.get_int(fhash) except KeyError: if ext == ".avi": with imageio.get_reader(str(fname)) as video: event_count = len(video) elif ext == ".tdms": tdmsfd = nptdms.TdmsFile(str(fname)) event_count = len(tdmsfd.object("Cell Track", "time").data) else: raise ValueError("unsupported file extension: {}".format(ext)) cfgec.set_int(fhash, event_count) return event_count
def load_signal_from_tdms(file_path: str, group_name: str, channel_name: str) -> np.ndarray: ''' Open a TDMS file and extract a single signal from that. Parameters ---------- str : file_path The Path of TDMS file that the user wants to extract the signal. str : group_name Name of group that contains the signal of interest. str : channel_name Name of signal of interest. Returns ------- data : np.ndarray The signal itself. time : np.ndarray The time series of the signal. ''' tdms_file = nptdms.TdmsFile(file_path) channel = tdms_file.object(group_name, channel_name) data = channel.data time = channel.time_track() file_name = file_path[::-1][:file_path[::-1].find('/')][::-1] t_n = time[-1] N = data.size dt = t_n / N fs = int(round(np.reciprocal(dt), 0)) dt = np.reciprocal(float(fs)) data_param = {'t_n': t_n, 'N': N, 'dt': dt, 'fs': fs, 'f_name': file_name} return data, time, data_param
def _parse(self): for file_name in self.file_list: tdms_file = nptdms.TdmsFile(file_name) for channel_name in self.channel_list: channel_object = tdms_file.object(u'未命名', channel_name) # acquire this channel's 'wf_start_time' property # and get its timestamp value for JSON serialize start_time = channel_object.property('wf_start_time') timestamp = time.mktime(start_time.timetuple()) tup = [timestamp] # acquire this channel's other properties others = [ v for k, v in channel_object.properties.items() if k != 'wf_start_time' ] tup.extend(others) # acquire channel data data = channel_object.data.tolist() tup.append(data) yield tup
def read_data(self): tdms_file = nptdms.TdmsFile(self.name) # Reads a tdms file. root_object = tdms_file.object() # tdms file information for name, value in root_object.properties.items(): print("{0}: {1}".format(name, value)) group_name = "Trap" channels = tdms_file.group_channels(group_name) self.dt = channels[0].properties[u'wf_increment'] # Sampling time self.fs = int(1.0 / self.dt) # Sampling frequency self.N = len(channels[0].time_track()) # Number of time points print("Sampling rate: %d Hz" % self.fs) print("Data size: %d sec \n" % int(self.N * self.dt)) PZT_nm2V = [5000, 5000, 3000] # PZT Volt to nm conversion factor # Read data t0 = channels[0].time_track() QPDy = (channels[1].data - np.median(reject_outliers( channels[1].data))) * self.QPD_nm2V[1] PZT0 = -(channels[4].data - np.median(channels[4].data)) * PZT_nm2V[1] QPDs = (channels[2].data) QPD0 = QPDy / QPDs # Fit PZT p0 = [f_drive, A_drive, 0, 0] lb = (f_drive - 0.01, A_drive * 0.9, -np.pi, -100) ub = (f_drive + 0.01, A_drive * 1.1, np.pi, 100) p, cov = curve_fit(triangle, t0, PZT0, p0, bounds=(lb, ub)) PZT_fit0 = triangle(t0, p[0], p[1], p[2], p[3]) # print("PZT fit = ", p) # Subtract low-frequency noise QPD_cut = np.array(QPD0) self.QPD_max = 2 * np.median(np.abs(QPD_cut)) QPD_cut[QPD_cut > self.QPD_max] = self.QPD_max QPD_cut[QPD_cut < -self.QPD_max] = -self.QPD_max N_lp = int(f_sample / f_drive) if N_lp % 2 == 0: N_lp += 1 self.t_lp = running_mean(t0, N_lp) self.QPD_lp = running_mean(QPD_cut, N_lp) t1 = t0 QPD1 = QPD0 - self.QPD_lp PZT1 = PZT0 PZT_fit1 = PZT_fit0 # Fit Sine p0 = [f_drive, 50, 0, 0] lb = (f_drive - 0.01, 0, -np.pi, -10) ub = (f_drive + 0.01, 100, np.pi, 10) p, cov = curve_fit(sine, t1[np.abs(QPD1) < self.QPD_max], QPD1[np.abs(QPD1) < self.QPD_max], p0, bounds=(lb, ub)) # Fit and subtract Trapzoid p0 = [f_drive, 20 * p[1], p[2], p[3], 1e-4] lb = (f_drive - 0.01, 4 * p[1], p[2] - 0.2, p[3] - 1, 1e-8) ub = (f_drive + 0.01, 100 * p[1], p[2] + 0.2, p[3] + 1, 1e-2) p, cov = curve_fit(trapzoid, t1[np.abs(QPD1) < self.QPD_max], QPD1[np.abs(QPD1) < self.QPD_max], p0, bounds=(lb, ub)) QPD_fit1 = trapzoid(t1, p[0], p[1], p[2], p[3], p[4]) # print("QPD_fit = ", p) dQPD1 = QPD1 - QPD_fit1 # Subtract low-frequency noise dQPD_cut = np.array(dQPD1) self.dQPD_max = 2 * np.median(np.abs(dQPD1)) dQPD_cut[dQPD_cut > self.dQPD_max] = self.dQPD_max dQPD_cut[dQPD_cut < -self.dQPD_max] = -self.dQPD_max N_lp = int(f_sample / f_drive / 4) if N_lp % 2 == 0: N_lp += 1 dQPD_lp = running_mean(dQPD_cut, N_lp) dQPD2 = dQPD1 ddQPD = dQPD2 - dQPD_lp self.t = running_mean(t1, N_lp) self.QPD = QPD1 self.QPD_fit = QPD_fit1 self.PZT = PZT1 self.PZT_fit = PZT_fit1 self.Force = ddQPD * self.stiffness_pN2nm[1]
def parseRaw(fname): dd = {} fn = fname.split("/")[-1] fd = nptdms.TdmsFile(fname) try: fd["meta"] except: log.error("No meta object, skipping " + fn) return -1 ch0 = fd["radar"]["ch0"] # .channel_data("radar","ch0") lat = fd["meta"]["lat"] lon = fd["meta"]["lon"] elev = fd["meta"]["elev"] time = fd["meta"]["time"] root = fd.properties startTime = root["start_time"] startTime = datetime.strptime(startTime, "%Y-%m-%dT%H:%M:%S.%f") dd["sig"] = "chirp" # All TDMS files are chirped dd["txCF"] = root["chirp_cf"] dd["txBW"] = root["chirp_bw"] / 100 dd["txlen"] = root["chirp_len"] # dd["chirpAmp"] = root["chirp_amp"] dd["txPRF"] = root["prf"] dd["fs"] = 1.0 / root["dt"] dd["stack"] = root["stacking"] dd["spt"] = root["record_len"] dd["trlen"] = root["dt"] * dd["spt"] # Some files have "pulse" and not "bark" try: bark = root["bark"] except KeyError: bark = root["pulse"] try: bark_len = root["bark_len"] except KeyError: bark_len = root["pulse_len"] try: bark_delay = root["bark_delay"] except KeyError: bark_delay = root["pulse_delay"] spb = int(np.ceil((bark_len + bark_delay) * dd["fs"])) # Extract ch0 dd["rx0"] = tdmsSlice(ch0, dd["spt"], bark, spb) ntrace = dd["rx0"].shape[1] if (ntrace < 10): log.warning("Skipping " + fn + " because less than 10 traces (" + str(ntrace) + ")") return -1 # Correct double length metadata error, or trim data if len(lat) > 2 * dd["rx0"].shape[1]: lat = lat[0:len(lat) - 1:2] lon = lon[0:len(lon) - 1:2] elev = elev[0:len(elev) - 1:2] time = time[0:len(time) - 1:2] else: lat = lat[0:len(lat) - 1] lon = lon[0:len(lon) - 1] elev = elev[0:len(elev) - 1] time = time[0:len(time) - 1] # Fill in every other time value for i in range(len(time) - 1): if time[i] == 0: time[i] = (time[i - 1] + time[i + 1]) / 2 # Fill in last time value diff = time[-2] - time[-3] time[-1] = time[-2] + diff # Time -> seconds since unix epoch time = time.astype(np.float64) epoch = datetime.utcfromtimestamp(0) initT = timedelta(0, time[0]) for i in range(len(time)): delta = timedelta(0, time[i]) - initT time[i] = ((startTime + delta) - epoch).total_seconds() # Crop metadata if rx0 shorter nt = dd["rx0"].shape[1] lat = lat[:nt] lon = lon[:nt] elev = elev[:nt] time = time[:nt] # Get rid of traces with non-unique time time, ai = np.unique(time, return_index=True) lat = lat[ai] lon = lon[ai] elev = elev[ai] dd["rx0"] = dd["rx0"][:, ai] dd["ntrace"] = dd["rx0"].shape[1] dd["rx0"] = dd["rx0"][:, 0:dd["ntrace"]].astype(np.float32) dd["lat"] = np.zeros(dd["ntrace"]).astype("float") dd["lon"] = np.zeros(dd["ntrace"]).astype("float") dd["alt"] = np.zeros(dd["ntrace"]).astype("float") dd["tfull"] = np.zeros(dd["ntrace"]).astype("int64") dd["tfrac"] = np.zeros(dd["ntrace"]).astype("double") for i in range(dd["ntrace"]): # print(time[i], lat[i], lon[i], elev[i]) dd["tfull"][i] = int(time[i]) - 37 # GPS to UTC dd["tfrac"][i] = time[i] - int(time[i]) dd["lat"][i] = lat[i] dd["lon"][i] = lon[i] dd["alt"][i] = elev[i] # Rotate out HW delay date = datetime.utcfromtimestamp(dd["tfull"][i] + dd["tfrac"][i]) if date.year != 2018 and date.month not in (5, 8): log.error("Unknown TDMS data source") return -1 # Handle offset changes over campaign # May is constant, but a split in Aug if date.month == 5: dd["rx0"] = np.roll(dd["rx0"], -14, axis=0) elif date.month == 8 and date.day in (17, 18, 19, 20): dd["rx0"] = np.roll(dd["rx0"], 158, axis=0) elif date.month == 8: dd["rx0"] = np.roll(dd["rx0"], 14, axis=0) else: log.warning("No offset correction found for " + fn) # add extra offset for high pass malaspina track (mystery) if (fn == "20180819-215243.tdms"): log.warning("Adding malaspina high pass offset to " + fn) dd["rx0"] = np.roll(dd["rx0"], -40, axis=0) return dd
def parse_tdms_zip(database, model, input, files, aids, **kwargs): """ uploads a set of .tdms .zip file and parses it for the database :param database: slycat.web.server.database.couchdb.connect() :param model: database.get("model", self._mid) :param input: boolean :param files: files to be parsed :param aids: artiftact id :param kwargs: """ # import error handling from source dac_error = imp.load_source( 'dac_error_handling', os.path.join(os.path.dirname(__file__), 'py/dac_error_handling.py')) dac_error.log_dac_msg("TDMS zip parser started.") # get user parameters MIN_TIME_STEPS = int(aids[0]) MIN_CHANNELS = int(aids[1]) SHOT_TYPE = aids[2] TIME_STEP_TYPE = aids[3] INFER_CHANNEL_UNITS = aids[4] INFER_SECONDS = aids[5] SUFFIX_LIST = aids[6] # keep a parsing error log to help user correct input data # (each array entry is a string) parse_error_log = dac_error.update_parse_log(database, model, [], "Progress", "Notes:") # push progress for wizard polling to database slycat.web.server.put_model_parameter(database, model, "dac-polling-progress", ["Extracting ...", 10.0]) # treat uploaded file as bitstream try: file_like_object = io.BytesIO(files[0]) zip_ref = zipfile.ZipFile(file_like_object) zip_files = zip_ref.namelist() except Exception as e: dac_error.quit_raise_exception( database, model, parse_error_log, "Couldn't read .zip file (too large or corrupted).") # loop through zip files and look for tdms files matching suffix list file_list = [] file_object = [] tdms_ref = [] for zip_file in zip_files: # get file name and extension head, tail = os.path.split(zip_file) ext = tail.split(".")[-1].lower() # is it a tdms file? if ext == 'tdms' or ext == 'tdm': # get suffix suffix = tail.split("_")[-1].split(".")[0] # should we read this file? if suffix in SUFFIX_LIST: try: file_object.append(io.BytesIO(zip_ref.read(zip_file))) tdms_ref.append(nptdms.TdmsFile(file_object[-1])) file_list.append(zip_file) except Exception as e: dac_error.quit_raise_exception( database, model, parse_error_log, "Couldn't read .tdms file.") # log files to be parsed for file_to_parse in file_list: parse_error_log = dac_error.update_parse_log( database, model, parse_error_log, "Progress", 'Found file to parse: "' + file_to_parse + '".') # launch thread to read actual tdms files stop_event = threading.Event() thread = threading.Thread(target=parse_tdms_thread, args=(database, model, tdms_ref, MIN_TIME_STEPS, MIN_CHANNELS, SHOT_TYPE, TIME_STEP_TYPE, INFER_CHANNEL_UNITS, INFER_SECONDS, dac_error, parse_error_log, stop_event)) thread.start()
def parse_tdms(database, model, input, files, aids, **kwargs): """ uploads a set of .tdms files and parses them for the database :param database: slycat.web.server.database.couchdb.connect() :param model: database.get("model", self._mid) :param input: boolean :param files: files to be parsed :param aids: normally artifact ID, but we are using it to pass parameters from the UI :param kwargs: """ # import error handling from source dac_error = imp.load_source( 'dac_error_handling', os.path.join(os.path.dirname(__file__), 'py/dac_error_handling.py')) dac_error.log_dac_msg("TDMS parser started.") # get user parameters MIN_TIME_STEPS = int(aids[0]) MIN_CHANNELS = int(aids[1]) SHOT_TYPE = aids[2] TIME_STEP_TYPE = aids[3] INFER_CHANNEL_UNITS = aids[4] INFER_SECONDS = aids[5] # keep a parsing error log to help user correct input data # (each array entry is a string) parse_error_log = dac_error.update_parse_log(database, model, [], "Progress", "Notes:") # count number of tdms files num_files = len(files) parse_error_log = dac_error.update_parse_log( database, model, parse_error_log, "Progress", "Uploaded " + str(num_files) + " file(s).") # push progress for wizard polling to database slycat.web.server.put_model_parameter(database, model, "dac-polling-progress", ["Extracting ...", 10.0]) # treat each uploaded file as bitstream file_object = [] tdms_ref = [] for i in range(0, num_files): try: file_object.append(io.BytesIO(files[i])) tdms_ref.append(nptdms.TdmsFile(file_object[i])) except Exception as e: dac_error.quit_raise_exception(database, model, parse_error_log, "Couldn't read TDMS file.") # start actual parsing as a thread stop_event = threading.Event() thread = threading.Thread(target=parse_tdms_thread, args=(database, model, tdms_ref, MIN_TIME_STEPS, MIN_CHANNELS, SHOT_TYPE, TIME_STEP_TYPE, INFER_CHANNEL_UNITS, INFER_SECONDS, dac_error, parse_error_log, stop_event)) thread.start()
def _init_data_with_tdms(self, tdms_filename): """Initializes the current RT-DC dataset with a tdms file. """ tdms_file = nptdms.TdmsFile(str(tdms_filename)) # time is always there table = "Cell Track" # Edit naming.dclab2tdms to add features for arg in naming.tdms2dclab: try: data = tdms_file[table][arg].data except KeyError: pass else: if data is None or len(data) == 0: # Ignore empty features. npTDMS treats empty # features in the following way: # - in nptdms 0.8.2, `data` is `None` # - in nptdms 0.9.0, `data` is an array of length 0 continue self._events[naming.tdms2dclab[arg]] = data if len(self._events) == 0: raise IncompleteTDMSFileFormatError( "No usable feature data found in '{}'!".format(tdms_filename)) # Set up configuration config_paths = [self.path.with_name(self._mid + "_para.ini"), self.path.with_name(self._mid + "_camera.ini")] for cp in config_paths: if not cp.exists(): raise IncompleteTDMSFileFormatError( "Missing file: {}".format(cp)) shpin_set = self.path.with_name(self._mid + "_SoftwareSettings.ini") if shpin_set.exists(): config_paths.append(shpin_set) tdms_config = Configuration(files=config_paths, disable_checks=True) dclab_config = Configuration() for cfgii in [naming.configmap, naming.config_map_set]: for section in cfgii: for pname in cfgii[section]: meta = cfgii[section][pname] convfunc = dfn.get_config_value_func(section, pname) if isinstance(meta, tuple): osec, opar = meta if osec in tdms_config and opar in tdms_config[osec]: val = tdms_config[osec].pop(opar) dclab_config[section][pname] = convfunc(val) else: dclab_config[section][pname] = convfunc(meta) # Additional information from log file rtfdc_log = self.path.with_name(self._mid + "_log.ini") if rtfdc_log.exists(): with rtfdc_log.open("r", errors="replace") as fd: loglines = fd.readlines() for line in loglines: if line.startswith("[EVENT LOG]"): sv = line.split("]")[1].strip() if sv: dclab_config["setup"]["software version"] = sv rtfdc_parm = self.path.with_name("parameters.txt") if rtfdc_parm.exists(): with rtfdc_parm.open("r", errors="replace") as fd: parlines = fd.readlines() p1 = None p2 = None p3 = None for line in parlines: if line.startswith("pulse_led"): fdur = float(line.split()[1]) dclab_config["imaging"]["flash duration"] = fdur elif line.startswith("numberofchannels"): nc = int(line.split()[1]) dclab_config["fluorescence"]["channel count"] = nc elif line.startswith("laser488"): p1 = float(line.split()[1]) dclab_config["fluorescence"]["laser 1 lambda"] = 488 dclab_config["fluorescence"]["laser 1 power"] = p1 elif line.startswith("laser561"): p2 = float(line.split()[1]) dclab_config["fluorescence"]["laser 2 lambda"] = 561 dclab_config["fluorescence"]["laser 2 power"] = p2 elif line.startswith("laser640"): p3 = float(line.split()[1]) dclab_config["fluorescence"]["laser 3 lambda"] = 640 dclab_config["fluorescence"]["laser 3 power"] = p3 elif line.startswith("samplerate"): sr = int(float(line.split()[1])) dclab_config["fluorescence"]["sample rate"] = sr elif line.startswith("samplesperframe"): spe = int(line.split()[1]) dclab_config["fluorescence"]["samples per event"] = spe elif line.startswith("Vmin"): vmin = float(line.split()[1]) dclab_config["fluorescence"]["signal min"] = vmin elif line.startswith("Vmax"): vmax = float(line.split()[1]) dclab_config["fluorescence"]["signal max"] = vmax elif line.startswith("median_pmt"): mfs = int(line.split()[1]) dclab_config["fluorescence"]["trace median"] = mfs # Add generic channel names (independent of lasers) for ii in range(1, 4): chn = "channel {} name".format(ii) fln = "fl{}_max".format(ii) if fln in self and chn not in dclab_config["fluorescence"]: dclab_config["fluorescence"][chn] = "FL{}".format(ii) lc = bool(p1) + bool(p2) + bool(p3) dclab_config["fluorescence"]["laser count"] = lc li = (p1 is not None) + (p2 is not None) + (p3 is not None) dclab_config["fluorescence"]["lasers installed"] = li dclab_config["fluorescence"]["channels installed"] = 3 # Additional information from commented-out log-file (manual) with config_paths[0].open("r", errors="replace") as fd: lns = [s[1:].strip() for s in fd.readlines() if s.startswith("#")] if lns and lns[0] == "[FLUOR]": if ("software version" not in dclab_config["setup"] and lns[1].startswith("fRTDC")): dclab_config["setup"]["software version"] = lns[1] for ll in lns[2:]: if ("sample rate" not in dclab_config["fluorescence"] and ll.startswith("Samplerate")): val = int(float(ll.split("=")[1])) dclab_config["fluorescence"]["sample rate"] = val elif ("signal min" not in dclab_config["fluorescence"] and ll.startswith("ADCmin")): val = float(ll.split("=")[1]) dclab_config["fluorescence"]["signal min"] = val elif ("signal max" not in dclab_config["fluorescence"] and ll.startswith("ADCmax")): val = float(ll.split("=")[1]) dclab_config["fluorescence"]["signal max"] = val self.config = dclab_config self._complete_config_tdms(tdms_config) self._init_filters() # Load log files log_files = config_paths for name in [self._mid + "_events.txt", self._mid + "_log.ini", self._mid + "_SoftwareSettings.ini", "FG_Config.mcf", "parameters.txt"]: pl = self.path.with_name(name) if pl.exists(): log_files.append(pl) for pp in log_files: with pp.open("r", errors="replace") as f: cfg = [s.strip() for s in f.readlines()] self.logs[pp.name] = cfg
def read_from_path(cls, tdms_path: pathlib.Path, meta: MetaData): tdms_operator = nptdms.TdmsFile(tdms_path, memmap_dir=tempfile.gettempdir()) return cls(tdms_operator, meta)
def tdms_to_hdf5(tdms_file, h5_file, load_data=True, chan_map='', memmap=True, compression_level=0): """ Converts TDMS data output from the LabView DAQ software used in the Viventi Lab to record from multiplexing neural implants. This method will most likely not interpret other TDMS files: see npTDMS for general file handling. Parameters ---------- tdms_file : path (string) h5_file : path (string) chan_map : path (string) Optional table specifying a channel permutation. The first p rows of the outgoing H5 file will be the contents of these channels in sequence. The next (N-p) rows will be any channels not specified, in the order they are found. memmap : bool compression_level : int Optionally compress the outgoing H5 rows with zlib compression. This can reduce the time cost caused by disk access. """ map_dir = tempfile.gettempdir() if memmap else None with tables.open_file(h5_file, mode='w') as h5_file: tdms_file = nptdms.TdmsFile(tdms_file, memmap_dir=map_dir) # assume for now there is only a single group -- see more files later # Catch an API change (older version first) try: t_group = tdms_file.groups()[0] group = tdms_file.object(t_group) chans = tdms_file.group_channels(t_group) except AttributeError: group = tdms_file.groups()[0] # Headstage channels and BNC channels are presently lumped into the "data" HDF5 array.. it might make sense # to separate them chans = group.channels() n_col = len(chans) n_row = len(chans[0]) # The H5 file will be constructed as follows: # * create a Group for the info section # * create a CArray with zlib(3) compression for the data channels # * create separate Arrays for special values # (SampRate[SamplingRate], numRow[nrRows], numCol[nrColumns], # OSR[OverSampling], numChan[nrColumns+nrBNCs]) special_conversion = dict(SamplingRate='sampRate', nrRows='numRow', nrColumns='numCol', OverSampling='OSR') h5_info = h5_file.create_group(h5_file.root, 'info') for (key, val) in group.properties.items(): if isinstance(val, str): # pytables doesn't support strings as arrays arr = h5_file.create_vlarray(h5_info, key, atom=tables.ObjectAtom()) arr.append(val) elif isinstance(val, np.datetime64): h5_file.create_array(h5_info, key, obj=val.astype('f8'), atom=tables.Time64Atom()) else: h5_file.create_array(h5_info, key, obj=val) if key in special_conversion: print('caught', key) # Put this array at the top level with new name h5_file.create_array('/', special_conversion[key], obj=val) # do extra extra conversions try: num_chan = group.properties['nrColumns'] + group.properties[ 'nrBNCs'] h5_file.create_array(h5_file.root, 'numChan', num_chan) except KeyError: pass try: mux_ratio = group.properties['OverSampling'] * group.properties[ 'nrRows'] Fs = float(group.properties['SamplingRate']) / mux_ratio h5_file.create_array(h5_file.root, 'Fs', Fs) except KeyError: print('Could not determine sampling rate') h5_file.flush() if not load_data: return h5_file # now get down to the data atom = tables.Float64Atom() if compression_level > 0: filters = tables.Filters(complevel=compression_level, complib='zlib') else: filters = None d_array = h5_file.create_earray(h5_file.root, 'data', atom=atom, shape=(0, n_row), filters=filters, expectedrows=n_col) # create a reverse lookup to index channels by number col_mapping = dict([(ch.properties['NI_ArrayColumn'], ch) for ch in chans]) # If a channel permutation is requested, lay down channels # in that order. Otherwise go in sequential order. if chan_map: chan_map = np.loadtxt(chan_map).astype('i') if chan_map.ndim > 1: print(chan_map.shape) # the actual channel permutation is in the 1st column # the array matrix coordinates are in the next columns chan_ij = chan_map[:, 1:3] chan_map = chan_map[:, 0] else: chan_ij = None # do any channels not specified at the end if len(chan_map) < n_col: left_out = set(range(n_col)).difference(chan_map.tolist()) left_out = sorted(left_out) chan_map = np.r_[chan_map, left_out] else: chan_map = list(range(n_col)) chan_ij = None for n in chan_map: # get TDMS column ch = col_mapping[n] # make a temp array here.. if all data in memory, then this is # slightly wasteful, but if it is mmap'd then this is more flexible d = ch.data[:] d_array.append(d[None, :]) print('copied channel', ch.path, d_array.shape) if chan_ij is not None: h5_file.create_array(h5_file.root, 'channel_ij', obj=chan_ij) return h5_file
def prepare_afe_primary_file(tdms_file, write_path=None): tdms_path, tdms_name = os.path.split(tdms_file) tdms_name = os.path.splitext(tdms_name)[0] if write_path is None: write_path = tdms_path if not os.path.exists(write_path): mkdir_p(write_path) new_primary_file = os.path.join(write_path, tdms_name + '.h5') with tables.open_file(new_primary_file, 'w') as hdf: tdms = nptdms.TdmsFile(tdms_file) # Translate metadata t_group = tdms.groups()[0] group = tdms.object(t_group) rename_arrays = dict(SamplingRate='sampRate', nrRows='numRow', nrColumns='numCol', OverSampling='OSR') h5_info = hdf.create_group('/', 'info') for (key, val) in group.properties.items(): if isinstance(val, str): # pytables doesn't support strings as arrays arr = hdf.create_vlarray(h5_info, key, atom=tables.ObjectAtom()) arr.append(val) else: hdf.create_array(h5_info, key, obj=val) if key in rename_arrays: hdf.create_array('/', rename_arrays[key], obj=val) # Parse channels chans = tdms.group_channels(t_group) elec_chans = [c for c in chans if 'CH_' in c.channel] bnc_chans = [c for c in chans if 'BNC_' in c.channel] # For now, files only have 1 row and all channels relate to that row num_per_electrode_row = len(elec_chans) # do extra extra conversions fs = float(group.properties['SamplingRate']) / ( num_per_electrode_row * group.properties['OverSampling']) hdf.create_array(hdf.root, 'Fs', fs) # ensure that channels are ordered correctly elec_mapping = dict([(ch.properties['NI_ArrayColumn'], ch) for ch in elec_chans]) # This value is currently always 1 -- but leave this factor in for future flexibility num_electrode_rows = len(elec_chans) // num_per_electrode_row if num_per_electrode_row * num_electrode_rows < len(elec_chans): print('There were excess TDMS channels: {}'.format( len(elec_chans))) channels_per_row = 32 sampling_offset = 6 hdf_array = hdf.create_carray('/', 'data', atom=tables.Float64Atom(), shape=(channels_per_row * num_electrode_rows, len(elec_chans[0].data))) for elec_group in range(num_electrode_rows): for c in range(channels_per_row): chan_a = elec_mapping[2 * c + sampling_offset] chan_b = elec_mapping[2 * c + 1 + sampling_offset] hdf_array[elec_group * channels_per_row + c, :] = 0.5 * (chan_a.data + chan_b.data) sampling_offset += num_per_electrode_row bnc_array = hdf.create_carray('/', 'bnc', atom=tables.Float64Atom(), shape=(len(bnc_chans), len(bnc_chans[0].data))) bnc_mapping = dict([(ch.properties['NI_ArrayColumn'] - len(elec_chans), ch) for ch in bnc_chans]) for n in range(len(bnc_mapping)): bnc_array[n, :] = bnc_mapping[n].data return
def readtdmsfile(filepath): """ read tdms file at filepath """ tdms_file = nptdms.TdmsFile(filepath) return tdms_file
# -*- coding: utf-8 -*- """ Created on Wed Jan 31 15:05:42 2018 @author: SWC """ import numpy as np; import nptdms as td; import cv2 #read tdms file of short video file_path = '' tdms_file = td.TdmsFile(file_path) group = tdms_file.groups() channel = tdms_file.group_channels(group[0]) channel_object = tdms_file.object(group[0],'data') data = channel_object.data video = np.reshape(data,(len(data)/1024/1048,1024,1280)) cv2.imshow('frame',video[0,:,:])
"\t██╔══╝ ██╔═══╝ ██╔══██║" + "\n" + "\t███████╗██║ ██║ ██║" + "\n" + "\t╚══════╝╚═╝ ╚═╝ ╚═╝") print("\tEngine Performance Analysis (v1.1)") print("\tJonathan Palafoutas") print("") print("\t"+pre_lc+str(n)+", "+pre_pt+str(n)+", "+pre_tc+str(n)) print("") print("Setting up ...") print("\tLibraries enabled") # ------------------------------------------------------------------------------ # Import LabView data (second step of setting up) # Load cells lc = nptdms.TdmsFile(TDMS_path + pre_lc + str(n) + ".TDMS") lc_time = lc["Untitled"][time_name][:] lc_LNG = lc["Untitled"][lc_LNG_name][:] lc_LOX = lc["Untitled"][lc_LOX_name][:] lc_engine = lc["Untitled"][lc_engine_name][:] print("\tLoad cell data imported") # Pressure transducers pt = nptdms.TdmsFile(TDMS_path + pre_pt + str(n) + ".TDMS") pt_time = pt["Untitled"][time_name][:] pt_LNG1 = pt["Untitled"][pt_LNG1_name][:] pt_LNG2 = pt["Untitled"][pt_LNG2_name][:] pt_LOX1 = pt["Untitled"][pt_LOX1_name][:] pt_LOX2 = pt["Untitled"][pt_LOX2_name][:] pt_LOX3 = pt["Untitled"][pt_LOX3_name][:]