def getDataFrame(filename):
if type(filename) == str:
tdms_file = TdmsFile(filename) #,memmap_dir=cwd)
tddf = tdms_file.as_dataframe()
else:
raise TypeError('I need a single filename')
return tddf
def load_data(path):
# Handles nested dictionaries for us
CSV_PATH, PLOT_PATH, RAW_CSV_DATA = output_directories(path)
sample_dict = vivdict()
directory_listing = [
os.path.join(path, directory) for directory in os.listdir(path)
]
directory_listing = list(
filter(
lambda x: not re.findall(r"(CSV DATA)|(PLOT DATA)|(RAW CSV DATA)", x),
directory_listing,
)
)
for directory in directory_listing:
"""
Example:
directory == '2020.07.30_14.23.02_2E2M_Control 3'
sample == 2E2M
sample_type == Control
sample_rep == 3
"""
log.info(f"Loading data for -> {directory}")
sample = re.split(r"[_\-]", directory)[-2]
sample_type = " ".join(re.split(r"[_\-]", directory)[-1].split(" ")[0:-1])
sample_rep = " ".join(re.split(r"[_\-]", directory)[-1].split(" ")[-1])
# Get rid of Index file
file = list(
filter(lambda x: not re.findall(r"index", x), glob.glob(f"{directory}\*"))
)
if len(file) == 1:
file = file[0]
try:
tdms_file = TdmsFile(file)
df = tdms_file.as_dataframe()
df = df[
list(filter(lambda x: re.findall(r"Timestampe|CO2", x), df.columns))
]
df.columns = ["CO2", "Time"]
df["Time"] = df["Time"].sub(df["Time"].min()).sub(40)
df.set_index("Time", inplace=True)
initial_condition = df[:-30].mean()
df = df.sub(initial_condition)
sample_dict[sample][sample_type][sample_rep] = df
df.to_csv(
os.path.join(RAW_CSV_DATA, f"{sample}_{sample_type}_{sample_rep}.csv"),
sep=",",
index=True,
)
except [FileNotFoundError, OSError] as e:
log(f"{e}\t File: {file} \t Directory: {directory}")
else:
log(f"Have some files here I don't like {directory} // \n{file} \n-- Skipping this")
return sample_dict
def velocity_calc(PDname, method='max'):
PDfile = TdmsFile(PDname)
PDdata = PDfile.as_dataframe(time_index=True, absolute_time=False)
#gets data for each photodiode
PD1 = PDdata[PDdata.columns[0::4]]
PD2 = PDdata[PDdata.columns[1::4]]
PD3 = PDdata[PDdata.columns[2::4]]
PD4 = PDdata[PDdata.columns[3::4]]
del PDdata
# Choose the method for the determination of the velocity
if method == 'diff':
D1 = PD1.diff()
D2 = PD2.diff()
D3 = PD3.diff()
D4 = PD4.diff()
elif method == 'max':
D1 = PD1
D2 = PD2
D3 = PD3
D4 = PD4
else:
sys.exit('The method you have chosen for the velicty calculation is' +
' not reconized. Please select a different method and retry.')
#finds the time point at which D# is at a max
del PD1, PD2, PD3, PD4
t1 = D1.idxmax()
t2 = D2.idxmax()
t3 = D3.idxmax()
t4 = D4.idxmax()
del D1, D2, D3, D4
#lengths between photodiodes
L1 = 0.127762
L2 = 0.129337
L3 = 0.130175
#takes the difference in time values to get values for each velocity
T1 = pd.Series(t2.values - t1.values)
T2 = pd.Series(t3.values - t2.values)
T3 = pd.Series(t4.values - t3.values)
V1 = L1/T1.values
V2 = L2/T2.values
V3 = L3/T3.values
# measurement error calculation
R1 = np.sqrt((-.5*(L1/T1.values**2)*1e-6)**2+(1/T1.values*0.003175)**2)
R2 = np.sqrt((-.5*(L2/T2.values**2)*1e-6)**2+(1/T2.values*0.003175)**2)
R3 = np.sqrt((-.5*(L3/T3.values**2)*1e-6)**2+(1/T3.values*0.003175)**2)
vel_data = pd.DataFrame(np.transpose(
np.vstack((V1, V2, V3, R1, R2, R3))))
vel_data.columns = ['V1', 'V2', 'V3', 'R1', 'R2', 'R3']
return vel_data
def read_from_tdms(file):
"""
Method to read data from National Instruments technical data management streaming files (TDMS).
Args:
file (`str`): File path + file name string.
Returns:
data (`pandas data frame`): Returns data structured in a pandas data frame.
"""
tdms_file = TdmsFile(file)
data = tdms_file.as_dataframe(time_index=False, absolute_time=False)
return data
def convert_TDMS2MAT(tdms_filename, mat_filename, struct):
tdms_file = TdmsFile(tdms_filename)
tdms_df = tdms_file.as_dataframe(time_index=False)
trans = {
key: key.replace("'", '').replace("/", '_')[1:]
for key in tdms_df.keys()
}
tdms_df_ = tdms_df.rename(columns=trans)
matdata = {
col_name: tdms_df_[col_name].values
for col_name in tdms_df_.columns.values
}
scipy.io.savemat(mat_filename, {struct: matdata},
do_compression=True,
oned_as='column')
def signal_plot(PDname=None, method='diff'):
if PDname is None:
PDname = FindFile('Photodiode')
PDfile = TdmsFile(PDname)
PDdata = PDfile.as_dataframe(time_index=True, absolute_time=False)
num_tests = int(len(PDdata.columns) / 4)
plot_num = test_enter(num_tests)
new_data = PDdata[PDdata.columns[plot_num * 4:(plot_num * 4) + 4]]
new_data.columns = [
'Test {0} '.format(plot_num) + 'PD1',
'Test {0} '.format(plot_num) + 'PD2',
'Test {0} '.format(plot_num) + 'PD3',
'Test {0} '.format(plot_num) + 'PD4'
]
new_data.index.name = 'Time (s)'
# Determine the values of the 1st finite difference
diff_val = new_data.diff()
# Plot the desired signals
new_data.plot(linewidth=3)
# Plot locations of the maximum values of the signals
plt.plot(new_data.idxmax(),
new_data.max(),
marker='o',
linestyle='None',
markersize=8,
label='Max',
color='black',
markerfacecolor='black')
# Plot locations of the maximum values of the 1st finite difference
plt.plot(diff_val.idxmax(),
np.diag(new_data.loc[diff_val.idxmax(), new_data.columns]),
marker='s',
linestyle='None',
markersize=8,
label='Grad',
color='black')
plt.legend()
plt.show()
# PDdata.plot(y=list(PDdata.columns[plot_num*4:(plot_num*4)+4]))
return new_data
def read_noise_tdms(filename, sampleRate=51200):
# read metadata and TDMS channel data
noisedata = TdmsFile(filename)
# parse start time of recording
# strip off folders
startTime = filename.split("/")[-1].split("_")[:2]
# join date (YYYYMMDD) to time (HHMMSS)
startTime = "_".join(startTime)
# convert tdms data to pd.DataFrame
noisedf = noisedata.as_dataframe()
# Make time index
noisedf.index = pd.TimedeltaIndex(data=1 / sampleRate *
np.arange(len(noisedf)),
unit="s")
return noisedf, startTime
class TdmsProcess(object):
def __init__(self, filename='', in_dir='./IN', op_type='to_pickle', extend=''):
self.file_name = filename
self.extend = extend
self.op = op_type
self.init_param()
self.file = os.path.join(in_dir, filename)
self.read_tdms()
# if op_type != 'to_df':
# self.to_df()
def init_param(self):
self.df = pd.DataFrame()
self.out_dir = './OUT'
self.op_dict = {'to_pickle': self.to_pickle,
'to_df': self.to_df,
'to_h5': self.to_h5
}
def read_tdms(self):
self.file_object = TdmsFile(self.file)
return self
def to_pickle(self):
time = datetime.strftime(datetime.now(), '%Y%m%d%H%M%S%f')
out_file_name = os.path.join(self.out_dir, self.extend, '{}.pkl'.format(self.file_name.split('.')[0]))
self.to_df().to_pickle(out_file_name)
return out_file_name
def to_h5(self):
# time = datetime.strftime(datetime.now(), '%Y%m%d%H%M%S%f')
out_file_name = os.path.join(self.out_dir, self.extend, '{}.h5'.format(self.file_name.split('.')[0]))
with pd.HDFStore(out_file_name) as h5:
h5['data'] = self.to_df()
return out_file_name
def to_df(self):
self.df = self.file_object.as_dataframe()
return self.df
print('Checking for new tdms files...')
for f in os.listdir(tdms_dir):
if f.endswith('.tdms'):
csv_name = f[:-21]
# Create new csv files if they don't exist in data dir
if not os.path.isfile(f'{data_dir}{csv_name}.csv'):
print(f' Creating csv file for {csv_name}')
# Create dirs if necessary
if not os.path.exists(data_dir):
os.makedirs(data_dir)
# Read tdms file & convert to dataframe
tdms_file = TdmsFile(f'{tdms_dir}{f}')
tdms_df = tdms_file.as_dataframe()
# Create empty dataframes for data & event info
data_df = pd.DataFrame()
# Loop through tdms file columns & populate dataframes
for column in tdms_df.columns:
if column.split('/')[1] == "'Channels'":
data_df[column.split('/')[-1][1:-1]] = tdms_df[column]
# Save data & event dataframes as csv files
data_df.to_csv(f'{data_dir}{csv_name}.csv', index=False)
print(f' Saved {csv_name}.csv')
print()
# ---------------------- #
i_power = (np.abs(i_wave))**2
i_period = 1 / i_freqs
i_sel = find((i_period >= 1 / f_h) &
(i_period < 1 / f_l)) # select frequency band for averaging
i_Cdelta = mother.cdelta
i_scale_avg = (i_scales * np.ones((nt, 1))).transpose()
i_scale_avg = i_power / i_scale_avg # As in Torrence and Compo (1998) equation 24
i_scale_avg = dj / Fs * i_Cdelta * i_scale_avg[i_sel, :].sum(axis=0)
i_max = max(i_scale_avg)
return i_max
###################################Load datafile###############################
print('Loading ', myfile)
tdms_file = TdmsFile(myfile, memmap_dir='/data1/ryan/') # set buffer directory
tddf = tdms_file.as_dataframe() # Load TDMS file AS DATAFRAME
##################optional: cut out the ramp-up only based on IMAG channel############
# tddf = tddf.loc[np.array(mf.getchdata(tddf,'IMAG',1)) > 0.95]
########################Processing#############################################
plt.ioff()
logf = open(outputdir + 'eventlog.csv', 'ab')
# h5f_ch1 = h5py.File('/data1/ryan/training_top_ch1.h5', 'w')
# Evaluate noise level in terms of scale-averaged value and set the threshold for peak detection
noisesample = getchdata(tddf, ch_to_load2, 1)[0:noise_slength]
threshold = 5 * noise_estimate(noisesample, f_high,
f_low) # 5 times noise level threshold
# Process the data
def chunker(seq, size):
return (seq[pos:pos + size] for pos in xrange(0, len(seq), size))
def Reduce_data(data_X, data_Y, x1):
Mx = np.column_stack((data_X[:, np.newaxis], data_Y[:, np.newaxis]))
M2 = Red.rdp(Mx, x1)
data_X2 = M2[:, 0].transpose()
data_Y2 = M2[:, 1].transpose()
return data_X2, data_Y2
def PandasResample(df, length):
td = (df.index[-1] - df.index[0]) / (length - 1)
return df.resample(td, how='mean').interpolate() # Handle NaNs when upsampling
tdms_file = TdmsFile('data/_1620291.tdms')
data = tdms_file.as_dataframe()
print data.columns
tizm = tdms_file.object("Unbenannt", "Zeit")
channel = tdms_file.object("Unbenannt", "OfenIO")
channel1 = tdms_file.object("Unbenannt", "T_X6")
channel2 = tdms_file.object("Unbenannt", "Soll")
# plt.plot(tizm.data,channel.data)
# ax.scatter(x, y, c=c, cmap=cmap)
plt.scatter(tizm.data, channel2.data, c=channel.data * 3., cmap=plt.cm.hot)
plt.show()
myfile = 'G:/Data/CCT4/8_24_2017_2_46_30_PM_A129_Q103_15986A.tdms' # filename; replace as needed
ch_to_load1 = 'S_top' #acoustic channel 1
ch_to_load2 = 'S_bot' #acoustic channel 2
ch_to_load3 = 'IMAG' # magnet current channel, value needs to be multiplied by 1920 to
#get current in Amperes
###########################Index location of the event ############################################
location = 12345678 # index of the sample event. Sinse sampling rate is 1 MHz, this is equal to
#################### microseconds from the start of the ramp
############################This loads the entire file as dataframe################################
print('Loading ', myfile)
tdms_file = TdmsFile(myfile, memmap_dir='c:/Temp') #set buffer directory
tddf = tdms_file.as_dataframe() #LOADS FILE myfile AS DATAFRAME
#############################Load single event - 5 ms duration window ###############################
ch0_sample = getchdata(tddf, ch_to_load1, 1)[location - 1000:location + 4000]
ch1_sample = getchdata(tddf, ch_to_load2, 1)[location - 1000:location + 4000]
imag_sample = getchdata(tddf, ch_to_load3, 1)[location - 1000:location + 4000]
##################################################Plot the event#####################################
plt.figure(1)
plt.plot(ch0_sample)
plt.plot(ch1_sample)
plt.show()
################################################Print magnret current value##########################
imag_value = 1920 * imag_sample.mean()
print('Magnet current is ', imag_value, ' A')
#!/usr/bin/env python
from nptdms import TdmsFile
import glob
import re
# Update paths below to directory with tdms files (input_dir) and directory for csv outputs (output_dir)
input_dir = '/Users/prioberoi/Documents/nist/netZero/data/raw/'
output_dir = '/Users/prioberoi/Documents/nist/netZero/data/raw/'
for filename in glob.iglob(input_dir+'*.tdms'): #update filepath to location with .tdms files from LabView
tdms_file = TdmsFile(filename)
temp = tdms_file.as_dataframe(time_index=False, absolute_time=False)
start_loc = re.search(' All Day.tdms', filename).start()-10
stop_loc = re.search(' All Day.tdms', filename).start()
temp.to_csv(path_or_buf=(output_dir+filename[start_loc:stop_loc]+".csv"), encoding='utf-8')
tdms_file_list = glob.glob('../audio_tdms/*.tdms')
csv_file_list = glob.glob('../audio_csv/*.csv')
# print(len(tdms_file_list))
mic_data = pd.DataFrame(columns=['mic1_motor', 'mic2'])
counter = 0
file_previous = None
#check if at least one csv file exists, if true, then skip creation of csv's
if os.path.isfile('../audio_csv/' + str(counter).zfill(2) + '-sound-mics.csv'):
for i, file in enumerate(tqdm(tdms_file_list)):
# add a condition to not go through this process over and over
tdms_file = TdmsFile(file)
tdms_df = tdms_file.as_dataframe().rename(
index=str,
columns={
"/'Untitled'/'Micrófono 1 Motor'": "mic1_motor",
"/'Untitled'/'Micrófono2'": "mic2"
})
#mic_data = pd.concat([mic_data, tdms_df], ignore_index=True)
# check after first file
if i > 0 and file_previous is not None:
#print(file[23:-7])
if file[23:-7] == file_previous:
print("Here", i)
mic_data = pd.concat([mic_data_previous, tdms_df],
ignore_index=True)
counter += 1
# save then reset dataframe
mic_data.to_csv('../audio_csv/' + str(counter).zfill(2) +
'-sound-mics.csv',
index=False)
