def main(args):
if len(sys.argv) > 1:
from os.path import splitext, split
from tools import load_data_using_shelve
from scipy.io import savemat, whosmat
path = sys.argv[1]
folder, filename = split(path)
#ignoring H,M,S from filename
keyid = splitext(filename)[0]
print 'Using keyid %s' % keyid
output_file = keyid+'.mat'
print 'Loading %s' % path
analysis = load_data_using_shelve(path, keyid, flag='r')
print 'Saving as %s' % output_file
analysis_dict = {}
for feature, data in analysis.time_series.items():
print 'Adding', feature
analysis_dict[feature] = data.getData()
savemat(output_file, analysis_dict, appendmat=True)
print whosmat(output_file)
else:
print 'syntax is : python py2mat.py inputfile'
def load_CMU_dataset(self, path):
print sio.whosmat(path)
matTF = sio.loadmat(path)#loading tensor of formatted faces' images
self.TF = matTF['FullTensor']
print 'Tensor order:', self.TF.shape
self.MF = np.reshape(self.TF,(30*11*21,1024))
print 'Matrization order:', self.MF.shape
def main(args):
if len(sys.argv) > 1:
from os.path import splitext, split
from tools import load_data_using_shelve
from scipy.io import savemat, whosmat
path = sys.argv[1]
folder, filename = split(path)
#ignoring H,M,S from filename
keyid = splitext(filename)[0]
print 'Using keyid %s' % keyid
output_file = keyid + '.mat'
print 'Loading %s' % path
analysis = load_data_using_shelve(path, keyid, flag='r')
print 'Saving as %s' % output_file
analysis_dict = {}
for feature, data in analysis.time_series.items():
print 'Adding', feature
analysis_dict[feature] = data.getData()
savemat(output_file, analysis_dict, appendmat=True)
print whosmat(output_file)
else:
print 'syntax is : python py2mat.py inputfile'
def main():
#establish paths to data
#often, I was dealing with data separated by folders, between these lists and some logic in the loops, I establish the full path
input_base = '/srv/home/lerandc/Experimental Sr PACBED/'
input_sub_folder = ['S1','S4','S5','S6','S7','S8','S9']
model_path = sys.argv[2]
out_name = sys.argv[3]
x_train_list = []
y_train_list = []
sets = []
int_radii = []
for current_folder in input_sub_folder:
input_folder = input_base + current_folder + '-20180717/' + current_folder + '/'
input_images = [image for image in os.listdir(input_folder) if 'mat' in image]
for image in input_images:
if 'PACBED' in image:
#load images and haadf depth predictions, get labels
img_array = sio.loadmat(input_folder + image)
label_array = sio.loadmat(input_folder + 'HAADFprediction_r8.mat')
cur_radii = image.rsplit('_',1)[-1][0:2]
cur_set = current_folder
#prepare data to be looped through
im_fields = sio.whosmat(input_folder + image)
img_array = img_array[im_fields[0][0]]
img_array = img_array.astype('double')
im_range = img_array.shape[2]
label_fields = sio.whosmat(input_folder+'HAADFprediction_r8.mat')
label_array = np.squeeze(label_array[label_fields[0][0]])
for i in range(im_range):
#prepare individual array to be formatted for prediction
img = img_array[:,:,i]
img = sm.imresize(np.squeeze(img),(157, 157))
img = img.astype('double')
img = scale_range(img, 0, 1)
#add image and labels to prediction arrays
img_size = img.shape[0]
label = label_array[i]
img = np.reshape(img,(img_size,img_size,1))
x_train_list.append(img)
y_train_list.append(label)
sets.append(cur_set)
int_radii.append(cur_radii)
#load and compile the final model
# compile setting:
model = load_model(model_path + 'FinalModel_resnet50.h5')
#make predictions and save results to mat file
p_arrays = model.predict(np.asarray(x_train_list), batch_size=32)
y_list = np.asarray(y_train_list)
sio.savemat(out_name,{'measured':y_list,'probabilities':p_arrays,'sets':sets,'radii':int_radii})
def _load_doc_term_data_from_mat_(path):
tmp_dict = {
key: None
for key in [key_name_data, key_name_term_labels, key_name_doc_labels]
}
# Get the fields from the matlab file
matlab_content = whosmat(path)
for index, element in enumerate(matlab_content):
# if co-occurence data, load
if element[0] == key_name_data:
tmp_dict[key_name_data] = loadmat(path)[key_name_data]
# if term label data, load and convert to list
elif element[0] == key_name_term_labels:
tmp_dict[key_name_term_labels] = loadmat(
path)[key_name_term_labels]
tmp_dict[key_name_term_labels] = tmp_dict[
key_name_term_labels].tolist()
# if doc label data, load, convert to list and take list inside...
elif element[0] == key_name_doc_labels:
tmp_dict[key_name_doc_labels] = loadmat(path)[key_name_doc_labels]
tmp_dict[key_name_doc_labels] = tmp_dict[
key_name_doc_labels].tolist()[0]
return tmp_dict
def write(self):
mat_vars = sio.whosmat(self.fileo)
if not set(['aa', 'trans']).isdisjoint([
item for sublist in mat_vars for item in sublist
]): #tell the user that they might overwrite their data
ret=QtWidgets.QMessageBox.warning(self, "pyCM Warning", \
"There is already data associated with this analysis step saved. Overwrite and invalidate subsequent steps?", \
QtWidgets.QMessageBox.Yes | QtWidgets.QMessageBox.No, QtWidgets.QMessageBox.No)
if ret == QtWidgets.QMessageBox.No: #don't overwrite
return
else:
#delete fitting parameters with pyCMcommon helper function, which negates FEA pre-processing as well.
clear_mat(self.fileo, ['x_out', 'aa_mask', 'spline_x'])
mat_contents = sio.loadmat(self.fileo)
new = {
'trans': {
'ref': self.refTrans,
'float': self.floatTrans
},
'aa': {
'pnts': self.ap,
'gsize': self.gsize
}
}
mat_contents.update(new) #update the dictionary
sio.savemat(self.fileo, mat_contents)
self.ui.statLabel.setText("Wrote data.")
self.unsaved_changes = False
def write_new(self):
if self.ui.refButton.isChecked():
str_d = 'ref'
if self.ui.floatButton.isChecked():
str_d = 'float'
if not hasattr(self, 'fileo'):
self.fileo, _, = get_open_file('*.mat', os.getcwd())
self.ui.releaseButton.setEnabled(True)
if self.fileo:
x_o = self.rawPnts[self.bool_pnt, 0]
y_o = self.rawPnts[self.bool_pnt, 1]
z_o = self.rawPnts[self.bool_pnt, 2]
sio.savemat(
self.fileo, {
str_d: {
'x_out': self.Outline,
'rawPnts': self.rawPnts,
'mask': self.bool_pnt,
'x': x_o,
'y': y_o,
'z': z_o,
'fname': self.filec
}
})
print 'Wrote %s data' % (str_d)
else:
mat_vars = sio.whosmat(self.fileo)
if str_d in mat_vars[
0]: #tell the user that they might overwrite their data
ret=QtGui.QMessageBox.warning(self, "pyCM Warning", \
"The %s dataset has already been written. Overwrite?"%(str_d), \
QtGui.QMessageBox.No,QtGui.QMessageBox.Yes,QtGui.QMessageBox.NoButton)
if ret == QtGui.QMessageBox.No: #don't overwrite
return
mat_contents = sio.loadmat(self.fileo)
x_o = self.rawPnts[self.bool_pnt, 0]
y_o = self.rawPnts[self.bool_pnt, 1]
z_o = self.rawPnts[self.bool_pnt, 2]
new = {
str_d: {
'x_out': self.Outline,
'rawPnts': self.rawPnts,
'mask': self.bool_pnt,
'x': x_o,
'y': y_o,
'z': z_o,
'fname': self.filec
}
}
mat_contents.update(new) #update the dictionary
sio.savemat(self.fileo, mat_contents)
print 'Wrote %s data' % (str_d)
def importMAT(self, path):
"""
import the cro road data as pandas DataFrame
@:param path [in] path to mat file
@:param df [out] road data as pandas DataFrame
"""
df = pd.DataFrame()
#load mat file
mat = sio.loadmat(path)
#get filename
filename = sio.whosmat(path)[0][0]
#get columns names
liste = mat[filename].dtype.descr
#itteration over column names
counter = 0
for i in liste:
#print(i[0])
for j in range(len(mat[filename][0][0][1].dtype.descr)):
#print(counter,j)
#print(mat[filename][0][0][1].dtype.descr[j][0])
#print(i[0]+'_'+mat[filename][0][0][1].dtype.descr[j][0])
df[i[0] + '_' + mat[filename][0][0][1].dtype.descr[j][0]] = [
float(
mat[filename][0][0][liste[counter][0]][0][0][j][k][0])
for k in range(
len(np.array(mat[filename][0][0][liste[0][0]][0][0]
[0])))
]
counter = counter + 1
return df
def grok_file(fname):
print('--- grok_file: ' + str(fname))
# [('name', (x,y), 'type'), ...]
print('# whosmat' + str(sio.whosmat(fname)))
data = sio.loadmat(fname, struct_as_record=False, squeeze_me=True)
dinfo = {}
for k in data['datainfo']._fieldnames:
dinfo[k] = data['datainfo'].__getattribute__(k)
print('')
print('experimenter: ' + str(dinfo['experimenter']))
print('description: ' + str(dinfo['description']))
print('date: ' + str(dinfo['date']))
print('sample no: ' + str(dinfo['SmplNo']))
print('animal: ' + str(dinfo['animal']))
print('ncell: ' + str(dinfo['Ncell']))
if type(dinfo['RecNo']) is int:
print('record: ' + str(dinfo['RecNo']))
print('start: ' + str(dinfo['RecStartTime'][0]))
# spikes[cell][spike]
else:
print('records:')
for i in range(len(dinfo['RecNo'])):
print('record: ' + str(dinfo['RecNo'][i]))
print('start: ' + str(dinfo['RecStartTime'][i]))
def write(self):
mat_vars=sio.whosmat(self.fileo)
if not set(['spline_x']).isdisjoint([item for sublist in mat_vars for item in sublist]): #tell the user that they might overwrite their data
ret=QtWidgets.QMessageBox.warning(self, "pyCM Warning", \
"There is already data associated with this analysis step saved. Overwrite and invalidate subsequent steps?", \
QtWidgets.QMessageBox.Yes | QtWidgets.QMessageBox.No, QtWidgets.QMessageBox.No)
if ret == QtWidgets.QMessageBox.No: #don't overwrite
return
else:
#delete fitting parameters with pyCMcommon helper function, which negates key FEA parameters.
clear_mat(self.fileo,['vtk','pickedCornerInd','FEA'])
if hasattr(self,'tck'): #then spline fitting has been done
mat_contents=sio.loadmat(self.fileo)
coefs=[np.reshape(self.tck[2],(len(self.tck[0])-self.tck[3]-1,-1))]
number=np.array([len(self.tck[0]),len(self.tck[1])])
order=np.array([self.tck[3], self.tck[4]])
new={'spline_x': {'form': 'B-', 'knots': [self.tck[0], self.tck[1]], 'kspacing': [self.gx, self.gy], 'coefs': coefs, 'number': number, 'order':order, 'dim': 1, 'tck': self.tck}, 'x_out':self.RefOutline, 'aa_mask':self.bool_pnt}
mat_contents.update(new)
sio.savemat(self.fileo,mat_contents)
self.ui.statLabel.setText("Output written.")
self.fitted=True
self.unsaved_changes=False
else:
self.ui.statLabel.setText("Nothing to write.")
def readmatfile(filename):
try:
outdata = sio.loadmat(filename)[sio.whosmat(filename)[0][0]]
except Exception:
print('read erro ' + filename)
outdata = []
return outdata
def load_mat(path):
variables = scio.whosmat(path)
target = variables[0][0]
data = scio.loadmat(path)
TT_array = data[target]
TT_list = TT_array.tolist()
return TT_list, TT_array
def write_new(self):
if self.ui.refButton.isChecked():
str_d='ref'
self.refWritten=True
if self.ui.floatButton.isChecked():
str_d='float'
self.floatWritten=True
if not hasattr(self,'fileo'):
self.fileo, _, = get_open_file('*.mat',os.getcwd())
if self.fileo:
x_o=self.rawPnts[self.bool_pnt,0]
y_o=self.rawPnts[self.bool_pnt,1]
z_o=self.rawPnts[self.bool_pnt,2]
sio.savemat(self.fileo,{str_d : {'x_out':self.Outline,'rawPnts':self.rawPnts,'mask': self.bool_pnt,'x':x_o,'y':y_o,'z':z_o,'fname':self.filec}})
if self.ui.refButton.isChecked():
self.ui.refButton.setStyleSheet("background-color :rgb(77, 209, 97);")
if self.ui.floatButton.isChecked():
self.ui.floatButton.setStyleSheet("background-color : rgb(77, 209, 97);")
#reset flag on ui to show that data has been modified
else:
if not self.fileo:
self.fileo, _, = get_open_file('*.mat',os.getcwd())
mat_vars=sio.whosmat(self.fileo)
if str_d in [item for sublist in mat_vars for item in sublist]: #tell the user that they might overwrite their data
ret=QtWidgets.QMessageBox.warning(self, "pyCM Warning", \
"There is already data for this step - doing this will invalidate all further existing analysis steps. Continue?"%(str_d), \
QtWidgets.QMessageBox.Yes | QtWidgets.QMessageBox.No, QtWidgets.QMessageBox.No)
if ret == QtWidgets.QMessageBox.No: #don't overwrite
return
mat_contents=sio.loadmat(self.fileo)
x_o=self.rawPnts[self.bool_pnt,0]
y_o=self.rawPnts[self.bool_pnt,1]
z_o=self.rawPnts[self.bool_pnt,2]
new={str_d : {'x_out':self.Outline,'rawPnts':self.rawPnts,'mask': self.bool_pnt,'x':x_o,'y':y_o,'z':z_o}}
mat_contents.update(new) #update the dictionary
if self.ui.refButton.isChecked():
self.ui.refButton.setStyleSheet("background-color : rgb(77, 209, 97);")
if self.ui.floatButton.isChecked():
self.ui.floatButton.setStyleSheet("background-color : rgb(77, 209, 97);")
sio.savemat(self.fileo,mat_contents)
#update status
self.ui.statLabel.setText("Wrote %s data to output file %s."%(str_d,self.fileo))
#check on write
if self.refWritten==True and self.floatWritten==True:
self.unsaved_changes=False
def loadImageFromMat(fname):
#loads an image array from a matlab file with file name "fname"
#assumes that the only variable in the matlab file
#fname is a string representing filename of the file
img_array = sio.loadmat(fname)
fields = sio.whosmat(fname)
img_array = img_array[fields[0][0]]
return img_array
def __init__(self, model):
self.model = sio.loadmat(model)
self.p = self.model['P']
self.w_m = self.model['w_m'][0][0]
self.lam = self.model['lambda'][0][0]
self.vars = sio.whosmat(model)
self.dimension = list(self.vars[5][1])[0]
self.pptw_w_m = self.__pptw_x_w_m__()
def dir_mat(path):
'''
list the variables in mat file.
return a list: [(name, shape, dtype), ...]
'''
import scipy.io as sio
path = get_absolute_path(path)
return sio.whosmat(path)
def fn(directory):
mat_file = directory + '/imuData.mat'
sio.whosmat(mat_file)
#input data
mat_to_obj = sio.loadmat(mat_file, struct_as_record=False, squeeze_me=True)
"""Reformating time_stamp to date: YYYY-MM-DD & time: hh:mm:ss"""
time_stamp = mat_to_obj['__header__']
time = time_stamp.decode("utf-8")
time = time[-20:]
date = time[:6] + time[-5:]
time = time[7:15]
"""Re-organizing class & nested-objects"""
sensorData = mat_to_obj['sensorData']
return sensorData
def load_columns_matlab(filepath, dbg=DebugNone()):
if dbg:
print("Reading matlab file: `{}` ...".format(str(filepath)))
debug(sio.whosmat(str(filepath)))
print()
# http://stackoverflow.com/questions/6273634/access-array-contents-from-a-mat-file-loaded-using-scipy-io-loadmat-python
with open(str(filepath),'rb') as file:
data = DataTree()
sio.loadmat(file, mdict=data, squeeze_me=True, struct_as_record=False,)
return data
def load_mat_file(matfile):
from scipy.io import loadmat, whosmat
return_variable_dict = {}
variables_in_matfile = whosmat(matfile) #获取mat文件中的所有变量,返回的是一个列表，每个列表中是一个元组
file_from_mat = loadmat(matfile)
for variable_i in variables_in_matfile:
variable_i_value = file_from_mat[variable_i[0]]
return_variable_dict[variable_i[0]] = variable_i_value
globals()[variable_i[0]] = variable_i_value #此处相当于将该变量声明并赋值后将其设置为全局变量
#locals()[variable_i[0]] = variable_i_value #此处只是相当于将该变量声明并赋值，并且locals是只读的，并不能修改该变量的原始值！！！！
#只需在前面声明赋值了此变量，并在此行代码后就会发现这个
return return_variable_dict
def open_matlab_file(matlab_filename):
try:
mat = scio.loadmat(matlab_filename, appendmat=True)
except:
try:
mat = scio.loadmat(matlab_filename, appendmat=False)
except FileNotFoundError:
print("File not found")
quit()
# Variable Declaration
b = scio.whosmat(matlab_filename)[0][0]
number_trials = 0
stim_time = []
stim_code = []
firing = []
separate_dictionary = {}
trialled_firing = []
for x in mat['StimTrig'][0][0][4]:
stim_time.append(x[0])
for x in mat['StimTrig'][0][0][5]:
if x[0] != 62:
stim_code.append(x[0])
else:
stim_code.append(0)
number_trials += 1
for x in mat[b][0][0][4]:
firing.append(x[0])
counter = 1
for i,x in enumerate(stim_code):
if x == 0:
separate_dictionary[counter] = i
counter += 1
Temporary_List = []
Temporary_Key = 1
for x in firing:
try:
if x <= stim_time[separate_dictionary[Temporary_Key]]:
Temporary_List.append(x)
else:
trialled_firing.append(Temporary_List)
Temporary_List = []
Temporary_Key += 1
except KeyError:
break
del Temporary_List
del Temporary_Key
del counter
del b
del mat
return stim_code, stim_time, firing, separate_dictionary, trialled_firing, number_trials
def open_matlab_file(matlab_filename):
try:
mat = scio.loadmat(matlab_filename, appendmat=True)
except:
try:
mat = scio.loadmat(matlab_filename, appendmat=False)
except FileNotFoundError:
print("File not found")
quit()
# Variable Declaration
b = scio.whosmat(matlab_filename)[0][0]
number_trials = 0
stim_time = []
stim_code = []
firing = []
separate_dictionary = {}
trialled_firing = []
for x in mat['StimTrig'][0][0][4]:
stim_time.append(x[0])
for x in mat['StimTrig'][0][0][5]:
if x[0] != 62:
stim_code.append(x[0])
else:
stim_code.append(0)
number_trials += 1
for x in mat[b][0][0][4]:
firing.append(x[0])
counter = 1
for i, x in enumerate(stim_code):
if x == 0:
separate_dictionary[counter] = i
counter += 1
Temporary_List = []
Temporary_Key = 1
for x in firing:
try:
if x <= stim_time[separate_dictionary[Temporary_Key]]:
Temporary_List.append(x)
else:
trialled_firing.append(Temporary_List)
Temporary_List = []
Temporary_Key += 1
except KeyError:
break
del Temporary_List
del Temporary_Key
del counter
del b
del mat
return stim_code, stim_time, firing, separate_dictionary, trialled_firing, number_trials
def open_file(self, file):
renamed_file = file.split('/')
file_path = renamed_file[:-1]
file_name = renamed_file[-1]
path = ''.join(x + platform_filename for x in file_path)
original_path = os.getcwd()
os.chdir(path)
try:
self.mat = sc_io.loadmat(file_name, appendmat=True)
except Exception as e:
try:
self.mat = sc_io.loadmat(file_name, appendmat=False)
except FileNotFoundError:
print("File not found: ", e)
temp_var = sc_io.whosmat(file)[0][0]
os.chdir(original_path)
for x in self.mat['StimTrig'][0][0][4]:
self.stimuli_time.append(x[0])
for x in self.mat['StimTrig'][0][0][5]:
if x[0] != 62:
self.stimuli_code.append(x[0])
else:
self.stimuli_code.append(0)
self.number_trials += 1
for x in self.mat[temp_var][0][0][4]:
self.firing.append(x[0])
counter = 1
for i, x in enumerate(self.stimuli_code):
if x == 0:
self.dictionary_marking_0s_index_in_stimuli_lists[counter] = i
counter += 1
temp_list = []
temp_key = 1
for x in self.firing:
try:
if x <= self.stimuli_time[
self.dictionary_marking_0s_index_in_stimuli_lists[
temp_key]]:
temp_list.append(x)
else:
self.trialled_firing.append(temp_list)
temp_list = []
temp_key += 1
except KeyError:
break
def get_data_dict(data_dir) -> dict:
imdb_mat = os.path.join(data_dir, "imdb.mat")
print('data file: ', imdb_mat)
# loading file into memory
sio.whosmat(imdb_mat)
f = sio.loadmat(imdb_mat)
# getting the important bit of the file
data = f['imdb'][0][0]
num_entries = len(data[2][0])
# turning the array into a dict of key:filename, value:gender
filename_gender_dict = {}
for i in range(num_entries):
key = str(data[2][0][i][0])
value = data[3][0][i]
filename_gender_dict[key] = value
print('number of records from data file: ',
len(filename_gender_dict.keys()))
return filename_gender_dict
def _check_mat_struct(fname):
"""Check if the mat struct contains 'EEG'."""
if not check_version("scipy", "0.12"):
raise RuntimeError("scipy >= 0.12 must be installed for reading EEGLAB" " files.")
from scipy import io
mat = io.whosmat(fname, struct_as_record=False, squeeze_me=True)
if "ALLEEG" in mat[0]:
raise NotImplementedError(
"Loading an ALLEEG array is not supported. Please contact" "mne-python developers for more information."
)
elif "EEG" not in mat[0]:
msg = "Unknown array in the .set file."
raise ValueError(msg)
def load_mat_file(mat_file_path):
all_files = os.listdir(mat_file_path)
for i in all_files:
if i.endswith('.mat'):
data_file = i
return_variable_dict = {}
variables_in_matfile = whosmat(data_file)
file_from_mat = loadmat(data_file)
for variable_i in variables_in_matfile:
variable_i_value = file_from_mat[variable_i[0]]
return_variable_dict[variable_i[0]] = variable_i_value
data_x = return_variable_dict['data_x']
label_y = return_variable_dict['label_y']
return data_x, label_y
def _check_mat_struct(fname):
"""Check if the mat struct contains 'EEG'."""
if not check_version('scipy', '0.12'):
raise RuntimeError('scipy >= 0.12 must be installed for reading EEGLAB'
' files.')
from scipy import io
mat = io.whosmat(fname, struct_as_record=False, squeeze_me=True)
if 'ALLEEG' in mat[0]:
raise NotImplementedError(
'Loading an ALLEEG array is not supported. Please contact'
'mne-python developers for more information.')
elif 'EEG' not in mat[0]:
msg = ('Unknown array in the .set file.')
raise ValueError(msg)
def open_file(self, file):
renamed_file = file.split('/')
file_path = renamed_file[:-1]
file_name = renamed_file[-1]
path = ''.join(x + platform_filename for x in file_path)
original_path = os.getcwd()
os.chdir(path)
try:
self.mat = sc_io.loadmat(file_name, appendmat=True)
except Exception as e:
try:
self.mat = sc_io.loadmat(file_name, appendmat=False)
except FileNotFoundError:
print("File not found: ", e)
temp_var = sc_io.whosmat(file)[0][0]
os.chdir(original_path)
for x in self.mat['StimTrig'][0][0][4]:
self.stimuli_time.append(x[0])
for x in self.mat['StimTrig'][0][0][5]:
if x[0] != 62:
self.stimuli_code.append(x[0])
else:
self.stimuli_code.append(0)
self.number_trials += 1
for x in self.mat[temp_var][0][0][4]:
self.firing.append(x[0])
counter = 1
for i, x in enumerate(self.stimuli_code):
if x == 0:
self.dictionary_marking_0s_index_in_stimuli_lists[counter] = i
counter += 1
temp_list = []
temp_key = 1
for x in self.firing:
try:
if x <= self.stimuli_time[self.dictionary_marking_0s_index_in_stimuli_lists[temp_key]]:
temp_list.append(x)
else:
self.trialled_firing.append(temp_list)
temp_list = []
temp_key += 1
except KeyError:
break
def _check_mat_struct(fname):
"""Check if the mat struct contains 'EEG'."""
if not check_version('scipy', '0.12'):
raise RuntimeError('scipy >= 0.12 must be installed for reading EEGLAB'
' files.')
from scipy import io
mat = io.whosmat(fname, struct_as_record=False,
squeeze_me=True)
if 'ALLEEG' in mat[0]:
raise NotImplementedError(
'Loading an ALLEEG array is not supported. Please contact'
'mne-python developers for more information.')
elif 'EEG' not in mat[0]:
msg = ('Unknown array in the .set file.')
raise ValueError(msg)
def whosmat(filename: str,
byte_order: str = None,
matlab_compatible: bool = False) -> list:
"""
List variables inside a MATLAB file.
Parameters
----------
filename : str
Name of the mat file.
Can also pass open file-like object.
byte_order : str or None, optional
None by default, implying byte order guessed from mat file.
Otherwise can be one of (‘native’, ‘=’, ‘little’, ‘<’, ‘BIG’, ‘>’).
matlab_compatible : bool, optional
Returns matrices as would be loaded by MATLAB.
Returns
-------
variable : list of tuples
A list of tuples, where each tuple holds the matrix name (a string),
its shape (tuple of ints), and its data class (a string).
Possible data classes are: int8, uint8, int16, uint16, int32, uint32,
int64, uint64, single, double, cell, struct, object, char, sparse,
function, opaque, logical, unknown.
"""
# Remove other than '.mat' extension if filename has the other extention.
if filename[-4:] != '.mat':
if filename.find('.') != -1:
bad_ext = filename[filename.find('.'):]
print('Warning: Filename "' + filename +
'" has a bad extension "' + bad_ext + '" .')
filename = filename[0:filename.find('.')]
print('Renamed to "' + filename + '.mat".')
# Add '.mat' force.
filename = filename + '.mat'
variables = io.whosmat(filename,
appendmat=False,
byte_order=byte_order,
matlab_compatible=matlab_compatible)
return variables
def load_columns_matlab(filepath, dbg=DebugNone()):
if dbg:
print("Reading matlab file: `{}` ...".format(str(filepath)))
debug(sio.whosmat(str(filepath)))
print()
# http://stackoverflow.com/questions/6273634/access-array-contents-from-a-mat-file-loaded-using-scipy-io-loadmat-python
with open(str(filepath), 'rb') as file:
data = DataTree()
sio.loadmat(
file,
mdict=data,
squeeze_me=True,
struct_as_record=False,
)
return data
def load_data(self,filein=None):
'''
loads dataset from filein. At the time, only supports .mat files.
'''
if filein is None:
filein=uiopen(title='Select mat or hdf5 file for import',filetypes=[('.mat files','*.mat'),('hdf5 files','*.h5'),('hdf5 files','*.hdf5')])
if filein=='.':
return
if filein.endswith('.mat'):
data_unk,var=loadmat(filein),whosmat(filein)
var = [v[0] for v in var]
elif filein.endswith('.hdf5') or filein.endswith('.h5'):
data_unk=h5py.File(filein, 'r')
var = data_unk.keys
data = {}
for k in var:
print()
tmp_var = np.asarray(data_unk[k])
if k in ('__header__', '__version__', '__globals__'):
continue
elif len(tmp_var.flatten())==1:
data['fr']=float(tmp_var)
elif tmp_var.ndim==2:
data['H']=tmp_var
elif tmp_var.ndim==3:
data['W']=tmp_var
# Checks inner dimension match if both H and W present in file.
if ('H' in data and 'W' in data) and data['H'].shape[0] != data['W'].shape[-1]:
# try flipping dims
if ('H' in data and 'W' in data) and data['H'].T.shape[0] == data['W'].T.shape[-1]:
data['H'] = data['H'].T
data['W'] = data['W'].T
else:
self.message('Error: Inner or outer dimensions of W [shape={}] and H [shape={}] do not match!'.format(data['H'].shape, data['W'].shape))
return
if 'H' in data:
if 'W' in data:
data['W_shape']=data['W'].shape
data['W']=data['W'].reshape(data['W'].shape[0]*data['W'].shape[1],data['W'].shape[2])
if 'fr' not in data:
data['fr']=data['H'].shape[1]/60 # Forces length to be 1 minute!
self.data=data
if not self.display:
return self
else:
self.message('Error: .mat file incompatible. Please select a .mat file with three variables: W (3D), H (2D), and fr (1-element float)')
def get_meta(mat_path, db):
meta = loadmat(mat_path)
print(whosmat(mat_path))
print("path shape ", np.shape(meta[db][0, 0]["full_path"]))
print("keys ", meta.keys())
print("2 keys ", meta[db].keys())
full_path = meta[db][0, 0]["full_path"][0]
dob = meta[db][0, 0]["dob"][0] # Matlab serial date number
gender = meta[db][0, 0]["gender"][0]
photo_taken = meta[db][0, 0]["photo_taken"][0] # year
face_score = meta[db][0, 0]["face_score"][0]
second_face_score = meta[db][0, 0]["second_face_score"][0]
face_location = meta[db][0, 0]["face_location"][0]
celeb_id = meta[db][0, 0]["celeb_id"][0]
age = [calc_age(photo_taken[i], dob[i]) for i in range(len(dob))]
return full_path, dob, gender, photo_taken, face_score, second_face_score, age
def load_file_info(file_name):
"""Load basic information about a file returns a list of MES_Record
instances (subclassed as MAT_Record or H5_Record depending on file format
"""
variant = guess_format(file_name)
if variant == 'mat':
vars = [x[0] for x in io.whosmat(file_name)
if 'Df' in x[0]]
handler = MAT_Record
elif variant == 'h5':
with h5py.File(file_name) as f:
vars = [k for k in f.keys() if 'Df' in k]
handler = H5_Record
else:
print "Can't load description"
vars = None
records = [handler(file_name, v) for v in vars]
return records
def imread_mat(filename, name=None):
"""
Read an 'image' from a MATLAB .MAT file. The file can be any version. Files
that are v7.3 require the h5py module. If no name is given, the first
variable is taken.
"""
try:
# Try general first (doesn't work for v7.3+ files)
# SciPy has this built in
# Supports loading just the given variable name
# Otherwise have to load all variables and skip special keys starting with "__" to find the variable to load
# Loaded matracies are already arrays
from scipy.io import loadmat
if name == None:
try:
# Try to get first variable name without loading entire file (only supported in SciPy 0.12+)
from scipy.io import whosmat
keys in whosmat(file_name)
if len(keys) == 0: raise KeyError()
name = keys[0][0]
except:
pass
x = loadmat(filename, variable_names=name)
if name == None:
name = '__' # we need to find first
for name in x.iterkeys():
if name[:2] != '__': break
if name[:2] == '__': raise KeyError() # no variables
return x[name] # can raise key error
except NotImplementedError:
# Try v7.3 file which is an HDF5 file
# We have to use h5py for this (or PyTables...)
# Always loads entire metadata (not just specific variable) but none of the data
# Data needs to be actually loaded (.value) and transposed (.T)
from h5py import File as HDF5File # TODO: if import error try using PyTables
with HDF5File(
filename, 'r'
) as x: # IOError if it doesn't exist or is the wrong format
if name == None:
try:
name = x.iterkeys().next()
except StopIteration:
raise KeyError() # no variables
return x[name].value.T # can raise key error
def readmat(filename):
s = sio.loadmat(filename)
info = sio.whosmat(filename)
ss = s[(str(info[0][0]))]
wavenumber = ss[:, 0]
sizex = len(wavenumber)
sizemx, sizemy = np.shape(ss)
sp = ss[:, 1:]
if (len(info)) > 1:
(l, *_) = s['wh']
w, h = l
else:
w = int(np.sqrt(sp.shape[1]))
h = sp.shape[1] // w
if w * h != sp.shape[1]:
w = sp.shape[1]
h = 1
p = sp.reshape(sizex, h, w, order='C')
return w, h, p, wavenumber, sp
def loadRawFile(self, rfile):
''' Load raw data time serie from '.mat' or '.dat'.
Recomendation: Use 'int16' types to save space.
'''
if rfile:
try:
if '.mat' in rfile:
name = sio.whosmat(rfile)[0][0]
self.raw = sio.loadmat(rfile)['data'][0]
elif '.dat' in rfile:
self.raw = np.fromfile(rfile, dtype='int16')
else:
print('Support only ".mat" or ".dat" files')
except:
print("Couldn't load data from '%s' file" % rfile)
else:
self.raw = list()
def output_to_text(user_settings, output_settings, m_dir):
import os
# user_settings : path to csv of user settings e.g. config/settings/default.csv
# output_settings : path to csv of output settings e.g. config/settings/output_settings.csv
# output_dir : path to *.mat files
user = csv_to_dict(user_settings)
ott = csv_to_dict(output_settings)
assert ott.has_key('singleFilename')
# outfile = ott['singleFilename']
outfile = "out.txt"
matfiles = [m_dir+"/"+f for f in os.listdir(m_dir) if f.endswith(".mat")]
test = matfiles[0]
fields = [x[0] for x in sio.whosmat(test)]
print fields
# print test
# d = sio.loadmat(test)
data = sio.loadmat(test)
print type(data)
delim = ","
def mwhos(*args, **kwargs):
"""
mfile.mwhos: Displays contents of a matfile
See the documentation for scipy.io.whosmat. This is just a wrapper for that.
Example:
In [1]: mfile.mwhos('file.mat')
Contents of file.mat:
('y', (1, 10), 'double')
('x', (1, 10), 'double')
2014.02.28 -- A. Manalaysay
"""
#from scipy import io
print("\n Contents of {:s}:\n".format(args[0]))
for a in io.whosmat(*args, **kwargs):
print('\t', a)
def load_mat(filename, key='feat'):
if os.path.exists(filename):
key = sio.whosmat(filename)[0][0]
##print('key = %s' % key)
data = sio.loadmat(filename)
if key in data:
mat = data[key]
else:
return None
if mat.shape[0] > 1:
mat = mat.reshape(mat.shape[0])
elif mat.shape[1] > 1:
mat = mat.reshape(mat.shape[1])
if np.isnan(mat).sum() > 0:
return None
return mat
else:
print('load_mat {} failed'.format(filename))
return None
def print_meta(self):
'''
dump file metadata, also perform some simple assertions
misc: filename seems to be {DATE}_R{SAMPLENO}.mat
'''
print('---')
print('fname: ' + str(self.fname))
# [('name', (x,y), 'type'), ...]
print('whosmat: "' + str(sio.whosmat(self.fname)) + '"')
print('experimenter: ' + str(self.experimenter))
print('description: ' + str(self.description))
print('date: ' + str(self.date))
print('sample no: ' + str(self.sampleno))
print('animal: ' + str(self.animal))
print('ncell: ' + str(self.ncell))
# spikes[cell][expno][spike]
try:
assert(len(self.spikes) == self.ncell)
except AssertionError:
print("error - data['spikes'] " + str(len(self.spikes))
+ ' : ' + str(self.ncell))
raise
print('stimulus: ')
assert(len(self.stimulus) == len(self.recno))
for s in self.stimulus:
print(' - ')
for k in s:
print(' ' + str(k) + ' : ' + str(s[k]))
print('records: # nrecs: ' + str(len(self.recno)))
for i in range(len(self.recno)):
print(' -')
print(' record: ' + str(self.recno[i]))
print(' start: ' + str(self.recstart[i]))
print(' spikes: ')
for j in range(len(self.spikes)):
print(' - ' + str(len(self.spikes[j][i])))
def write(self):
mat_vars=sio.whosmat(self.fileo)
if not set(['aa', 'trans']).isdisjoint([item for sublist in mat_vars for item in sublist]): #tell the user that they might overwrite their data
ret=QtWidgets.QMessageBox.warning(self, "pyCM Warning", \
"There is already data associated with this analysis step saved. Overwrite and invalidate subsequent steps?", \
QtWidgets.QMessageBox.Yes | QtWidgets.QMessageBox.No, QtWidgets.QMessageBox.No)
if ret == QtWidgets.QMessageBox.No: #don't overwrite
return
else:
#delete fitting parameters with pyCMcommon helper function, which negates FEA pre-processing as well.
clear_mat(self.fileo,['x_out','aa_mask','spline_x'])
mat_contents=sio.loadmat(self.fileo)
new={'trans': {'ref':self.refTrans, 'float':self.floatTrans},'aa': {'pnts': self.ap, 'gsize': self.gsize}}
mat_contents.update(new) #update the dictionary
sio.savemat(self.fileo,mat_contents)
self.ui.statLabel.setText("Wrote data.")
self.unsaved_changes=False
def imread_mat(filename, name = None):
"""
Read an 'image' from a MATLAB .MAT file. The file can be any version. Files
that are v7.3 require the h5py module. If no name is given, the first
variable is taken.
"""
try:
# Try general first (doesn't work for v7.3+ files)
# SciPy has this built in
# Supports loading just the given variable name
# Otherwise have to load all variables and skip special keys starting with "__" to find the variable to load
# Loaded matracies are already arrays
from scipy.io import loadmat
if name == None:
try:
# Try to get first variable name without loading entire file (only supported in SciPy 0.12+)
from scipy.io import whosmat
keys in whosmat(file_name)
if len(keys) == 0: raise KeyError()
name = keys[0][0]
except: pass
x = loadmat(filename, variable_names = name)
if name == None:
name = '__' # we need to find first
for name in x.keys():
if name[:2] != '__': break
if name[:2] == '__': raise KeyError() # no variables
return x[name] # can raise key error
except NotImplementedError:
# Try v7.3 file which is an HDF5 file
# We have to use h5py for this (or PyTables...)
# Always loads entire metadata (not just specific variable) but none of the data
# Data needs to be actually loaded (.value) and transposed (.T)
from h5py import File as HDF5File # TODO: if import error try using PyTables
with HDF5File(filename, 'r') as x: # IOError if it doesn't exist or is the wrong format
if name == None:
try: name = next(iter(x.keys()))
except StopIteration: raise KeyError() # no variables
return x[name].value.T # can raise key error
def _load_doc_term_data_from_mat_(path):
tmp_dict = {key: None for key in [key_name_data,
key_name_term_labels,
key_name_doc_labels]}
# Get the fields from the matlab file
matlab_content = whosmat(path)
for index, element in enumerate(matlab_content):
# if co-occurence data, load
if element[0] == key_name_data:
tmp_dict[key_name_data] = loadmat(path)[key_name_data]
# if term label data, load and convert to list
elif element[0] == key_name_term_labels:
tmp_dict[key_name_term_labels] = loadmat(path)[key_name_term_labels]
tmp_dict[key_name_term_labels] = tmp_dict[key_name_term_labels].tolist()
# if doc label data, load, convert to list and take list inside...
elif element[0] == key_name_doc_labels:
tmp_dict[key_name_doc_labels] = loadmat(path)[key_name_doc_labels]
tmp_dict[key_name_doc_labels] = tmp_dict[key_name_doc_labels].tolist()[0]
return tmp_dict
def loadFromMAT(self):
matinfos = whosmat(str(self.filename))
matfile = loadmat(str(self.filename))
self.nbRows = 30#5
self.nbCols = matinfos[0][1][1]
self.table.setRowCount(self.nbRows)
self.table.setColumnCount(self.nbCols)
for val in matfile.values():
if isinstance(val, np.ndarray):
mdata = val
for j in xrange(self.nbCols):
headerLabel = str(mdata[0 ,j])
item = QTableWidgetItem(headerLabel)
item.setFlags(QtCore.Qt.ItemIsSelectable | QtCore.Qt.ItemIsEditable)
self.table.setHorizontalHeaderItem(j, item)
for i in xrange(self.nbRows):
for j in xrange(self.nbCols):
item = QTableWidgetItem(str(mdata[i ,j]))
item.setFlags(QtCore.Qt.ItemIsSelectable)
self.table.setItem(i, j, item)
def sciIOtest():
cifar10_dir = "cifar-10-batches-mat"
filePath = os.path.join(cifar10_dir,'test_batch')
try:
dataLables = sio.whosmat(filePath)
datadict = sio.loadmat(filePath)
except BaseException as err:
print(err)
#print("datadict data type"+str(type(datadict)))
print(datadict)
print("Mat File List variables :")
print(dataLables)
print("datadict size " + str(len(datadict)))
print("/////////////////////////////////////////////////")
X = datadict['data']
print("X data type:" + str(type(X)))
print(X)
X = numpy.reshape(X,(10000,3,32,32))
print("X shape : " + str(numpy.shape(X)))
Y = datadict['labels']
print(Y)
print("Y shape " + str(numpy.shape(Y)))
print("Y ends")
def who(archmat):
variables=whosmat(archmat)
for ind in range(len(variables)):
print(variables[ind])
from scipy.io import loadmat, savemat, whosmat
print(whosmat('octave_struct.mat'))
def loadnaive(mfile):
mat = loadmat(mfile)
my_struct = mat['my_struct'][0, 0]
print(my_struct.dtype)
print(my_struct['field1'][0, 0])
print(my_struct['field2'][0, 0])
def load_with_squeeze_me(mfile):
mat = loadmat(mfile, squeeze_me=True)
my_struct = mat['my_struct']
print(my_struct.dtype)
print(my_struct['field1'])
print(my_struct['field2'])
def load_with_struct_as_record_false(mfile):
mat = loadmat(mfile, struct_as_record=False, squeeze_me=True)
my_struct = mat['my_struct']
print(my_struct.field1)
print(my_struct.field2)
mat = loadmat(mfile, struct_as_record=False)
my_struct = mat['my_struct'][0, 0]
print(my_struct.field1[0, 0])
print(my_struct.field2[0, 0])
import numpy as np
from scipy.io import loadmat, savemat, whosmat
vect = np.arange(10)
print(vect.shape)
savemat('np_vector.mat', {'vect': vect})
mat = loadmat('np_vector.mat')
print(whosmat('np_vector.mat'))
print(mat['vect'].shape)
savemat(filename, {"var1": x})
# load data
data = loadmat(filename)
print(data["var1"])
filename = os.path.join(f_dir, "example2.mat")
x = np.arange(4).reshape(2, 2)
y = np.sin(x)
# save data
savemat(filename, {"var1": x, "y": y})
# show variables
print(whosmat(filename))
# load data
data = loadmat(filename)
print(data["y"])
import netCDF4
filename = os.path.join(f_dir, "example1.nc")
f = netCDF4.Dataset(filename, "w", format="NETCDF4")
grp_temp = f.createGroup("temperature")
subgrp_air = grp_temp.createGroup("air")
subgrp_soil = grp_temp.createGroup("soil")
def trial_selected(self, event):
widget = event.widget
self.selected_trial = widget.get(widget.curselection())
self.trial_text.configure(text='Trial: ' + str(self.selected_trial))
self.image_panel.get_tk_widget().delete('all')
self.image_panel.get_tk_widget().grid_remove()
self.tk.call(self.image_panel.get_tk_widget(), 'delete', 1, 1)
del self.figure
del self.image_panel
self.figure = Figure(figsize=(9, 6), dpi=100)
self.image_panel = FigureCanvasTkAgg(self.figure, self)
self.a = self.figure.add_subplot(111)
if self.selected_trial is not None:
if str(str(self.file) + " " + str(self.selected_trial)) not in self.mat.keys():
try:
self.mat[str(self.file) + " " + str(self.selected_trial)] = sc_io.loadmat(self.file, appendmat=True)
except Exception as e:
try:
self.mat[str(self.file) + " " + str(self.selected_trial)] = sc_io.loadmat(self.file, appendmat=True)
except FileNotFoundError:
print("File not found: ", e)
temp_var = sc_io.whosmat(self.file)[0][0]
for x in self.mat[str(self.file) + " " + str(self.selected_trial)]['StimTrig'][0][0][4]:
self.stimuli_time.append(x[0])
for x in self.mat[str(self.file) + " " + str(self.selected_trial)]['StimTrig'][0][0][5]:
if x[0] != 62:
self.stimuli_code.append(x[0])
else:
self.stimuli_code.append(0)
self.number_trials += 1
for x in self.mat[str(self.file) + " " + str(self.selected_trial)][temp_var][0][0][4]:
self.firing.append(x[0])
counter = 1
for i, x in enumerate(self.stimuli_code):
if x == 0:
self.dictionary[counter] = i
counter += 1
temp_list = []
temp_key = 1
for x in self.firing:
try:
if x <= self.stimuli_time[self.dictionary[temp_key]]:
temp_list.append(x)
else:
self.trialled_firing.append(temp_list)
temp_list = []
temp_key += 1
except KeyError:
break
index = self.selected_trial
if index == "1":
self.a.cla()
for x in self.trialled_firing[int(index)-1]:
self.a.plot([x, x], [0, 10], "r-")
self.a.plot(self.stimuli_time[0:self.dictionary[1]+1],
self.stimuli_code[0:self.dictionary[1]+1], 'ko', ms=6)
for i, x in enumerate(self.stimuli_code[0:self.dictionary[1]+1]):
self.a.annotate(s=str(x), xy=(self.stimuli_time[i], x), xytext=(self.stimuli_time[i], 10.2), color='0.2', size=13, weight="bold")
self.a.axis(xmin=0, xmax=self.stimuli_time[self.dictionary[1]])
self.a.set_xlabel("Time (s)")
self.a.set_ylabel("Amplitude of Stimuli")
else:
self.a.cla()
for x in self.trialled_firing[int(index)-1]:
self.a.plot([x, x], [0, 10], "r-")
self.a.plot(self.stimuli_time[self.dictionary[int(index)-1]:self.dictionary[int(index)]+1],
self.stimuli_code[self.dictionary[int(index)-1]:self.dictionary[int(index)]+1],
'ko', ms=6)
for i, x in enumerate(self.stimuli_code[self.dictionary[int(index)-1]:self.dictionary[int(index)]+1]):
self.a.annotate(s=str(x), xy=(self.stimuli_time[i+self.dictionary[int(index)-1]], x),
xytext=(self.stimuli_time[i+self.dictionary[int(index)-1]], 10.2),
color='0.2', size=13, weight="bold")
self.a.axis(xmin=self.stimuli_time[self.dictionary[int(index)-1]],
xmax=self.stimuli_time[self.dictionary[int(index)]])
self.a.set_xlabel("Time (s)")
self.a.set_ylabel("Amplitude of Stimuli")
self.image_panel.get_tk_widget().grid(row=1, column=2, sticky='nsew', padx=0, pady=0)
self.image_panel.get_tk_widget().configure(highlightthickness=0, relief='groove', borderwidth=0)
