def setup_module():
temp_dir = None
try:
import scipy.io
temp_dir = tempfile.mkdtemp()
for i in range(0, len(mat_files)):
mat_files[i] = os.path.join(temp_dir, mat_files[i])
scipy.io.savemat(file_name=mat_files[0], mdict=to_julia)
hdf5storage.savemat(file_name=mat_files[1], mdict=to_julia)
#julia_command(script_names[0], mat_files[0], mat_files[2])
julia_command(script_names[0], mat_files[1], mat_files[3])
#hdf5storage.loadmat(file_name=mat_files[2],
# mdict=from_julia_v7_to_v7p3)
hdf5storage.loadmat(file_name=mat_files[3],
mdict=from_julia_v7p3_to_v7p3)
except:
pass
else:
ran_julia_successful[0] = True
finally:
for name in mat_files:
if os.path.exists(name):
os.remove(name)
if temp_dir is not None and os.path.exists(temp_dir):
os.rmdir(temp_dir)
def setup_module():
teardown_module()
matlab_command = "run('" + script_names[0] + "')"
subprocess.check_call(['matlab', '-nosplash', '-nodesktop',
'-nojvm', '-r', matlab_command])
scipy.io.loadmat(file_name=mat_files[1], mdict=types_v7)
hdf5storage.loadmat(file_name=mat_files[0], mdict=types_v7p3)
hdf5storage.savemat(file_name=mat_files[2], mdict=types_v7p3)
matlab_command = "run('" + script_names[1] + "')"
subprocess.check_call(['matlab', '-nosplash', '-nodesktop',
'-nojvm', '-r', matlab_command])
scipy.io.loadmat(file_name=mat_files[3], mdict=python_v7)
hdf5storage.loadmat(file_name=mat_files[2], mdict=python_v7p3)
def load_matlab(self, filename, key='results'):
import hdf5storage
data = hdf5storage.loadmat(filename)[key]
expected_shape = (len(self.stimuli), 1)
if not data.shape == expected_shape:
raise ValueError('Wrong number of saliency maps! Expected {}, got {}'.format(expected_shape, data.shape))
self._saliency_maps = [data[i, 0] for i in range(len(self.stimuli))]
def loaddata(self, matfilename=None, picklefilename=None):
"""
Loads data from saved files. If matfilename is not None, all numerical data is read from ithe file (but not
the solver objects). If picklefilename is not None, solver objects are read from the file.
:param matfilename:
:param picklefilename:
:return:
"""
if picklefilename is not None:
with open(picklefilename, "r") as f:
solvers = cPickle.load(f)
self.set_solvers(solvers)
if matfilename is not None:
mdict = hdf5storage.loadmat(matfilename)
self.signaldim = mdict[u'signaldim']
self.dictdim = mdict[u'dictdim']
self.numdata = mdict[u'numdata']
self.deltas = mdict[u'deltas'].copy()
self.rhos = mdict[u'rhos'].copy()
if mdict[u'err'] is not None:
self.err = mdict[u'err'].copy()
if mdict[u'ERCsuccess'] is not None:
self.ERCsuccess = mdict[u'ERCsuccess'].copy()
if u'simData' in mdict.keys():
self.simData = mdict[u'simData']
def loadmat(cls, file_name, path='/', **kwargs):
"""Load from a Matlab (HDF5 format) file
:param file_name: file name
:param path: path toread data from
:param kwargs: keyword passed to hdf5storage.loadmat
"""
return cls(hdf5storage.loadmat(file_name, marshaller_collection=cls.__mc()), **kwargs)
def load_matlab(self, filename, key='results'):
import hdf5storage
data = hdf5storage.loadmat(filename)[key]
if len(data.shape) == 2 and len(self.stimuli) > 1:
if data.shape[0] == 1:
data = data[0]
elif data.shape[1] == 1:
data = data[:, 0]
else:
raise ValueError('Data has wrong shape: {} (need 1xN, Nx1 or N)'.format(data.shape))
if len(data.shape) > 2:
raise ValueError('Data has wrong shape: {} (need 1xN, Nx1 or N)'.format(data.shape))
expected_shape = (len(self.stimuli),)
if not data.shape == expected_shape:
raise ValueError('Wrong number of saliency maps! Expected {}, got {}'.format(expected_shape, data.shape))
self._saliency_maps = [data[i] for i in range(len(self.stimuli))]
def main(argv):
assert len(argv) == 2, 'Usage: source_dir output_dir'
source_dir = argv[0]
output_dir = argv[1]
pathes = glob.glob(source_dir + "/*.mat")
for mat_path in pathes:
print 'Converting {}'.format(mat_path)
mat = hdf5storage.loadmat(mat_path)
attrs, datasets = parse(mat)
name = os.path.splitext(os.path.basename(mat_path))[0]
f = h5py.File(os.path.join(output_dir, name + '.hdf5'), 'w')
for key, val in attrs.items():
f.attrs[key] = val
for key, val in datasets.iteritems():
f.create_dataset(key, data=val)
def forest():
print(session['data'])
if session['data'].split('.')[-1] == 'csv':
data = pd.read_csv(session['data'])
elif session['data'].split('.')[-1] == 'mat':
mat = hd.loadmat(session['data'])
X = pd.DataFrame(mat['X'])
Y = pd.DataFrame(mat['y'])
Y.rename(columns={0: 'CLASS'}, inplace=True)
data = pd.concat([X, Y], axis=1)
global filename
if os.path.isfile(os.path.join('static/img/', filename)):
os.remove(os.path.join('static/img/', filename))
if request.method == 'GET':
return render_template('forest.html',
column=data.columns,
data=session['data'].split('/')[-1])
else:
kelas = request.form['kelas']
normal = request.form['normal']
abnormal = request.form['abnormal']
t_size = float(request.form['tsize']) / 100
cont = float(request.form['cont']) / 100
n_tree = int(request.form['tree'])
samples = int(request.form['sample'])
# Normal Abnormal
dt_normal = data.loc[data[kelas] == int(normal)]
dt_abnormal = data.loc[data[kelas] == int(abnormal)]
# print(dt_normal)
# Split Data
normal_train, normal_test = train_test_split(dt_normal,
test_size=t_size,
random_state=42)
abnormal_train, abnormal_test = train_test_split(dt_abnormal,
test_size=t_size,
random_state=42)
train = pd.concat([normal_train, abnormal_train])
test = pd.concat([normal_test, abnormal_test])
# train[kelas] = train[kelas].map({0:1,1:-1})
data[kelas] = data[kelas].map({0: 1, 1: -1})
test[kelas] = test[kelas].map({0: 1, 1: -1})
# Model
model = IsolationForest(n_estimators=n_tree,
contamination=cont,
max_samples=samples,
random_state=100)
pred = model.fit_predict(data.drop([kelas], axis=1))
# print(pred)
# # Model 2
model2 = IsolationForest(n_estimators=n_tree,
contamination=cont,
max_samples=samples,
random_state=100)
model2.fit(train.drop([kelas], axis=1))
pred2 = model2.predict(test.drop([kelas], axis=1))
# # Hasil Prediksi
dt = data.drop([kelas], axis=1)
df_pred = pd.DataFrame(pred)
df_pred.columns = [kelas]
dt = pd.concat([dt, df_pred], axis=1)
# Confusion Matrix
cm = confusion_matrix(data[kelas], pred)
df_cm = pd.DataFrame(cm, ['Anomali', 'Normal'],
['Prediksi Anomali', 'Prediksi Normal'])
plt.figure(figsize=(6, 4))
sns.set(font_scale=1.2)
sns.heatmap(df_cm, annot=True, annot_kws={'size': 16}, fmt='g')
# Random filename
filename = ''.join(
random.choices(string.ascii_uppercase + string.digits, k=5))
filename = filename + '.png'
# print(filename)
plt.savefig(os.path.join('static/img/', filename))
plt.clf()
# Confusion Matrix 2
cm2 = confusion_matrix(test[kelas], pred2)
df_cm2 = pd.DataFrame(cm2, ['Anomali', 'Normal'],
['Prediksi Anomali', 'Prediksi Normal'])
plt.figure(figsize=(6, 4))
sns.set(font_scale=1.2)
sns.heatmap(df_cm2, annot=True, annot_kws={'size': 16}, fmt='g')
# Random filename
filename2 = ''.join(
random.choices(string.ascii_uppercase + string.digits, k=5))
filename2 = filename2 + '.png'
# print(filename)
plt.savefig(os.path.join('static/imgv2/', filename2))
# Convert to Dataframe
df_pred = pd.DataFrame(pred)
df_pred2 = pd.DataFrame(pred2)
# Metrik Evaluasi 1
f1_s = f1_score(data[kelas], df_pred, average='weighted')
ps = precision_score(data[kelas], df_pred, average='weighted')
rs = recall_score(data[kelas], df_pred, average='weighted')
# Metrik Evaluasi 2
f1_s1 = f1_score(test[kelas], df_pred2, average='weighted')
ps1 = precision_score(test[kelas], df_pred2, average='weighted')
rs1 = recall_score(test[kelas], df_pred2, average='weighted')
# tp, fp, fn, tn = cm.ravel()
# rec=tp/(tp+fn)
rs = round(rs, 3)
rs1 = round(rs1, 3)
# prec=tp/(tp+fp)
ps = round(ps, 3)
ps1 = round(ps1, 3)
# f1_score=2*((prec*rec)/(prec+rec))
f1_s = round(f1_s, 3)
f1_s1 = round(f1_s1, 3)
filepath = 'img/' + filename
filepath2 = 'imgv2/' + filename2
# print(session['data'].split('.')[:-1].join())
print(session['data'].split('.'))
return render_template('forest.html',
tsize=int(t_size * 100),
cont=float(cont * 100),
tree=n_tree,
sample=samples,
dt=dt.values,
column=dt.columns,
cm=df_cm,
ps=ps,
rs=rs,
f1_s=f1_s,
ps1=ps1,
rs1=rs1,
f1_s1=f1_s1,
kelas=kelas,
filepath=filepath,
filepath2=filepath2,
data=session['data'].split('/')[-1])
name_base= DATA_DIR + '/EVs/' + strategy + suffix + '/'
#names = glob.glob(name_base + "\*.mat")
data = {}
file_name = DATA_DIR + '/EVs/'+ strategy + '_' + str(n) + '.mat'
try:
os.remove(file_name)
except:
pass
for i in xrange(1, n + 1):
name= name_base + str(i) + '.mat'
data = {}
data["ev_" + str(i)] = hdf5storage.loadmat(name)
hdf5storage.savemat(file_name, data)
print "saved file " + str(i)
#try:
# if(DELETE):
# os.remove(name)
#except:
# pass
'''
name= name_base + str(i) + '.mat'
with h5py.File(name, 'r') as mat_file:
#f.create_group(str(i))
h5py.h5o.copy(mat_file.id, mat_file.name, f.id, str(i))
# In[9]:
vr = 'R0'
# # Load the AOD files
#
# ## Load the AOD files from 4STAR
# In[10]:
ar = hs.loadmat(fp+'/aod_ict/all_aod_KORUS_ict.mat')
# In[11]:
ar.keys()
# ## Adjust the AOD to reflect dirt contamination
# In[12]:
aod_names = sorted([a for a in ar.keys() if ('AOD' in a) and not ('UNC' in a)])
#import caffe
#sys.path.append('/home/tzeng/caffe_3d/python/caffe')
#foo = imp.load_source('caffe.io', '/home/tzeng/caffe_3d/python/caffe/__init__.py')
import caffe.io
from caffe.proto import caffe_pb2
print os.path.dirname(caffe_pb2.__file__)
#from caffe.proto import caffe_pb2
mat_file ='/home/tzeng/space/SegEM_project/data/train_segem.mat'
#mat_file ='/home/tzeng/caffe_flx_kernel/data/snems3d_train_RF8.mat'
#mat_file ='/home/tzeng/caffe_flx_kernel/data/snems3d_test_pad_2_47_47.mat'
#mat_file= '/home/tzeng/caffe_flx_kernel/data/snems3d_train_RF8_20Percent.mat'
#mat_file= '/home/tzeng/caffe_flx_kernel/data/snems3d_predict_norm.mat'
#snems3d_train_pad_4_47_47.mat'
#mat_file ='/home/tzeng/caffe_3d/data/snems3d_test_pad25.mat'
#mat_file ='/home/tzeng/caffe_3d/data/test'
out =hdf5storage.loadmat(mat_file,format='7.3')
#print len(out)
size = out['data'].shape;
size=size[1];
print size
k=1
#db_path_data='/home/tzeng/caffe_3d/data/mri_test_pad'
db_path_data='/home/tzeng/space/SegEM_project/data/train_segem'
#db_path_data='/home/tzeng/caffe_3d/data/snems3d_train_pad25'
#db_path_data='/home/tzeng/caffe_flx_kernel/data/snems3d_train_pad_4_47_47_rotations_hFlip'
#db_path_data='/home/tzeng/caffe_flx_kernel/data/snems3d_predict_norm'
#db_path_data='/home/tzeng/caffe_flx_kernel/data/snems3d_test_norm'
#db_path_data='/home/tzeng/caffe_flx_kernel/data/snems3d_test_pad_2_47_47_FlipRatation'
#snems3d_test_submit_pad25'
db_data_lb=leveldb.LevelDB(db_path_data, create_if_missing=True, error_if_exists=False)
batch = leveldb.WriteBatch()
def main():
NUM_FEATURES = 21 # number of RGB channel features to use for classifying each pixel
dtype = torch.float
#device = torch.device("cpu")
device = torch.device("cuda:0") # Uncomment this to run on GPU
# Load training data
print('Load training data...')
# filepath = "C:/Users/CTLab/Documents/George/Arri_analysis_4-29-19/kmeans_data_5-6-19.mat"
# if 'mat2' in locals(): print('Yes')
# mat2 = hdf5storage.loadmat(filepath)
# x = torch.from_numpy(mat2['X_single'][:,:NUM_FEATURES]) # make 24 -> 21 features as needed
# y = torch.from_numpy(mat2['y_label']).long()-1 # convert to long tensor for loss function
# y = y.squeeze(1)
x, y = load_trainingdata()
m_training = x.size()[0]
num_features = x.size()[1]
num_classes = np.size(np.unique(y))
classes = [
"Artery", "Bone", "Cartilage", "Dura", "Fascia", "Fat", "Muscle",
"Nerve", "Skin", "Parotid", "PerichondriumWCartilage", "Vein"
]
#%%Load validation data
print('Load validation data...')
filepath_val = "C:/Users/CTLab/Documents/George/Arri_analysis_5-6-19/FreshCadaver004_20190429_data_5-6-19_pyfriendly.mat"
mat3 = hdf5storage.loadmat(filepath_val)
x_val = torch.from_numpy(mat3['X'][:, :NUM_FEATURES]).float(
) # make 24 -> 21 features as needed; convert to float tensor
y_val = torch.from_numpy(
mat3['y_label']).long() - 1 # convert to long tensor for loss function
y_val = y_val.squeeze(1)
m_validation = x_val.size()[0]
#%% Train network
print('Initialize model...')
# N is batch size; D_in is input dimension;
# H is hidden dimension; D_out is output dimension.
D_in, H, D_out = NUM_FEATURES, 100, num_classes
## Create random Tensors to hold inputs and outputs
#x = torch.randn(N, D_in)
#y = torch.randn(N, D_out)
# Use the nn package to define our model and loss function.
model = torch.nn.Sequential(
torch.nn.Linear(D_in, H),
torch.nn.ReLU(),
torch.nn.Linear(H, H),
torch.nn.ReLU(),
torch.nn.Linear(H, H),
torch.nn.ReLU(),
torch.nn.Linear(H, H),
torch.nn.ReLU(),
torch.nn.Linear(H, D_out),
torch.nn.Softmax(
dim=1
) # hidden activations are of shape m x D_out, where output features per training example are in 2nd dimension (along rows)
)
model = model.cuda()
#loss_fn = torch.nn.MSELoss(reduction='sum')
loss_fn = torch.nn.CrossEntropyLoss(
weight=None,
size_average=None,
ignore_index=-100,
reduce=None,
reduction='mean'
) # input has to be a Tensor of size either (minibatch, C)(minibatch,C)
# Use the optim package to define an Optimizer that will update the weights of
# the model for us. Here we will use Adam; the optim package contains many other
# optimization algoriths. The first argument to the Adam constructor tells the
# optimizer which Tensors it should update.
learning_rate = 1e-4
optimizer = torch.optim.Adam(model.parameters(), lr=learning_rate)
#for t in range(500):
# # Forward pass: compute predicted y by passing x to the model.
# y_pred = model(x)
#
# # Compute and print loss.
# loss = loss_fn(y_pred, y)
# print(t, loss.item())
#
# # Before the backward pass, use the optimizer object to zero all of the
# # gradients for the variables it will update (which are the learnable
# # weights of the model). This is because by default, gradients are
# # accumulated in buffers( i.e, not overwritten) whenever .backward()
# # is called. Checkout docs of torch.autograd.backward for more details.
# optimizer.zero_grad()
#
# # Backward pass: compute gradient of the loss with respect to model
# # parameters
# loss.backward()
#
# # Calling the step function on an Optimizer makes an update to its
# # parameters
# optimizer.step()
n_epochs = 100 # or whatever
batch_size = 128000 # or whatever
print('Start training model...')
cache_loss = []
# pushing tensors to CUDA device if available (you have to reassign them)
x = x.to(device)
y = y.to(device)
# pushing tensors to CUDA device if available (you have to reassign them)
x_val_gpu = x_val.to(device)
y_val_gpu = y_val.to(device)
for epoch in range(n_epochs):
# Store training and validation output scores to calculate accuracy of model after each epoch
num_train_batches = int(np.ceil(m_training / batch_size))
num_validation_batches = int(np.ceil(m_validation / batch_size))
cache_training_acc = np.zeros(num_train_batches) # numpy rank-1 array
cache_validation_acc = np.zeros(
num_validation_batches) # numpy rank-1 array
# x is a torch Variable
permutation = torch.randperm(m_training)
for i in range(0, m_training, batch_size):
optimizer.zero_grad()
indices = permutation[i:i + batch_size]
batch_x, batch_y = x[indices], y[
indices] # includes last mini-batch even if its size < batch_size
# # pushing tensors to CUDA device if available (you have to reassign them)
# batch_x = batch_x.to(device)
# batch_y = batch_y.to(device)
# Compute and print loss.
outputs = model(batch_x)
loss = loss_fn(outputs, batch_y)
# if np.mod(i/batch_size, 50) == 0:
# print('Epoch:', epoch, ' Batch ', i/batch_size, 'out of', np.floor(x.size()[0]/batch_size), ' Loss:', loss.item())
# # Cache outputs to calculate training accuracy at end of epoch
pred_training_labels = scores_to_labels(outputs.detach().cpu())
cache_training_acc[int(i / batch_size)] = accuracy(
batch_y.cpu().numpy(), pred_training_labels.numpy())
# After each epoch, save training and validation losses to plot learning curve
if (m_training - i) <= batch_size: # last mini-batch of epoch
# empty CUDA cache every epoch
torch.cuda.empty_cache()
with torch.no_grad():
model.eval()
# Cycle through mini-batches of validation data to calculate mean loss
permutation2 = torch.randperm(m_validation)
val_loss = []
for j in range(0, m_validation, batch_size):
indices_val = permutation2[j:j + batch_size]
batch_x_val, batch_y_val = x_val_gpu[
indices_val], y_val_gpu[indices_val]
# batch_x_val, batch_y_val = x_val[indices_val], y_val[indices_val]
# # pushing tensors to CUDA device if available (you have to reassign them)
# batch_x_val_gpu = batch_x_val.to(device)
# batch_y_val_gpu = batch_y_val.to(device)
# out = model(batch_x_val_gpu)
# loss_val = loss_fn(out, batch_y_val_gpu)
out = model(batch_x_val)
loss_val = loss_fn(out, batch_y_val)
val_loss.append(loss_val.item())
# Cache outputs to calculate validation accuracy at end of epoch
pred_validation_labels = scores_to_labels(
out.detach().cpu())
cache_validation_acc[int(j / batch_size)] = accuracy(
batch_y_val.cpu().numpy(),
pred_validation_labels.numpy())
# cache_pred_validation[j:j+batch_size] = scores_to_labels(out.cpu().detach().numpy())
# Calculate mean loss across validation mini-batches -- **NOTE does not take weighted mean of mini-batches' losses if last mini-batch's size is smaller
mean_val_loss = np.mean(val_loss)
acc_train = np.mean(cache_training_acc)
acc_val = np.mean(cache_validation_acc)
# # Calculate training and validation accuracy
## train_pred = scores_to_labels(cache_out_training)
# acc_train = accuracy(y.numpy(), cache_pred_training)
#
## val_pred = scores_to_labels(cache_out_validation)
# acc_val = accuracy(y_val.numpy(), cache_pred_validation)
# Print trainng and validation losses
print('Epoch:', epoch, ' Batch ',
i / batch_size, 'out of',
np.floor(m_training / batch_size), ' Loss:',
loss.item(), 'Val loss:', mean_val_loss,
' Train acc:', acc_train, 'Val acc:', acc_val)
# print('Epoch:', epoch, ' Batch ', i/batch_size, 'out of', np.floor(m_training/batch_size), ' Loss:', loss.item())
# Save losses to plot learning curve
cache_loss.append((epoch, loss.item(), mean_val_loss,
acc_train, acc_val))
# reset model to training mode for next epoch
model.train()
# For other mini-batches, just print training loss every nth mini-batch
elif np.mod(i / batch_size, 50) == 0:
print('Epoch:', epoch, ' Batch ', i / batch_size, 'out of',
np.floor(m_training / batch_size), ' Loss:',
loss.item())
loss.backward()
optimizer.step()
#%% Save cached variables to analyze training / learning curve
with open('arriANN_train_loss.pkl', 'wb') as ff:
pickle.dump(cache_loss, ff)
# Save trained model's parameters for inference
PATH = "C:/Users/CTLab/Documents/George/Arri_analysis_5-6-19/arrinet_ann_model_parameters_5-8-19.pt"
torch.save(model.state_dict(), PATH)
print('Done')
#%% Predict on validation set at end
model.eval()
y_pred = predict(model, num_classes, batch_size, device, x_val)
y_val_true = y_val.numpy()
print('Computing confusion matrix...')
conf = confusion_matrix(y_val_true, y_pred)
category=DeprecationWarning)
grupos = 5
np.random.seed(32)
#n_est = 100
lista_bases = ['Lymphoma']
lista_fam = ['S', 'T', 'K', 'D']
tiempo = np.zeros(shape=(1))
for indice_lst in lista_bases:
for indice_fam in lista_fam:
nombre_base = indice_lst + '_Caracteristicas' + indice_fam + '.mat'
Datos = hdf5storage.loadmat(nombre_base)
if indice_fam == 'S':
Caracteristicas = Datos['CaracteristicasS']
elif indice_fam == 'T':
Caracteristicas = Datos['CaracteristicasT']
elif indice_fam == 'K':
Caracteristicas = Datos['CaracteristicasK']
else:
Caracteristicas = Datos['CaracteristicasD']
etiqueta = Datos['etiquetas']
etiquetas = etiqueta.reshape((etiqueta.shape[0]))
Resultados = np.zeros(shape=(4, 5))
ResultadosSTD = np.zeros(shape=(4, 5))
sid = int(float(sys.argv[1]))
else: # debug in IDE
sid = 10
tf_dir = data_dir + '/tfAnalysis/P' + str(sid) + '/'
input_dir = data_dir + 'preprocessing/P' + str(sid) + '/preprocessing2.mat'
if not os.path.exists(tf_dir):
os.makedirs(tf_dir)
Session_num, UseChn, EmgChn, TrigChn = get_channel_setting(sid)
#original_fs=[Frequencies[i,1] for i in range(Frequencies.shape[0]) if Frequencies[i,0] == pn][0]
fs = 1000
if sid == 11 or sid == 12:
fs = 500
mat = hdf5storage.loadmat(input_dir)
data = mat['Datacell']
channelNum = int(mat['channelNum'][0, 0])
data = np.concatenate((data[0, 0], data[0, 1]), 0)
del mat
# stim0 is trigger channel, stim1 is trigger position calculated from EMG signal.
chn_names = np.append(["seeg"] * channelNum,
["emg0", "emg1", "stim_trigger", "stim_emg"])
chn_types = np.append(["seeg"] * channelNum, ["emg", "emg", "stim", "stim"])
info = mne.create_info(ch_names=list(chn_names),
ch_types=list(chn_types),
sfreq=fs)
raw = mne.io.RawArray(data.transpose(), info)
# gesture/events type: 1,2,3,4,5
##
if False:
mov = '/groups/branson/bransonlab/al/C001H001S0007_c.avi'
win_mat = '/groups/branson/bransonlab/al/C001H001S0007_c.avi.readFrames_1450_allenwin_16b.mat'
py_mat = '/groups/branson/home/kabram/temp/C001H001S0007_c.avi.readFrames_1450_kabram_py.mat'
H = sio.loadmat(win_mat)
nfr = H['I'].shape[0]
ims = np.zeros([512,768,nfr])
for ndx in range(nfr):
ims[:,:,ndx] = H['I'][ndx][0]
import os
if os.path.exists(py_mat):
P = hdf5storage.loadmat(py_mat)
py_ims = P['I']
else:
py_ims = None
## random access
cap = movies.Movie(mov)
dd = []
for ndx in range(10):
curfr = np.random.randint(nfr)
curim = cap.get_frame(curfr)[0][:,:,0]
ff = curim.astype('float64')+1-ims[:,:,curfr]
cur_i = [curfr, np.abs(ff).max()]
if py_ims is not Noneimp:
ffp = curim.astype('float64')-py_ims[:,:,curfr]
cur_i.append(np.abs(ffp).max())
fp ='C:/Users/sleblan2/Research/KORUS-AQ/'
# In[7]:
vr = 'R0'
# # Load files
#
# ## Load the AOD files from 4STAR
# In[8]:
ar = hs.loadmat(fp+'/aod_ict/all_aod_KORUS_ict.mat')
# In[9]:
ar.keys()
# ## Adjust the AOD to reflect dirt contamination
# In[10]:
arc = {}
arc['AOD0501'] = ar['AOD0501']-ar['UNCAOD0501']
action='store_true')
# In[ ]:
in_ = vars(parser.parse_args())
do_read = in_.get('doread',False)
# # Load the saved files
# In[5]:
ar = hs.loadmat(matfile)
# In[6]:
ar.keys()
# In[12]:
dds = ['20160827','20160830','20160831','20160902','20160904','20160906','20160908',
'20160910','20160912','20160914','20160918','20160920','20160924','20160925','20160927']
# In[2]:
hs.__version__
# In[3]:
from mpl_toolkits.basemap import Basemap,cm
# In[4]:
get_ipython().magic(u'matplotlib notebook')
# In[6]:
fp_lut_mat = 'C:\\Users\\sleblan2\\Research\\NAAMES\\lut\\v2_NAAMES_lut.mat'
print('Loading the lut file:{}'.format(fp_lut_mat))
if not os.path.isfile(fp_lut_mat):
print('File {} does not exist'.format(fp_lut_mat))
raise IOError('LUT File not found: {}'.format(fp_lut_mat))
luts = hs.loadmat(fp_lut_mat)
# In[ ]:
# In[53]:
hs.savemat(fp+'/aod_ict/{vv}/all_aod_ict_{vv}.mat'.format(vv=vv),ar)
# ## Optionally load the file
# In[1]:
import hdf5storage as hs
# In[8]:
ar = hs.loadmat(fp+'/aod_ict/all_aod_ict.mat')
# ## Plot a histogram of all the AOD
# In[29]:
from plotting_utils import prelim
# In[30]:
plt.figure()
plt.hist(ar['AOD0501'][ar['fl']],bins=30,range=(0,1.0),alpha=0.5,normed=False,edgecolor='None',color='g',label='600-1800 m')
plt.hist(ar['AOD0501'][ar['fl1']],bins=30,range=(0,1.0),alpha=0.5,normed=False,edgecolor='None',color='b',label='below 600 m')
#plt.hist(ar['AOD0501'][ar['fl2']],bins=30,range=(0,1.0),alpha=0.5,normed=False,edgecolor='None',color='r',label='above 1800 m')
def main():
Classifier_path = '/Users/hyuns/Desktop/LDAModel/'
classifier_list = sorted(glob.glob(Classifier_path + '*.pickle'),
key=os.path.getmtime,
reverse=True)
print("classifer:", classifier_list[0])
lda = joblib.load(classifier_list[0])
root_path = '/Users/hyuns/Desktop/HGU/2020-2/Capstone/Drone Project/EEGData/VR300_Data/0729/hs/'
resampleRate = 100
mat_path = root_path + 'Online/'
current_list = sorted(glob.glob(mat_path + '*.mat'),
key=os.path.getmtime,
reverse=True)
score = 0
for mat_file in current_list:
print(mat_file)
ans = mat_file[-5]
mat = hdf5storage.loadmat(mat_file)
channelNames = mat['channelNames']
eegData = mat['eegData']
samplingFreq = mat['samplingFreq']
samplingFreq = samplingFreq[0, 0]
stims = mat['stims']
channelNum = channelNames.shape
channelNum = channelNum[1]
eegData = np.transpose(eegData)
buttonNum = 7
#Bandpass Filter
eegData = butter_bandpass_filter(eegData,
0.1,
30,
samplingFreq,
order=4)
#Epoching
epochSampleNum = int(np.floor(1.0 * samplingFreq))
offset = int(np.floor(0.0 * samplingFreq))
baseline = int(np.floor(1.0 * samplingFreq))
####### averaging whole epochs
Epochs_Aver = np.zeros((buttonNum, channelNum, epochSampleNum))
Epochs_final = np.zeros((buttonNum, channelNum, resampleRate))
featureNum = channelNum * resampleRate
for i in range(buttonNum):
Epochs_Aver[i] = Epoching(eegData, stims, (i + 1), samplingFreq,
channelNum, epochSampleNum, offset,
baseline)
Epochs_final[i] = resampling(Epochs_Aver[i], resampleRate,
channelNum)
Features = Convert_to_FeatureVector(Epochs_final, buttonNum,
featureNum)
Answers = lda.decision_function(Features)
answer = np.argmax(Answers) + 1
print(Answers)
if int(ans) == int(answer):
score = score + 1
print('order: ', ans, 'predict: ', answer)
print('score:', score)
def h5_to_panda(_file, _id):
if _id == "PI":
thisid = "Number_of_sips"
else:
thisid = _id
dataid = "data2/" + thisid
pid = "PVALS/" + thisid
out = hdf5storage.loadmat(_file, variable_names=[dataid, pid, "LABELS"])
datapoints = unrv_data(out[dataid])
pvals = unrv_data(out[pid])
labels = unrv_labels(out["LABELS"])
labels = [label[0,0] for label in labels]
labels[0] = "control"
yeast_data = datapoints[0]
sucrose_data = datapoints[1]
yeast_ps = pvals[0]
sucrose_ps = pvals[1]
numdtpoints = yeast_data.size-np.count_nonzero(~np.isnan(yeast_data))
if _id == "PI":
PIout = {"Label": [], "Data": [], "Median": [], "Signif": []}
contr = (yeast_data[:, 0] - sucrose_data[:, 0]) / (yeast_data[:, 0] + sucrose_data[:, 0])
Pvals = {}
for col in range(yeast_data.shape[1]): # different labels
PIcol = (yeast_data[:, col] - sucrose_data[:, col]) / (yeast_data[:, col] + sucrose_data[:, col])
PImedian = np.nanmedian(PIcol)
PIcol = PIcol[~np.isnan(PIcol)]
s, PIpval = stat.ranksums(contr, PIcol)
if np.isnan(PIpval):
PIpval = 1
Pvals[labels[col]] = PIpval
for row in range(yeast_data.shape[0]): # different datapoints same label
if ~np.isnan(yeast_data[row, col]) and ~np.isnan(sucrose_data[row, col]):
PIout["Label"].append(labels[col])
PIout["Data"].append(PIcol[row])
PIout["Median"].append(PImedian)
PIout["Signif"].append("Yes" if math.log10(1./PIpval)>2 else "No")
PIout = pd.DataFrame(PIout)
return PIout, Pvals
else:
Yout = {"Label": [], "Data": [], "Median": [], "Signif": []}
Sout = {"Label": [], "Data": [], "Median": [], "Signif": []}
for row in range(yeast_data.shape[0]): # different datapoints same label
for col in range(yeast_data.shape[1]): # different labels
if ~np.isnan(yeast_data[row, col]):
Yout["Label"].append(labels[col])
Yout["Data"].append(yeast_data[row, col])
Yout["Median"].append(np.nanmedian(yeast_data[:,col]))
Yout["Signif"].append("Yes" if math.log10(1./yeast_ps[col])>2 else "No")
if ~np.isnan(sucrose_data[row, col]):
Sout["Label"].append(labels[col])
Sout["Data"].append(sucrose_data[row, col])
Sout["Median"].append(np.nanmedian(sucrose_data[:,col]))
Sout["Signif"].append("Yes" if np.log10(1./sucrose_ps[col])>2 else "No")
Ydf = pd.DataFrame(Yout)
Sdf = pd.DataFrame(Sout)
Pvals = {}
for ind, label in enumerate(labels):
Pvals[label] = [yeast_ps[ind], sucrose_ps[ind]]
return [Ydf, Sdf], Pvals
# In[4]:
vv = 'v6'
# # Load some files for easier processing
# ## get the retrieval results
# In[5]:
m_dict = hs.loadmat(fp+'20130913_retrieval_output.mat')
# In[6]:
m_dict.keys()
# In[7]:
if not 'emas_tau_full' in vars():
print 'not defined, loading from file'
emas_tau_full = m_dict['emas'][1]; emas_ref_full = m_dict['emas'][3]; emas_utc_full = m_dict['emas'][5]
modis_tau = m_dict['modis'][1]; modis_ref = m_dict['modis'][3]
# -*- coding: utf-8 -*-
import matplotlib.pyplot as plt
import keras
from sklearn import preprocessing
from sklearn.preprocessing import StandardScaler
from sklearn import metrics
import random
import numpy as np
from matplotlib import pyplot as p
from scipy.io import loadmat, savemat
import hdf5storage
'TRAINING SET'
# load train set from .mat files
y_trainbig = hdf5storage.loadmat('../data/test_x_big.mat')
y_trainbig = y_trainbig['g']
y_train = y_trainbig[:, 50000:300000]
del y_trainbig
x_trainbig = hdf5storage.loadmat('../data/test_y_big.mat')
x_trainbig = x_trainbig['s']
x_train = x_trainbig[0:3, 50000:300000]
del x_trainbig
'shuffle data in the training set'
indi = np.arange(y_train.shape[1])
np.random.shuffle(indi)
y_train = y_train[:, indi]
x_train = x_train[:, indi]
'TEST SET'
def readData(input_file, sliceNo):
if os.path.isdir(input_file):
fl = [os.path.join(input_file, f)
for f in os.listdir(input_file)
if os.path.isfile(os.path.join(input_file, f))]
print('{:} is a directory.'.format(input_file))
if sliceNo is not None:
if sliceNo == 'mean':
# read each file and average
data0 = np.load(fl[0])
data0 = np.zeros(data0.shape, dtype=np.float64, order='F')
for f in fl:
data0 = data0 + np.load(f)
paData = data0 / len(fl)
else:
try:
sliceNo = int(sliceNo)
paData = np.load(
os.path.join(input_file,
'{:06d}.npy'.format(sliceNo)))
paData = paData.astype(np.float64)
except ValueError: # assuming a list was given
sliceNo = sorted([int(v) for v in sliceNo.split()])
data0 = np.load(
os.path.join(input_file,
'{:06d}.npy'.format(sliceNo[0])))
paData = np.zeros((data0.shape[0], data0.shape[1],
sliceNo[1] - sliceNo[0]),
dtype=np.float64, order='F')
paData[:, :, 0] = data0
for ind in range(sliceNo[0] + 1, sliceNo[1]):
paData[:, :, ind - sliceNo[0]] =\
np.load(os.path.join(input_file,
'{:06d}.npy'.format(ind)))
else:
fl = sorted(fl)
data0 = np.load(fl[0])
paData = np.zeros((data0.shape[0], data0.shape[1], len(fl)),
dtype=np.float64, order='F')
for i in range(len(fl)):
f = fl[i]
paData[:, :, i] = np.load(f)
ReconUtility.updateProgress(i+1, len(fl))
else:
(basename, ext) = os.path.splitext(input_file)
in_format = ext[1:]
# read out data
if in_format == 'h5':
print('Reading data from {:}'.format(input_file))
f = h5py.File(input_file, 'r')
paData = np.array(f['data'], order='F')
f.close()
print('Done loading.')
elif in_format == 'npy':
paData = np.load(input_file)
paData = np.copy(paData, order='F')
elif in_format == 'mat':
variableName = input('Variable in mat: ')
paData = np.array(h5.loadmat(input_file)[variableName], order='F')
else:
print('input format %s not supported' % in_format)
return
if sliceNo is not None:
if sliceNo == 'mean':
print('reconstructing averaged')
paData = np.array(np.mean(paData, axis=2), order='F')
else:
sliceNo = int(sliceNo)
print('reconstructing slice #{:d}'.format(sliceNo))
paData = np.copy(paData[:, :, sliceNo], order='F')
paData = paData.astype(np.float64)
return paData
class doc3dSELoader(data.Dataset):
pass
if __name__ == "__main__":
ROOT = 'C:/Users/yuttapichai.lam/dev-environment/dataset/'
FNAME = '1_1_2-pm_Page_144-Tv60001'
img_path = ROOT + 'img/' + FNAME + '.png'
img = m.imread(img_path, mode='RGB')
# img = np.array(img, dtype=np.uint8)
norm_path = ROOT + 'norm/' + FNAME + '.exr'
norm = cv2.imread(norm_path, cv2.IMREAD_ANYCOLOR | cv2.IMREAD_ANYDEPTH)
norm = cv2.cvtColor(norm, cv2.COLOR_RGB2BGR)
# norm = np.array(norm, dtype=np.float)
bm_path = ROOT + 'bm/' + FNAME + '.mat'
bm = h5.loadmat(bm_path)['bm']
# bm = np.array(bm, dtype=np.float)
img = img[:, :, ::-1]
im = img.transpose((2, 0, 1))
im = im.astype(np.float) / 255.0
im = np.expand_dims(im, 0)
im = torch.from_numpy(im).float()
bm = bm / np.array([448, 448])
bm = (bm - 0.5) * 2
bm0 = bm[:, :, 0]
bm1 = bm[:, :, 1]
bm = np.stack([bm0, bm1], axis=-1)
# bm = np.reshape(bm, (1, 448, 448, 2))
f=1.e-4
omega = Hk[3]
x,z,H,P,debug=sr.SolveRefl(k=k,Nsq0=Nsq0,z0=z0,omega=omega*f,f=f,wall=True,x=xmod*1e3,
H=H,J=192,Incoming=False)
ind = np.where(x>(x[-1]-70e3))[0]
res=np.mean(np.abs(np.real(P[:,ind]*np.exp(1j*0.)))*H[np.newaxis,ind])
return res,k
if __name__ == '__main__':
todo = ['Shelf100km','Shelf1km03','Shelf1km04','Shelf020km']
Hwkb=[]
xmod=np.linspace(0.,260.,261)
for nn,td in enumerate(todo):
D = hdf5storage.loadmat('../ttide15/Tas3d/%s/Diags0360.mat'%td)
H = D['Depth'].transpose()[:,-1]
x = -D['x']/1e3-30.+xmod[-1]
ray=mf.loadmatbunch ('../TasmaniaRays.mat')
ray = ray['ray']
z=np.arange(0.,5000.,2.)
N= np.interp(z,ray['z'],np.sqrt(ray['N2']))
N0 = np.mean(N)
Hwkb.append(np.interp(-xmod,-x,H))
lam = np.linspace(30,180.0,140)*1e3
ks = np.pi*2./lam
oms = np.linspace(1.,2.,3)
oms = oms[1:]
z0 = np.linspace(-6500,0.,1000)
from mne.time_frequency import tfr_morlet
import math
from gonogo.config import *
sid = 6 #4
l_freq = 0.5
h_freq = 2
data_dir = '/Volumes/Samsung_T5/data/ruijin/gonogo/preprocessing/P' + str(sid)
plot_dir = data_dir + '/erpPlots/'
if not os.path.exists(plot_dir):
os.makedirs(plot_dir)
#Session_num,UseChn,EmgChn,TrigChn = get_channel_setting(sid)
#original_fs=[Frequencies[i,1] for i in range(Frequencies.shape[0]) if Frequencies[i,0] == pn][0]
loadPath = data_dir + '/preprocessing/preprocessingv2.mat'
mat = hdf5storage.loadmat(loadPath)
fs = mat['Fs']
rtime = mat['ReactionTime']
rtime = np.concatenate((rtime[0, 0], rtime[0, 1]), axis=0)
data = mat['DATA']
data = np.concatenate((data[0, 0], data[0, 1]), axis=0) #(2160440, 63)
events = mat['Trigger']
events = np.concatenate((events[0, 0], events[0, 1]), axis=0) # two sessions
events[:, [1, 2]] = events[:, [
2, 1
]] # swap 1st and 2nd column to: timepoint, duration, event code
events = events.astype(int)
del mat
chn_num = data.shape[1]
dds = ['20160827','20160830','20160831','20160902','20160904','20160906','20160908',
'20160910','20160912','20160914','20160918','20160920','20160924','20160925','20160927']
# In[4]:
rts = []
sps = []
# In[5]:
for daystr in dds:
print daystr
rt = hs.loadmat(fp+'{}_zen_cld_retrieved.mat'.format(daystr))
s = sio.loadmat(fp+'4STAR_{}starzen.mat'.format(daystr))
sp = Sp.Sp(s)
rts.append(rt)
sps.append(sp)
# ## Load the cloud probe incloud flag
# In[6]:
from load_utils import mat2py_time,toutc
# In[7]:
# ## Load the 4STAR starsun
# In[44]:
f_star = fp+'data\\{}starsun.mat'.format(daystr)
# In[45]:
try:
s = sio.loadmat(f_star+'_small.mat')
except IOError:
s = sio.loadmat(f_star)
except NotImplementedError:
s = hs.loadmat(f_star+'_small.mat')
# In[46]:
s.keys()
# In[47]:
s['utc'] = lm.toutc(lm.mat2py_time(s['t']))
# In[48]:
s['tau_aero'].shape
a0 = nearest_neighbor(star_a[i]['Start_UTC'],star_a[i]['AOD_polycoef_a0'],ssfr_a[i]['Start_UTC'],dist=3.0/3600.0)
ssfr_a[i]['a0'] = a0
# In[220]:
ssfr_a[0]['flagacaod'].shape,ssfr_a[0]['Start_UTC'].shape
# ## Load the LUT for 2wvl reflectance retrieval
# In[30]:
lut = hs.loadmat(fp+'rtm/v5_irr_ORACLES_lut.mat')
# In[31]:
lut.keys()
# ## Combine into one array
# In[249]:
nm = ssfr_a[1].keys()
'20160910','20160912','20160914','20160918','20160920','20160924','20160925','20160927']
# In[5]:
rts = []
sps = []
# In[6]:
for daystr in dds:
print daystr
rt = hs.loadmat(fp+'{}_zen_cld_retrieved.mat'.format(daystr))
s = sio.loadmat(fp+'4STAR_{}starzen.mat'.format(daystr))
sp = Sp.Sp(s)
rts.append(rt)
sps.append(sp)
# ## Load the cloud probe incloud flag
# In[7]:
from load_utils import mat2py_time,toutc
# In[8]:
suffix = ""
if(N_EV <= 11): # +1 aggregator
suffix = "_10"
elif(N_EV <= 101): # +1 aggregator
suffix = "_100"
elif(N_EV <= 1001):
suffix = "_1000"
elif(N_EV <= 10001):
suffix = "_10000"
elif (N_EV <= 100001) :
suffix = "_100000"
elif (N_EV <= 1000001) :
suffix = "_1000000"
file_aggr = DATA_DIR + '/Aggregator/aggregator' + suffix + '.mat'
aggr = hdf5storage.loadmat(file_aggr)
D = aggr['D'][()]
'''
data = sio.loadmat(name)
#data = h5py.File(name, 'r') # open read-only
for key,val in data.items() :
if(key == 'r_norm'):
r_norm = data[key][()]
if(key == 's_norm'):
def display():
global filename_pca
if os.path.isfile(os.path.join('static/img/', filename_pca)):
os.remove(os.path.join('static/img/', filename_pca))
if session['data'].split('.')[-1] == 'csv':
df = pd.read_csv(session['data'])
elif session['data'].split('.')[-1] == 'mat':
mat = hd.loadmat(session['data'])
X = pd.DataFrame(mat['X'])
Y = pd.DataFrame(mat['y'])
Y.rename(columns={0: 'CLASS'}, inplace=True)
df = pd.concat([X, Y], axis=1)
shape = df.shape
desc = df.describe().values
lstCol = ['count', 'mean', 'std', 'min', '25%', '50%', '75%', 'max']
lstc = pd.DataFrame(lstCol)
descc = pd.DataFrame(desc)
lstc.rename(columns={0: 'Stat'}, inplace=True)
descr = pd.concat([lstc, descc], axis=1)
print(descr)
# -- Standarisasi Data --
kelas = df.columns[-1]
# print(df.columns)
sb_x = df.iloc[:, :-2]
sb_y = df.loc[:, kelas]
sb_x = StandardScaler().fit_transform(sb_x)
pca = PCA(n_components=2)
attr_x = pca.fit_transform(sb_x)
# print(attr_x)
red_data = pd.DataFrame(attr_x, columns=['A', 'B'])
print(red_data.shape)
print(df.loc[:, kelas].shape)
print(df.loc[:, kelas])
finalData = pd.concat([red_data, df.loc[:, kelas]], axis=1)
# print(red_data.values)
fig = plt.figure(figsize=(8, 8))
ax = fig.add_subplot(1, 1, 1)
ax.set_xlabel('A', fontsize=15)
ax.set_ylabel('B', fontsize=15)
outcomes = [1, 0]
colors = ['r', 'g']
for Outcome, color in zip(outcomes, colors):
index = finalData[kelas] == Outcome
ax.scatter(finalData.loc[index, 'A'],
finalData.loc[index, 'B'],
c=color,
s=50)
ax.legend(outcomes)
ax.grid()
# Random filename
filename_pca = ''.join(
random.choices(string.ascii_uppercase + string.digits, k=5))
filename_pca = filename_pca + '.png'
# print(filename)
plt.savefig(os.path.join('static/img/', filename_pca))
filepath = 'img/' + filename_pca
return render_template(
'display.html',
tables=[
df.to_html(table_id='dataset',
border=0,
classes='display table table-striped table-hover')
],
titles=df.columns.values,
data=df.values,
column=df.columns,
shape=shape,
descr=descr.values,
filepath=filepath)
days = [''] # In[7]: day = '20160914' # ## Load 4STAR gap distance calculations for ORACLES 2016 # In[9]: gap = hs.loadmat(fp+'ORACLES_2016_gap_distance.mat') # In[10]: gap['ldelta_alt'].shape # In[287]: gap['days'] = [i[0] for i in gap['ldelta_lat_days']] # In[303]:
import numpy as np from hdf5storage import loadmat from os.path import join import sys imset = sys.argv[1].lower().capitalize() DS_PATH = 'D:/datasets/processed/voc2012' GT_PATH = join(DS_PATH, 'Truth', imset) PRED_PATH = join(DS_PATH, 'Deeplab_Prediction', imset) ds_info = loadmat(join(DS_PATH, 'dataset_info.mat')) num_img = 350 if imset.lower() != 'val': num_img = 1449-num_img LGT_FMT = join(PRED_PATH, imset.lower()+'_%06d_logits.mat') LGT_MAT_NAME = 'logits_img' GT_FMT = join(GT_PATH, imset.lower()+'_%06d_pixeltruth.mat') GT_MAT_NAME = 'truth_img' conf_res = 0.05 conf_hist = np.zeros((int(1./conf_res)), dtype=np.uint64) for idx in range(1, num_img+1): gt = loadmat(GT_FMT % idx)[GT_MAT_NAME].ravel() mask = (gt>0) & (gt<255) logits = loadmat(LGT_FMT % idx)[LGT_MAT_NAME][...,1:].reshape(-1, 20)
hs.savemat(fp+'/aod_ict_2017/{vv}/all_aod_ict_{vv}_2017.mat'.format(vv=vv),ar)
# ## Optionally load the file
# In[8]:
import hdf5storage as hs
# In[9]:
ar = hs.loadmat(fp+'/aod_ict_2017/{vv}/all_aod_ict_{vv}_2017.mat'.format(vv=vv))
# ## Plot a histogram of all the AOD
# In[10]:
from plotting_utils import prelim
# In[198]:
plt.figure()
plt.hist(ar['AOD0501'][ar['fl']],bins=30,range=(0,1.0),alpha=0.5,normed=False,edgecolor='None',color='g',label='600-1800 m')
def tang_run_acc(data, labels, traintestfilenames=None):
"""
"""
[height, width, nbands] = data.shape
all_pixels = np.array(list(itertools.product(range(width),range(height))))
labelled_pixels = np.array(filter(lambda (x,y): labels[y,x] != 0, all_pixels))
flat_labels = labels.transpose().reshape(height*width)
nlabels = len(set(flat_labels.tolist())) - 1
padded_data = np.pad(data, ((9,9),(9,9),(9,9)), 'wrap')
print('requesting %d MB memory' % (labelled_pixels.shape[0] * 271*nbands * 4 / 1000000.0))
labelled_pix_feat = np.zeros((labelled_pixels.shape[0], 271*nbands), dtype=np.float32)
def compute_features():
x = tf.placeholder(tf.float32, shape=(19,19,nbands+18))
feat = tang_net(x)
sess = tf.Session()
sess.run(tf.global_variables_initializer())
for pixel_i, pixel in enumerate(tqdm(labelled_pixels)):
# this iterates through columns first
[pixel_x, pixel_y] = pixel
subimg = padded_data[pixel_y:(pixel_y+19), pixel_x:(pixel_x+19), :]
feed_dict = {x: subimg}
labelled_pix_feat[pixel_i,:] = sess.run(feat, feed_dict)
compute_features()
for traintestfilename in traintestfilenames:
mat_contents = None
try:
mat_contents = sio.loadmat(os.path.join(DATA_PATH, traintestfilename))
except:
mat_contents = hdf5storage.loadmat(os.path.join(DATA_PATH, traintestfilename))
train_mask = mat_contents['train_mask'].astype(int).squeeze()
test_mask = mat_contents['test_mask'].astype(int).squeeze()
# resize train/test masks to labelled pixels
train_mask_skip_unlabelled = train_mask[flat_labels!=0]
test_mask_skip_unlabelled = test_mask[flat_labels!=0]
# get training set
trainY = flat_labels[train_mask==1]
trainX = labelled_pix_feat[train_mask_skip_unlabelled==1,:]
print('training now')
start = time.time()
clf = SVC(kernel='linear')
clf.fit(trainX, trainY)
end = time.time()
print(end - start)
# now test
test_chunk_size = 1000
testY = flat_labels[test_mask==1]
Yhat = np.zeros(testY.shape)
testX = labelled_pix_feat[test_mask_skip_unlabelled==1,:]
C = np.zeros((nlabels,nlabels))
print('testing now')
for i in tqdm(range(0,len(testY),test_chunk_size)):
p_label = clf.predict(testX[i:i+test_chunk_size,:]);
Yhat[i:i+test_chunk_size] = np.array(p_label).astype(int)
C += confusion_matrix(testY[i:i+test_chunk_size], p_label, labels=list(range(1,nlabels+1)))
pred_image = np.zeros(flat_labels.shape)
pred_image[test_mask==1] = Yhat
mat_outdata = {}
mat_outdata[u'metrics'] = {}
mat_outdata[u'metrics'][u'CM'] = C
mat_outdata[u'pred_image'] = pred_image.reshape((width, height)).transpose()
hdf5storage.write(mat_outdata, filename=os.path.join(DATA_PATH, traintestfilename+'_117_WST3D_expt.mat'), matlab_compatible=True)
def load_matfile(filepath, variable_names=None, **kwargs):
"""
Load mat-file and return dictionary.
Function loads mat-file by traversing the structure converting data into
low-level numpy arrays of different types. There is no guarantee that any
kind of data is read in correctly. The data dtype can also vary depending
on the mat-file (v7.3 returns floats instead of integers). But it should
work identically for v7.3 and prior mat-files. Arrays are squeezed if
possible. Relies on the :mod:`hdf5storage` package.
Parameters
----------
filepath : str
Filepath and name of mat-file.
variable_names : list of strings
Top-level variables to be loaded.
**kwargs : keywords
Addidional keyword arguments are passed to :func:`hdf5storage.loadmat`.
Returns
-------
data : dict
Dictionary containing the data as dictionaries or numpy arrays.
"""
# define a recursively called function to traverse structure
def traverse_struct(struct):
# for dictionaries, iterate through keys
if isinstance(struct, dict):
out = dict()
for key, value in struct.items():
out[key] = traverse_struct(value)
return out
# for ndarray, iterate through dtype names
elif isinstance(struct, np.ndarray):
# collect dtype names if available
names = struct.dtype.names
# if no fields in array
if names is None:
if struct.dtype == np.dtype('O') and struct.shape == (1, 1):
return traverse_struct(struct[0, 0])
else:
return struct.squeeze()
else: # if there are fields, iterate through fields
out = dict()
for name in names:
out[name] = traverse_struct(struct[name])
return out
else:
return struct
output = hdf.loadmat(filepath, variable_names=variable_names, **kwargs)
# loadmat returns dictionary, go through keys and call traverse_struct
for key, value in output.items():
if key.startswith('__') and key.endswith('__'):
pass
else:
output[key] = traverse_struct(value)
return output
#!/usr/bin/env python # coding: utf-8 # In[3]: import hdf5storage import pandas as pd dataname='alldata_1316_fixed.mat' mat = hdf5storage.loadmat(dataname) pdata = pd.DataFrame(mat['Data'],columns = ['DATE','PERMNO','VOL','SHROUT','retadj','LME','ret','prca','OPENPRC','divamt','facpra','facshr','SHRCD','EXCHCD','DLRET','DLPRC','DISTCD','PERMCO','weight_port','lprc','ME','ticker_idx','SIZEPORT','BTMPORT','OPPORT','INVPORT','RRGRP']) # startdate = 20150901 # enddate = 20150901 # ind=(pdata['DATE'] >= startdate) & (pdata['DATE'] <= enddate) DataPc=pdata entry=[1,2,19,8,5,10,11,12,6,15,16,17,20,21,23,25,9] #entry = [1,2,19,8,5,9,10,11,12,6,15,16,17,20,21,22,23,24,25,26,27] entryew=[i-1 for i in entry] DataPcnew=DataPc.iloc[:,entryew].copy() #DataPcnew.columns.values.tolist() DataPcnew['prca']=DataPcnew['prca'].abs() DataPcnew['lprc']=DataPcnew['lprc'].abs() # In[11]:
images = read_images_to_np(path,
n,
n,
extension="all",
allowmax=True,
maxnbr=1000,
d_type=np.float32,
mode="RGB")
gray_images = [cv2.cvtColor(image, cv2.COLOR_BGR2GRAY) for image in images]
plt.figure()
plt.imshow(gray_images[RANDOM_NO], cmap='gray')
plt.title("Filtering by matrix multiplicatin")
plt.show(block=False)
print("finished loading the images\n")
if LOAD_MAT:
mat = hdf5storage.loadmat('filtering.mat')
k_padded = mat["k_padded"]
A = mat["A_unfolded"]
K = mat["kernel"]
print("the Kernel\n", K, "\n\n")
# print("the Kernel padded \n",k_padded,"\n\n")
# k_padded = sio.loadmat("filtering.mat")["kernel"]
# A = sio.loadmat("filtering.mat")["A_unfolded"]
X = gray_images[RANDOM_NO]
x = X.reshape(-1, 1)
# K = np.array(range(1,m**2+1)).reshape(m,m)
else:
K = np.ones((m, m), np.float32)
k_padded = np.pad(K, ((skip, skip), (skip, skip)), 'constant')
print("the Kernel\n", K, "\n\n")
help='saving path that is a folder')
parser.add_argument('--task_name',
type=str,
default='track2',
help='task name for loading networks, saving, and log')
# NTIRE2020_Validation_Clean NTIRE2020_Validation_RealWorld
opt = parser.parse_args()
sample_folder = os.path.join(opt.val_path, opt.task_name)
utils.check_path(sample_folder)
imglist = utils.get_jpgs(opt.path1)
for imgname in imglist:
# Read
data_path1 = os.path.join(opt.path1, imgname)
data1 = hdf5.loadmat(data_path1)['cube']
data_path2 = os.path.join(opt.path2, imgname)
data2 = hdf5.loadmat(data_path2)['cube']
data_path3 = os.path.join(opt.path3, imgname)
data3 = hdf5.loadmat(data_path3)['cube']
data_path4 = os.path.join(opt.path4, imgname)
data4 = hdf5.loadmat(data_path4)['cube']
data_path5 = os.path.join(opt.path5, imgname)
data5 = hdf5.loadmat(data_path5)['cube']
data_path6 = os.path.join(opt.path6, imgname)
data6 = hdf5.loadmat(data_path6)['cube']
data_path7 = os.path.join(opt.path7, imgname)
data7 = hdf5.loadmat(data_path7)['cube']
data_path8 = os.path.join(opt.path8, imgname)
data8 = hdf5.loadmat(data_path8)['cube']
data_avg = (data1 + data2 + data3 + data4 + data5 + data6 + data7 +
f[np.where(f < 500)],
Sxx[np.where(f < 500)[0], :],
shading='gouraud',
cmap='Greys',
norm=Normalize())
fig.colorbar(cf, ax=axes[2])
axes[2].set_ylabel('Frequency [MHz]')
axes[2].set_xlabel('Time samples')
axes[2].set_title("rp_2 spectogram")
plt.tight_layout(pad=1.2)
plt.savefig("spectogram.pdf")
plt.show()
path = 'Z:\\traces\\CMOS\\'
mat = hdf5storage.loadmat(path + 'fullssa.mat')
# np.complex64
traces = (mat['traces'])
Y = mat['Y']
Y = np.reshape(Y, (np.size(Y), ))
SAMPLES = np.shape(traces)[1]
Queries = np.shape(traces)[0]
from main_fft import plot_roc
plot_roc(traces, Y, None, 40, 'fullssa_base', [0, 1000])
#plot_roc(traces, Y, "Z:\\filters\\shuffled_rp_8\\filter_BPF.mat", 40, 'rp8_bpf', [480, 800])
#plot_roc(traces, Y, "Z:\\filters\\shuffled_rp_8\\filter_SNR_FFT.mat", 40, 'rp8_snr', [480, 800])
exit(0)
from grasp.process.utils import getRawData, get_trigger
'''
Function: pick up the non-signal channel and trigger channel by plotting and visual check.
Usage:
1,use - to scale down the plot
'''
sid = 16
sessions = 4 # 4 sessions
movements = 4 # 4 movements
session = 0 # evaluate the channle on one session
print("Read raw data from disk.")
seegfile = data_dir + 'PF' + str(sid) + '/SEEG_Data/PF' + str(
sid) + '_F_' + str(session + 1) + '.mat'
mat = hdf5storage.loadmat(seegfile)
raw = mat['Data']
#triggerRaw = mat['Data'][29, :]
fs = int(mat['Fs'][0][0])
chnRaw = mat[
'ChnName'] # total channels: len(chnRaw)-->update the channel_settings.py
channels = list(chnRaw)
while not isinstance(channels[0], np.str_):
channels = [channels[i][0] for i in range(len(chnRaw))]
#channels = np.asarray([chnRaw[i][0][0][0] for i in range(len(chnRaw))]) # list with len=126
#ch_names = [channelsName.strip() for channelsName in channels]
ch_index_str = [
str(chi) + '_' + channels[chi].strip() for chi in [*range(len(channels))]
]
ch_index = [*range(len(channels))] #147
def test_append(self):
self.assertEqual(self.FileSample.fname, 'test_in_progress.mat')
self.assertEqual(self.FileSample.n, 0)
self.FileSample.append(np.matrix([[1], [2], [1], [2], [1], [1]]),
np.matrix([1]), 10)
self.assertEqual(self.FileSample.n, 10)
self.assertTrue(os.path.exists(self.FileSample.fname))
from hdf5storage import loadmat
x = loadmat(self.FileSample.fname)
self.assertEqual(sorted(
x.keys()), ['LnPDF_1', 'MTSpace_1', 'i', 'n', 'non_zero_samples'])
self.assertEqual(x['n'], 10)
self.FileSample.append(np.matrix([[1], [2], [1], [2], [1], [1]]),
np.matrix([1]), 20)
self.assertEqual(self.FileSample.n, 30)
self.assertTrue(os.path.exists(self.FileSample.fname))
x = loadmat(self.FileSample.fname)
self.assertEqual(sorted(x.keys()), [
'LnPDF_1', 'LnPDF_2', 'MTSpace_1', 'MTSpace_2', 'i', 'n',
'non_zero_samples'
])
self.assertEqual(x['n'], 30)
self.FileSample.append(np.matrix([[1], [2], [1], [2], [1], [1]]),
np.matrix([-np.inf]), 20)
self.assertEqual(self.FileSample.n, 50)
x = loadmat(self.FileSample.fname)
self.assertEqual(sorted(x.keys()), [
'LnPDF_1', 'LnPDF_2', 'MTSpace_1', 'MTSpace_2', 'i', 'n',
'non_zero_samples'
])
self.assertEqual(x['n'], 50)
self.tearDown()
self.setUp()
self.assertEqual(self.FileSample.fname, 'test_in_progress.mat')
self.assertEqual(self.FileSample.n, 0)
self.FileSample.append(np.matrix([[1], [2], [1], [2], [1], [1]]),
np.matrix([1]), 10,
[{
'mu': np.array([[1, 2], [2, 1]]),
'sigma': np.array([[0, 0.1], [0.1, 0]])
}])
self.assertEqual(self.FileSample.n, 10)
self.assertTrue(os.path.exists(self.FileSample.fname))
from hdf5storage import loadmat
x = loadmat(self.FileSample.fname)
self.assertEqual(sorted(x.keys()), [
'LnPDF_1', 'MTSpace_1', 'i', 'n', 'non_zero_samples',
'scale_factor_1'
])
self.assertEqual(x['n'], 10)
self.FileSample.append(np.matrix([[1], [2], [1], [2], [1], [1]]),
np.matrix([1]), 20,
[{
'mu': np.array([[1, 2], [2, 1]]),
'sigma': np.array([[0, 0.1], [0.1, 0]])
}])
self.assertEqual(self.FileSample.n, 30)
self.assertTrue(os.path.exists(self.FileSample.fname))
x = loadmat(self.FileSample.fname)
self.assertEqual(sorted(x.keys()), [
'LnPDF_1', 'LnPDF_2', 'MTSpace_1', 'MTSpace_2', 'i', 'n',
'non_zero_samples', 'scale_factor_1', 'scale_factor_2'
])
self.assertEqual(x['n'], 30)
self.FileSample.append(np.matrix([[1], [2], [1], [2], [1], [1]]),
np.matrix([-np.inf]), 20,
[{
'mu': np.array([[1, 2], [2, 1]]),
'sigma': np.array([[0, 0.1], [0.1, 0]])
}])
self.assertEqual(self.FileSample.n, 50)
x = loadmat(self.FileSample.fname)
self.assertEqual(sorted(x.keys()), [
'LnPDF_1', 'LnPDF_2', 'MTSpace_1', 'MTSpace_2', 'i', 'n',
'non_zero_samples', 'scale_factor_1', 'scale_factor_2'
])
self.assertEqual(x['n'], 50)
## filter to deal with some 0 reflectances
h = np.ones((3, 3))
h[1, 1] = 0
## loop over all images
for f in list_file:
cube_name = re.findall('\d+', f)[0]
path_f = indexPath + '\\' + cube_name
if not os.path.exists(path_f):
os.makedirs(path_f)
# pdb.set_trace()
indices = dict()
R = dict()
data = hdf5storage.loadmat(os.path.join(hyperPath, f))['data']
f_ = f.replace('_rd_rf.mat', '')
for color in list_wv_color.keys():
R.update({
color:
np.average(data[:, :,
idx_colors[color][0]:idx_colors[color][1] + 1],
axis=2)
})
# R_wv = {}
for wv in list_wv:
R[wv] = np.average(data[:, :, idx_wv[wv]], axis=2)
# pdb.set_trace()
# getting rid of the "divided by zero" issue
'N_evals': 30, # Max epochs
'L_rate_N1N2': 2e-4,
'W_N1': 1e-4,
'W_N2': 1e-4,
'enc_layers': [300, 200],
'dec_layers': [200],
'AE_activation': 'tanh',
'N1_layers': [80, 160, 320, 640, 320, 160, 80],
'N2_layers': [80, 160, 320, 640, 320, 160, 80],
'datastep': 10,
'bat_n_all': True,
'actual_epoch': 0,
}
## Load Data
PI = hdf5storage.loadmat('./data/PI_coma.mat') # Point clouds Indexes
PI = np.squeeze(PI['PI']) - 1
data = hdf5storage.loadmat('./data/coma_FEM.mat') # Load dataset
pix1 = data['meshes_noeye'].reshape(data['meshes_noeye'].shape[0],
data['meshes_noeye'].shape[1], 3).astype(
'float32') # Vertices of the meshes
pix = pix1[:, PI]
outliers = np.asarray([6710, 6792, 6980]) - 1
remeshed = np.asarray(
[820, 1200, 7190, 11700, 12500, 14270, 15000, 16300, 19180, 20000]) - 1
save_every = 100
# Define Chamfer
def distance_matrix(array1, array2):
other_mirror_loss = (np.sum(np.square(x_target_anchor_features - x_other_pair_features), axis=-1) + \
np.sum(np.square(x_target_anchor_features - x_other_mirror_features), axis=-1)) / 2
else:
other_mirror_loss = np.sum(np.square(x_target_anchor_features -
x_other_pair_features),
axis=-1)
# Store losses themselves
global_target_loss.append(target_mirror_loss)
global_int_loss.append(other_mirror_loss)
# Get local directory
local_dir = result_dir + '/target%d_sample%d' % (target_person,
best_anchor_idx)
try:
contents = hdf5storage.loadmat(local_dir + '/attack.mat')
if mask_style == 'eyeglasses':
# Load the results of the indirect attack (only for eyeglasses currently, can be anything else)
try:
contents_indirect = hdf5storage.loadmat(
'paper_attacks_indirect_eyeglasses/%s/%s/%s/target%d_sample%d/attack.mat'
% (submeta_folder, core_folder, core_weights,
target_person, best_anchor_idx))
adv_indirect_loss = contents_indirect['adv_true_feature_loss']
global_indirect_loss.append(adv_indirect_loss)
except:
print('Skipping target %d! No indirect attack found' %
target_person)
except:
print('Skipping target %d!' % target_person)
def main():
############## seeg data ##########
sid = 10 # 4
fs = 1000
class_number = 5
Session_num, UseChn, EmgChn, TrigChn, activeChan = get_channel_setting(sid)
loadPath = data_dir + 'preprocessing' + '/P' + str(
sid) + '/preprocessing2.mat'
mat = hdf5storage.loadmat(loadPath)
data = mat['Datacell']
channelNum = int(mat['channelNum'][0, 0])
data = np.concatenate((data[0, 0], data[0, 1]), 0)
del mat
# standardization
# no effect. why?
if 1 == 1:
chn_data = data[:, -3:]
data = data[:, :-3]
scaler = StandardScaler()
scaler.fit(data)
data = scaler.transform((data))
data = np.concatenate((data, chn_data), axis=1)
# stim0 is trigger channel, stim1 is trigger position calculated from EMG signal.
chn_names = np.append(["seeg"] * len(UseChn), ["stim0", "emg", "stim1"])
chn_types = np.append(["seeg"] * len(UseChn), ["stim", "emg", "stim"])
info = mne.create_info(ch_names=list(chn_names),
ch_types=list(chn_types),
sfreq=fs)
raw = mne.io.RawArray(data.transpose(), info)
# gesture/events type: 1,2,3,4,5
events0 = mne.find_events(raw, stim_channel='stim0')
events1 = mne.find_events(raw, stim_channel='stim1')
# events number should start from 0: 0,1,2,3,4, instead of 1,2,3,4,5
events0 = events0 - [0, 0, 1]
events1 = events1 - [0, 0, 1]
# print(events[:5]) # show the first 5
# Epoch from 4s before(idle) until 4s after(movement) stim1.
raw = raw.pick(["seeg"])
epochs = mne.Epochs(raw, events1, tmin=0, tmax=4, baseline=None)
# or epoch from 0s to 4s which only contain movement data.
# epochs = mne.Epochs(raw, events1, tmin=0, tmax=4,baseline=None)
epoch1 = epochs['0'].get_data(
) # 20 trials. 8001 time points per trial for 8s.
epoch2 = epochs['1'].get_data()
epoch3 = epochs['2'].get_data()
epoch4 = epochs['3'].get_data()
epoch5 = epochs['4'].get_data()
list_of_epochs = [epoch1, epoch2, epoch3, epoch4, epoch5]
total_len = list_of_epochs[0].shape[2]
# validate=test=2 trials
trial_number = [list(range(epochi.shape[0])) for epochi in list_of_epochs
] # [ [0,1,2,...19],[0,1,2...19],... ]
test_trials = [random.sample(epochi, 2) for epochi in trial_number]
# len(test_trials[0]) # test trials number
trial_number_left = [
np.setdiff1d(trial_number[i], test_trials[i])
for i in range(class_number)
]
val_trials = [
random.sample(list(epochi), 2) for epochi in trial_number_left
]
train_trials = [
np.setdiff1d(trial_number_left[i], val_trials[i]).tolist()
for i in range(class_number)
]
# no missing trials
assert [
sorted(test_trials[i] + val_trials[i] + train_trials[i])
for i in range(class_number)
] == trial_number
test_epochs = [
epochi[test_trials[clas], :, :]
for clas, epochi in enumerate(list_of_epochs)
] # [ epoch0,epoch1,epch2,epoch3,epoch4 ]
val_epochs = [
epochi[val_trials[clas], :, :]
for clas, epochi in enumerate(list_of_epochs)
]
train_epochs = [
epochi[train_trials[clas], :, :]
for clas, epochi in enumerate(list_of_epochs)
]
wind = 500
stride = 500
X_train = []
y_train = []
X_val = []
y_val = []
X_test = []
y_test = []
for clas, epochi in enumerate(test_epochs):
Xi, y = slide_epochs(epochi, clas, wind, stride)
assert Xi.shape[0] == len(y)
X_test.append(Xi)
y_test.append(y)
X_test = np.concatenate(X_test, axis=0) # (1300, 63, 500)
y_test = np.asarray(y_test)
y_test = np.reshape(y_test, (-1, 1)) # (5, 270)
for clas, epochi in enumerate(val_epochs):
Xi, y = slide_epochs(epochi, clas, wind, stride)
assert Xi.shape[0] == len(y)
X_val.append(Xi)
y_val.append(y)
X_val = np.concatenate(X_val, axis=0) # (1300, 63, 500)
y_val = np.asarray(y_val)
y_val = np.reshape(y_val, (-1, 1)) # (5, 270)
for clas, epochi in enumerate(train_epochs):
Xi, y = slide_epochs(epochi, clas, wind, stride)
assert Xi.shape[0] == len(y)
X_train.append(Xi)
y_train.append(y)
X_train = np.concatenate(X_train, axis=0) # (1300, 63, 500)
y_train = np.asarray(y_train)
y_train = np.reshape(y_train, (-1, 1)) # (5, 270)
chn_num = X_train.shape[1]
train_set = myDataset(X_train, y_train)
val_set = myDataset(X_val, y_val)
test_set = myDataset(X_test, y_test)
batch_size = 32
train_loader = DataLoader(dataset=train_set,
batch_size=batch_size,
shuffle=True,
pin_memory=False)
val_loader = DataLoader(dataset=val_set,
batch_size=batch_size,
shuffle=True,
pin_memory=False)
test_loader = DataLoader(dataset=test_set,
batch_size=batch_size,
shuffle=True,
pin_memory=False)
########## end seeg #########################
global args, enable_cuda
################################################################ INIT #################################################################################
args = parser.parse_args()
cwd = os.getcwd()
#dpath=os.path.dirname(cwd)
dpath = '/Volumes/Samsung_T5/data/braindecode/'
result_dir = dpath + 'result/'
if not os.path.exists(result_dir):
os.makedirs(result_dir)
#Paths for data, model and checkpoint
data_path = os.path.join(dpath, 'Data/')
model_save_path = os.path.join(dpath, 'Models',
'Model_GumbelregHighgamma_M' + str(args.M))
checkpoint_path = os.path.join(
dpath, 'Models', 'Checkpoint_GumbelregHighgamma_M' + str(args.M))
if not os.path.isdir(os.path.join(dpath, 'Models')):
os.makedirs(os.path.join(dpath, 'Models'))
#Check if CUDA is available
enable_cuda = torch.cuda.is_available()
if (args.verbose):
print('GPU computing: ', enable_cuda)
#Set random seed
if (args.seed == 0):
args.seed = randint(1, 99999)
#Initialize devices with random seed
torch.manual_seed(args.seed)
torch.backends.cudnn.deterministic = True
torch.backends.cudnn.benchmark = False
training_accs = []
val_accs = []
test_accs = []
#Create a vector of length epochs, decaying start_value to end_value exponentially, reaching end_value at end_epoch
def exponential_decay_schedule(start_value, end_value, epochs, end_epoch):
t = torch.FloatTensor(torch.arange(0.0, epochs))
p = torch.clamp(t / end_epoch, 0, 1)
out = start_value * torch.pow(end_value / start_value, p)
return out
#Network loss function
def loss_function(output, target, model, lamba, weight_decay):
l = nn.CrossEntropyLoss()
sup_loss = l(output, target)
reg = model.regularizer(lamba, weight_decay)
return sup_loss, reg
#Create schedule for temperature and regularization threshold
temperature_schedule = exponential_decay_schedule(args.start_temp,
args.end_temp,
args.epochs,
int(args.epochs * 3 / 4))
thresh_schedule = exponential_decay_schedule(10.0, 1.1, args.epochs,
args.epochs)
#Load data
num_subjects = 5
input_dim = [44, 1125]
#train_loader1,val_loader1,test_loader1 = all_subject_loader_HGD(batch_size=args.batch_size,train_split=args.train_split,path=data_path,num_subjects=num_subjects)
################################################################ SUBJECT-INDEPENDENT CHANNEL SELECTION #################################################################################
if (args.verbose):
print('Start training')
torch.manual_seed(args.seed)
torch.backends.cudnn.deterministic = True
torch.backends.cudnn.benchmark = False
#Instantiate model
model = SelectionNet(input_dim, args.M).float()
if (enable_cuda):
model.cuda()
model.set_freeze(False)
optimizer = torch.optim.Adam(model.parameters(), args.lr)
prev_val_loss = 100
patience_timer = 0
early_stop = False
epoch = 0
fig, ax = plt.subplots()
while epoch in range(args.epochs) and (not early_stop):
#Update temperature and threshold
model.set_thresh(thresh_schedule[epoch])
model.set_temperature(temperature_schedule[epoch])
#Perform training step
train(train_loader, model, loss_function, optimizer, epoch,
args.weight_decay, args.lamba, args.gradacc, args.verbose)
val_loss = validate(val_loader, model, loss_function, epoch,
args.weight_decay, args.lamba, args.verbose)
tr_acc, val_acc, test_acc = test(train_loader, val_loader, test_loader,
model, loss_function,
args.weight_decay, args.verbose)
#Extract selection neuron entropies, current selections and probability distributions
H, sel, probas = model.monitor()
ax.plot(probas.detach().numpy())
fig.savefig(result_dir + 'prob_dist' + str(epoch) + '.png')
ax.clear()
#fig.clear()
#If selection convergence is reached, enable early stopping scheme
if ((torch.mean(H.data) <= args.entropy_lim)
and (val_loss > prev_val_loss - args.stop_delta)):
patience_timer += 1
if (args.verbose):
print('Early stopping timer ', patience_timer)
if (patience_timer == args.patience):
early_stop = True
else:
patience_timer = 0
H, sel, probas = model.monitor()
torch.save(model.state_dict(), checkpoint_path)
prev_val_loss = val_loss
epoch += 1
if (args.verbose):
print('Channel selection finished')
#Store subject independent model
model.load_state_dict(torch.load(checkpoint_path))
pretrained_path = str(model_save_path +
'all_subjects_channels_selected.pt')
torch.save(model.state_dict(), pretrained_path)
################################################################ SUBJECT FINETUNING #################################################################################
## freeze the selection layer and train the model other part
if (args.verbose):
print('Start subject specific training')
for k in range(1, num_subjects + 1):
if (args.verbose):
print('Start training for subject ' + str(k))
torch.manual_seed(args.seed)
torch.backends.cudnn.deterministic = True
torch.backends.cudnn.benchmark = False
#Load subject independent model and freeze selection neurons
model = SelectionNet(input_dim, args.M)
model.load_state_dict(torch.load(pretrained_path))
if (enable_cuda):
model.cuda()
model.set_freeze(True)
#Load subject dependent data
train_loader, val_loader, test_loader = within_subject_loader_HGD(
subject=k,
batch_size=args.batch_size,
train_split=args.train_split,
path=data_path)
optimizer = torch.optim.Adam(model.parameters(), args.lr)
prev_val_loss = 100
patience_timer = 0
early_stop = False
epoch = 0
while epoch in range(args.epochs) and (not early_stop):
#Perform train step
train(train_loader, model, loss_function, optimizer, epoch,
args.weight_decay, args.lamba, args.gradacc, args.verbose)
val_loss = validate(val_loader, model, loss_function, epoch,
args.weight_decay, args.lamba, args.verbose)
tr_acc, val_acc, test_acc = test(train_loader, val_loader,
test_loader, model, loss_function,
args.weight_decay, args.verbose)
#Extract selection neuron entropies, current selections and probability distributions
H, sel, probas = model.monitor()
#Perform early stopping
if (val_loss > prev_val_loss - args.stop_delta):
patience_timer += 1
if (args.verbose):
print('Early stopping timer ', patience_timer)
if (patience_timer == args.patience):
early_stop = True
else:
patience_timer = 0
torch.save(model.state_dict(), checkpoint_path)
prev_val_loss = val_loss
epoch += 1
#Store model with lowest validation loss
model.load_state_dict(torch.load(checkpoint_path))
path = str(model_save_path + 'finished_subject' + str(k) + '.pt')
torch.save(model.state_dict(), path)
#Evaluate model
tr_acc, val_acc, test_acc = test(train_loader, val_loader, test_loader,
model, loss_function,
args.weight_decay, args.verbose)
training_accs.append(tr_acc)
val_accs.append(val_acc)
test_accs.append(test_acc)
################################################################ TERMINATION #################################################################################
print('Selection', sel.data)
print('Training accuracies', training_accs)
print('Validation accuracies', val_accs)
print('Testing accuracies', test_accs)
tr_med = statistics.median(training_accs)
val_med = statistics.median(val_accs)
test_med = statistics.median(test_accs)
tr_mean = statistics.mean(training_accs)
val_mean = statistics.mean(val_accs)
test_mean = statistics.mean(test_accs)
print('Training median accuracy', tr_med)
print('Validation median accuracy', val_med)
print('Testing median accuracy', test_med)
print('Training mean accuracy', tr_mean)
print('Validation mean accuracy', val_mean)
print('Testing mean accuracy', test_mean)
#________________Launch simulation______________
model['vp'][20:40,10:20,5:10]= 3550
model['taup'][20:40,10:20,5:10]= 0.03
h5mat.savemat(filenames['csts'], csts , appendmat=False, format='7.3', store_python_metadata=True, truncate_existing=True)
h5mat.savemat(filenames['model'], model , appendmat=False, format='7.3', store_python_metadata=True, truncate_existing=True)
filepath=os.getcwd()
cmdlaunch='cd ../src/; mpirun -np 1 ./SeisCL_MPI '+filepath+'/SeisCL > ../tests/out 2>../tests/err'
print(cmdlaunch)
pipes = subprocess.Popen(cmdlaunch,stdout=subprocess.PIPE, stderr=subprocess.PIPE,shell=True)
while (pipes.poll() is None):
time.sleep(1)
sys.stdout.write('Forward calculation completed \n')
sys.stdout.flush()
dout = h5mat.loadmat(filenames['dout'])
din={}
din['src_pos']=dout['src_pos']
din['rec_pos']=dout['rec_pos']
din['vx0']=dout['vxout']
din['vy0']=dout['vyout']
din['vz0']=dout['vzout']
h5mat.savemat(filenames['din'], din , appendmat=False, format='7.3', store_python_metadata=True, truncate_existing=True)
##________________Calculate gradient______________
#model['vp'][20:40,10:20,5:10]= 3500
#model['taup'][20:40,10:20,5:10]= 0.02
#csts['gradout']=1
#csts['resout']=1
#csts['gradfreqs']=np.append(csts['gradfreqs'], csts['f0'])
store_python_metadata=True,
truncate_existing=True)
filepath = os.getcwd()
cmdlaunch = 'cd ../src/; mpirun -np 1 ./SeisCL_MPI ' + filepath + '/SeisCL > ../tests/out 2>../tests/err'
print(cmdlaunch)
pipes = subprocess.Popen(cmdlaunch,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
shell=True)
while (pipes.poll() is None):
time.sleep(1)
sys.stdout.write('Forward calculation completed \n')
sys.stdout.flush()
dout = h5mat.loadmat(filenames['dout'])
din = {}
din['src_pos'] = dout['src_pos']
din['rec_pos'] = dout['rec_pos']
din['vx'] = np.transpose(dout['vxout'])
din['vz'] = np.transpose(dout['vzout'])
#din['p']=np.transpose(dout['pout'])
h5mat.savemat(filenames['din'],
din,
appendmat=False,
format='7.3',
store_python_metadata=True,
truncate_existing=True)
#________________Calculate gradient______________
csts['MOVOUT'] = 0
def channel_Mat2Array(path):
channel = hdf5storage.loadmat(path)
return channel['result'][0, 0]['results'][0, 0]['pixel'][0, 0]
tf.reset_default_graph() # debugging, clear all tf variables
#tf.enable_eager_execution() # placeholders are not compatible
import model_convNet
import scipy.io
_FLOATX = tf.float32
## LOAD DATA
mainFile = '/Users/seshads1/Documents/code/ACLEW/cnn_wavenet_like/1/'
#mainFile = '/l/seshads1/code/syllCount/syll_cout/'
loadFile = mainFile + 'pyDat_test.mat'
loaddata = hdf5storage.loadmat(loadFile)
x_test = loaddata['x_test'] # test features
t_test = loaddata['t_test'] # test target labels
## PARAMETERS
residual_channels = 256
filter_width = 5
dilations = [1, 1, 1, 1, 1, 1, 1]
input_channels = x_test[0][0].shape[2]
no_classes = 3
postnet_channels= 256
S = model_convNet.CNET(name='S',
input_channels=input_channels,
output_channels = no_classes,
import pandas as pd
import numpy as np
import sys
import scipy
from scipy import stats
import hdf5storage
options = hdf5storage.Options(oned_as = 'column', matlab_compatible = True, action_for_matlab_incompatible = 'error')
input_list=["wholebrain_t1t2","wholebrain_dbm"] #MODIFY to match the filenames of the wholebrain files created in your sample_extract*py scripts. should be name of the file without the .mat extension
z_dict = {}
for file in input_list:
fname = file + ".mat"
res = hdf5storage.loadmat(fname) #load raw data
x_z = np.asarray(stats.zscore(res['X'],axis=None)) #zscore, across both subjects and vertices
#x_z_s = x_z - np.min(x_z) #shift so all positive
z_dict[file] = x_z
#concatenate each zscored shifted matrix together
#forms vertex X subject*n_metrics matrix
file=input_list[0]
#print(file)
wb_z_all = z_dict[file]
for file in input_list[1:]:
print(file)
wb_z_all = np.concatenate((wb_z_all, z_dict[file]),axis=1)
wb_z_s_all = wb_z_all - np.min(wb_z_all)
# itgidx
filepath1 = 'itgidx.xlsx'
itgidx = np.array(pd.read_excel(filepath1, header=None))
# signal
loadlist = list(['F:\\Miniscope imaging data\\Analysis\\201808\\GPF201808_#1.4_CtxA\\GPF201808_#1.4_CtxA_day1_SignalMatrix.mat', \
'F:\\Miniscope imaging data\\Analysis\\201808\\GPF201808_#1.4_CtxA\\GPF201808_#1.4_CtxA_day2_SignalMatrix.mat', \
'F:\\Miniscope imaging data\\Analysis\\201808\\GPF201808_#1.4_CtxA\\GPF201808_#1.4_CtxA_day3_SignalMatrix.mat', \
'F:\\Miniscope imaging data\\Analysis\\201808\\GPF201808_#1.4_CtxA\\GPF201808_#1.4_CtxA_day4_SignalMatrix.mat'])
# 추후 수정요망
msPeak_signal = np.zeros((len(loadlist), 185, 8330)) # 추후 수정요망
signal_matrix = np.zeros((itgidx.shape[0], itgidx.shape[1], 8330))
for filename in np.arange(len(loadlist)):
mat_tmp = hdf5storage.loadmat(loadlist[filename])
mat_signal_tmp = mat_tmp['msPeak_signal']
for neuron in np.arange(mat_signal_tmp.shape[0]):
itg_index = list(itgidx[:, filename]).index(neuron +
1) # matlab to python
signal_matrix[itg_index, filename,
0:mat_signal_tmp.shape[1]] = mat_signal_tmp[neuron, :]
if not (filename == 0) and itg_index == 0:
print(filename, neuron, itg_index)
ans = np.sum(signal_matrix, axis=2)
# In[] freezing index load for day3
syn = 14 # notebook 1 frame 때, miniscope frame
tr = 1.3 # time range
# In[255]:
fpo = getpath('ORACLES')
# In[256]:
fpo
# In[263]:
model = hs.loadmat(fpo+'model/v3_ORACLES_lut.mat')
# In[259]:
model.keys()
# In[267]:
1.0/np.cos(model['sza']*np.pi/180.0)
# In[270]:
# In[4]:
fp =getpath('ORACLES')#'C:/Userds/sleblan2/Research/ORACLES/'
fp
# # Load files
# ## Load the 2016 data
# In[5]:
ar6 = hs.loadmat(fp+'/aod_ict/v8/R3/all_aod_ict_R3_2016.mat')
# In[410]:
ar6['flac'] = (ar6['qual_flag']==0)&(ar6['flag_acaod']==1)
ar6['flacr'] = (ar6['qual_flag']==0)&(ar6['flag_acaod']==1)&(ar6['fl_routine'])
ar6['flaco'] = (ar6['qual_flag']==0)&(ar6['flag_acaod']==1)&~(ar6['fl_routine'])
# In[81]:
ar6['flr'] = (ar6['qual_flag']==0) & (ar6['fl_routine'])
ar6['flo'] = (ar6['qual_flag']==0) & ~(ar6['fl_routine'])
import numpy as np
from hdf5storage import loadmat, savemat
from os.path import join, isfile
from scipy.ndimage.morphology import distance_transform_edt as distance_transform
DS_PATH = 'D:/datasets/processed/voc2012'
ds_info = loadmat(join(DS_PATH, 'dataset_info.mat'))
if not isfile('incorr_dist_hist.mat'):
import sys
imset = sys.argv[1].lower().capitalize()
PRED_PATH = join(DS_PATH, 'Deeplab_Prediction', imset)
GT_PATH = join(DS_PATH, 'Truth', imset)
num_img = ds_info['num_'+imset.lower()]
num_classes = 20
bg_class = 0
LOGIT_FMT = imset.lower()+'_%06d_logits.mat'
LOGIT_MAT_NAME = 'logits_img'
GT_FMT = imset.lower()+'_%06d_pixeltruth.mat'
GT_MAT_NAME = 'truth_img'
res = .5
nb = int(50./res)
dist_hist = np.zeros((nb), dtype=np.uint64)
for idx in range(1, num_img+1):
logit_im = loadmat(join(PRED_PATH, LOGIT_FMT % idx))[LOGIT_MAT_NAME][...,1:].reshape(-1, num_classes)
def main():
# ----------------------------------------
# Preparation
# ----------------------------------------
noise_level_img = 7.65 / 255.0 # default: 0, noise level for LR image
noise_level_model = noise_level_img # noise level of model, default 0
model_name = 'drunet_gray' # 'drunet_gray' | 'drunet_color' | 'ircnn_gray' | 'ircnn_color'
testset_name = 'Set3C' # test set, 'set5' | 'srbsd68'
x8 = True # default: False, x8 to boost performance
iter_num = 8 # number of iterations
modelSigma1 = 49
modelSigma2 = noise_level_model * 255.
show_img = False # default: False
save_L = True # save LR image
save_E = True # save estimated image
save_LEH = False # save zoomed LR, E and H images
border = 0
# --------------------------------
# load kernel
# --------------------------------
kernels = hdf5storage.loadmat(os.path.join('kernels',
'Levin09.mat'))['kernels']
sf = 1
task_current = 'deblur' # 'deblur' for deblurring
n_channels = 3 if 'color' in model_name else 1 # fixed
model_zoo = 'model_zoo' # fixed
testsets = 'testsets' # fixed
results = 'results' # fixed
result_name = testset_name + '_' + task_current + '_' + model_name
model_path = os.path.join(model_zoo, model_name + '.pth')
device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
torch.cuda.empty_cache()
# ----------------------------------------
# L_path, E_path, H_path
# ----------------------------------------
L_path = os.path.join(testsets,
testset_name) # L_path, for Low-quality images
E_path = os.path.join(results, result_name) # E_path, for Estimated images
util.mkdir(E_path)
logger_name = result_name
utils_logger.logger_info(logger_name,
log_path=os.path.join(E_path,
logger_name + '.log'))
logger = logging.getLogger(logger_name)
# ----------------------------------------
# load model
# ----------------------------------------
if 'drunet' in model_name:
from models.network_unet import UNetRes as net
model = net(in_nc=n_channels + 1,
out_nc=n_channels,
nc=[64, 128, 256, 512],
nb=4,
act_mode='R',
downsample_mode="strideconv",
upsample_mode="convtranspose")
model.load_state_dict(torch.load(model_path), strict=True)
model.eval()
for _, v in model.named_parameters():
v.requires_grad = False
model = model.to(device)
elif 'ircnn' in model_name:
from models.network_dncnn import IRCNN as net
model = net(in_nc=n_channels, out_nc=n_channels, nc=64)
model25 = torch.load(model_path)
former_idx = 0
logger.info('model_name:{}, image sigma:{:.3f}, model sigma:{:.3f}'.format(
model_name, noise_level_img, noise_level_model))
logger.info('Model path: {:s}'.format(model_path))
logger.info(L_path)
L_paths = util.get_image_paths(L_path)
test_results_ave = OrderedDict()
test_results_ave['psnr'] = [] # record average PSNR for each kernel
for k_index in range(kernels.shape[1]):
logger.info('-------k:{:>2d} ---------'.format(k_index))
test_results = OrderedDict()
test_results['psnr'] = []
k = kernels[0, k_index].astype(np.float64)
util.imshow(k) if show_img else None
for idx, img in enumerate(L_paths):
# --------------------------------
# (1) get img_L
# --------------------------------
img_name, ext = os.path.splitext(os.path.basename(img))
img_H = util.imread_uint(img, n_channels=n_channels)
img_H = util.modcrop(img_H, 8) # modcrop
img_L = ndimage.filters.convolve(img_H,
np.expand_dims(k, axis=2),
mode='wrap')
util.imshow(img_L) if show_img else None
img_L = util.uint2single(img_L)
np.random.seed(seed=0) # for reproducibility
img_L += np.random.normal(0, noise_level_img,
img_L.shape) # add AWGN
# --------------------------------
# (2) get rhos and sigmas
# --------------------------------
rhos, sigmas = pnp.get_rho_sigma(sigma=max(0.255 / 255.,
noise_level_model),
iter_num=iter_num,
modelSigma1=modelSigma1,
modelSigma2=modelSigma2,
w=1.0)
rhos, sigmas = torch.tensor(rhos).to(device), torch.tensor(
sigmas).to(device)
# --------------------------------
# (3) initialize x, and pre-calculation
# --------------------------------
x = util.single2tensor4(img_L).to(device)
img_L_tensor, k_tensor = util.single2tensor4(
img_L), util.single2tensor4(np.expand_dims(k, 2))
[k_tensor, img_L_tensor] = util.todevice([k_tensor, img_L_tensor],
device)
FB, FBC, F2B, FBFy = sr.pre_calculate(img_L_tensor, k_tensor, sf)
# --------------------------------
# (4) main iterations
# --------------------------------
for i in range(iter_num):
# --------------------------------
# step 1, FFT
# --------------------------------
tau = rhos[i].float().repeat(1, 1, 1, 1)
x = sr.data_solution(x, FB, FBC, F2B, FBFy, tau, sf)
if 'ircnn' in model_name:
current_idx = np.int(
np.ceil(sigmas[i].cpu().numpy() * 255. / 2.) - 1)
if current_idx != former_idx:
model.load_state_dict(model25[str(current_idx)],
strict=True)
model.eval()
for _, v in model.named_parameters():
v.requires_grad = False
model = model.to(device)
former_idx = current_idx
# --------------------------------
# step 2, denoiser
# --------------------------------
if x8:
x = util.augment_img_tensor4(x, i % 8)
if 'drunet' in model_name:
x = torch.cat((x, sigmas[i].float().repeat(
1, 1, x.shape[2], x.shape[3])),
dim=1)
x = utils_model.test_mode(model,
x,
mode=2,
refield=32,
min_size=256,
modulo=16)
elif 'ircnn' in model_name:
x = model(x)
if x8:
if i % 8 == 3 or i % 8 == 5:
x = util.augment_img_tensor4(x, 8 - i % 8)
else:
x = util.augment_img_tensor4(x, i % 8)
# --------------------------------
# (3) img_E
# --------------------------------
img_E = util.tensor2uint(x)
if n_channels == 1:
img_H = img_H.squeeze()
if save_E:
util.imsave(
img_E,
os.path.join(
E_path, img_name + '_k' + str(k_index) + '_' +
model_name + '.png'))
# --------------------------------
# (4) img_LEH
# --------------------------------
if save_LEH:
img_L = util.single2uint(img_L)
k_v = k / np.max(k) * 1.0
k_v = util.single2uint(np.tile(k_v[..., np.newaxis],
[1, 1, 3]))
k_v = cv2.resize(k_v, (3 * k_v.shape[1], 3 * k_v.shape[0]),
interpolation=cv2.INTER_NEAREST)
img_I = cv2.resize(img_L,
(sf * img_L.shape[1], sf * img_L.shape[0]),
interpolation=cv2.INTER_NEAREST)
img_I[:k_v.shape[0], -k_v.shape[1]:, :] = k_v
img_I[:img_L.shape[0], :img_L.shape[1], :] = img_L
util.imshow(np.concatenate([img_I, img_E, img_H], axis=1),
title='LR / Recovered / Ground-truth'
) if show_img else None
util.imsave(
np.concatenate([img_I, img_E, img_H], axis=1),
os.path.join(E_path,
img_name + '_k' + str(k_index) + '_LEH.png'))
if save_L:
util.imsave(
util.single2uint(img_L),
os.path.join(E_path,
img_name + '_k' + str(k_index) + '_LR.png'))
psnr = util.calculate_psnr(
img_E, img_H, border=border) # change with your own border
test_results['psnr'].append(psnr)
logger.info('{:->4d}--> {:>10s} --k:{:>2d} PSNR: {:.2f}dB'.format(
idx + 1, img_name + ext, k_index, psnr))
# --------------------------------
# Average PSNR
# --------------------------------
ave_psnr = sum(test_results['psnr']) / len(test_results['psnr'])
logger.info(
'------> Average PSNR of ({}), kernel: ({}) sigma: ({:.2f}): {:.2f} dB'
.format(testset_name, k_index, noise_level_model, ave_psnr))
test_results_ave['psnr'].append(ave_psnr)
s = sio.idl.readsav(fp+'model/sp_v1_20130219_4STAR.out')
print s.keys()
print 'sp', s.sp.shape
print 'sp (wp, wvl, z, re, ta)'
# ### Compare to lut from NAAMES
# In[9]:
fp_lut_mat = 'C:\\Users\\sleblan2\\Research\\NAAMES\\lut\\v2_NAAMES_lut.mat'
print('Loading the lut file:{}'.format(fp_lut_mat))
if not os.path.isfile(fp_lut_mat):
print('File {} does not exist'.format(fp_lut_mat))
raise IOError('LUT File not found: {}'.format(fp_lut_mat))
luts = hs.loadmat(fp_lut_mat)
# In[22]:
luts.keys()
# In[23]:
luts['irr_up'].shape
# In[27]:
luts['rad'].shape, s.sp.shape
# In[4]:
dds = ['20151104','20151109','20151112','20151114','20151117','20151118','20151123']
# In[5]:
rts = []
sps = []
# In[6]:
for daystr in dds:
print daystr
rt = hs.loadmat(fp+'{}_zen_cld_retrieved.mat'.format(daystr))
s = sio.loadmat(fp+'{}starzen.mat'.format(daystr))
sp = Sp.Sp(s)
rts.append(rt)
sps.append(sp)
# # Start plotting the results
# In[7]:
rt.keys()
# In[8]:
