def _roll_standardize(data, n):
arr = np.array(data)
value = [ut.standardize(arr[i - n:i])[-1] for i in range(n, len(arr))]
value = np.concatenate([[0] * n, value])
ss = pd.Series(value).ffill().fillna(0)
value = ss.values
return value.astype(float)
def construct_raicar_components(self):
'''
Averages the aligned ICA runs and calculates the reproducibility for each component. avgMethod and
canonSigns controls the method of component formation and reproducibility indices calculated.
'''
if not os.path.exists(self.racDirectory):
try:
os.mkdir(self.racDirectory)
except OSError:
pass
alnFiles = glob.glob(os.path.join(self.alnDirectory, 'alnRun_*.h5'))
if len(alnFiles) == 0:
print 'ERROR : Components have not been aligned yet.'
return
# temp variables to hold the answer
gc.collect()
raicarSources = []
raicarMixing = []
repro = []
for f in alnFiles:
print 'Constructing raicar comfponent from file %s' % f
fPtr = tb.open_file(f, 'r')
sc = fPtr.get_node('/aligned/sources').read()
ac = fPtr.get_node('/aligned/mixing').read()
fPtr.close()
if self.canonSigns:
sc, ac = self.canonicalize_signs(sc, ac)
methodToUse = self.avgMethod + '_average_aligned_runs'
avgSource, avgMix, rep = getattr(self, methodToUse)(sc, ac)
raicarSources.append(avgSource)
raicarMixing.append(avgMix)
repro.append(rep)
# collapse and make a component
self.raicarSources = np.vstack(raicarSources)
self.raicarMixing = np.vstack(raicarMixing).T
self.reproducibility = repro
# adjust std. dev. of RAICAR sources
self.raicarSources = standardize(self.raicarSources, stdtype='row')
# save the result, PyTables again
h5Ptr = tb.open_file(os.path.join(self.racDirectory, 'components.h5'),
mode="w",
title='RAICAR Component')
raicar = h5Ptr.create_group(h5Ptr.root, 'raicar', 'RAICAR Component')
h5Ptr.create_array(raicar, 'sources', self.raicarSources, "S")
h5Ptr.create_array(raicar, 'mixing', self.raicarMixing, "A")
h5Ptr.close()
# this can just be pickled - it's not that large
fPtr = open(os.path.join(self.racDirectory, 'reproducibility.db'),
'wb')
cPickle.dump(self.reproducibility, fPtr, protocol=-1)
fPtr.close()
def construct_raicar_components(self):
'''
Averages the aligned ICA runs and calculates the reproducibility for each component. avgMethod and
canonSigns controls the method of component formation and reproducibility indices calculated.
'''
if not os.path.exists(self.racDirectory):
try:
os.mkdir(self.racDirectory)
except OSError:
pass
alnFiles = glob.glob(os.path.join(self.alnDirectory,'alnRun_*.h5'))
if len(alnFiles) == 0:
print 'ERROR : Components have not been aligned yet.'
return
# temp variables to hold the answer
gc.collect()
raicarSources = []
raicarMixing = []
repro = []
for f in alnFiles:
print 'Constructing raicar comfponent from file %s' % f
fPtr = tb.open_file(f,'r')
sc = fPtr.get_node('/aligned/sources').read()
ac = fPtr.get_node('/aligned/mixing').read()
fPtr.close()
if self.canonSigns:
sc,ac = self.canonicalize_signs(sc,ac)
methodToUse = self.avgMethod+'_average_aligned_runs'
avgSource,avgMix,rep = getattr(self,methodToUse)(sc,ac)
raicarSources.append(avgSource)
raicarMixing.append(avgMix)
repro.append(rep)
# collapse and make a component
self.raicarSources = np.vstack(raicarSources)
self.raicarMixing = np.vstack(raicarMixing).T
self.reproducibility = repro
# adjust std. dev. of RAICAR sources
self.raicarSources = standardize(self.raicarSources,stdtype='row')
# save the result, PyTables again
h5Ptr = tb.open_file(os.path.join(self.racDirectory,'components.h5'),mode="w",title='RAICAR Component')
raicar = h5Ptr.create_group(h5Ptr.root,'raicar','RAICAR Component')
h5Ptr.create_array(raicar,'sources',self.raicarSources,"S")
h5Ptr.create_array(raicar,'mixing',self.raicarMixing,"A")
h5Ptr.close()
# this can just be pickled - it's not that large
fPtr = open(os.path.join(self.racDirectory,'reproducibility.db'),'wb')
cPickle.dump(self.reproducibility,fPtr,protocol=-1)
fPtr.close()
def PreparingData(self):
if self._status == "l":
my_shelve = shelve.open(self.__filename)
return self.__filename
elif self._status == "s":
X_train, labels_train, list_ch_train = ut.read_data(
data_path=self.__pathDS, split="train") # train
X_test, labels_test, list_ch_test = ut.read_data(
data_path=self.__pathDS, split="test") # test
features_train = ut.read_Features(data_path=self.__pathDS,
split="train") # features train
features_test = ut.read_Features(data_path=self.__pathDS,
split="test") # features train
assert list_ch_train == list_ch_test, "Mistmatch in channels!"
# Normalize?
X_train, X_test = ut.standardize(X_train, X_test)
# X_tr, X_vld, lab_tr, lab_vld = train_test_split(X_train, labels_train,
# stratify=labels_train, random_state=123)
# One-hot encoding:
y_tr = ut.one_hot(labels_train)
# y_vld = ut.one_hot(lab_vld)
y_test = ut.one_hot(labels_test)
my_shelve = shelve.open(self.__filename, 'n')
my_shelve['data_train'] = X_train
# my_shelve['data_vld'] = X_vld
my_shelve['data_test'] = X_test
my_shelve['labels_train'] = y_tr
my_shelve['labels_test'] = y_test
# my_shelve['labels_vld'] = y_vld
my_shelve['labels_test'] = y_test
my_shelve['features_train'] = features_train
my_shelve['features_test'] = features_test
return self.__filename
def construct_bicar_components(self):
'''
Averages the aligned ICA runs (both temporal and spatial) and calculates the reproducibility
for each component. avgMethod and canonSigns controls the method of component formation and
reproducibility indices calculated.
'''
if not os.path.exists(self.racDirectory):
try:
os.mkdir(self.racDirectory)
except OSError:
pass
# have to do both temporal and spatial
tAlnFiles = glob.glob(os.path.join(self.alnDirectory, 'alnRun_t_*.h5'))
sAlnFiles = glob.glob(os.path.join(self.alnDirectory, 'alnRun_s_*.h5'))
if len(tAlnFiles) == 0 or len(sAlnFiles) == 0:
if self.reportLevel > 0:
print 'ERROR : Components have not been aligned yet.'
return
# temp variables to hold the answer
raicarSources = []
raicarMixing = []
repro = []
for f in tAlnFiles:
if self.reportLevel > 1:
print 'Constructing temporal bicar component from file %s' % f
fPtr = tb.openFile(f, 'r')
sc = fPtr.getNode('/aligned/sources').read()
ac = fPtr.getNode('/aligned/mixing').read()
fPtr.close()
if self.canonSigns:
sc, ac = self.canonicalize_signs(sc, ac)
methodToUse = self.avgMethod + '_average_aligned_runs'
avgSource, avgMix, rep = getattr(self, methodToUse)(sc, ac)
raicarSources.append(avgSource)
raicarMixing.append(avgMix)
repro.append(rep)
# collapse and make a component
self.temporalSources = np.vstack(raicarSources)
self.temporalMixing = np.vstack(raicarMixing).T
self.reproducibility = repro
# adjust std. dev. of RAICAR sources
self.temporalSources = standardize(self.temporalSources, stdtype='row')
# save the result, PyTables again
h5Ptr = tb.openFile(os.path.join(self.racDirectory,
'temporal_components.h5'),
mode="w",
title='RAICAR Component')
bicar = h5Ptr.createGroup(h5Ptr.root, 'bicar', 'RAICAR Component')
h5Ptr.createArray(bicar, 'sources', self.temporalSources, "S")
h5Ptr.createArray(bicar, 'mixing', self.temporalMixing, "A")
h5Ptr.close()
# repeat the whole thing for the spatial sources
raicarSources = []
raicarMixing = []
repro = []
for f in sAlnFiles:
if self.reportLevel > 1:
print 'Constructing spatial bicar component from file %s' % f
fPtr = tb.openFile(f, 'r')
sc = fPtr.getNode('/aligned/sources').read()
ac = fPtr.getNode('/aligned/mixing').read()
fPtr.close()
if self.canonSigns:
sc, ac = self.canonicalize_signs(sc, ac)
methodToUse = self.avgMethod + '_average_aligned_runs'
avgSource, avgMix, rep = getattr(self, methodToUse)(sc, ac)
raicarSources.append(avgSource)
raicarMixing.append(avgMix)
repro.append(rep)
# collapse and make a component
self.spatialSources = np.vstack(raicarSources)
self.spatialMixing = np.vstack(raicarMixing).T
# average reproducibility
for i in range(0, len(self.reproducibility)):
self.reproducibility[
i] = 0.5 * self.reproducibility[i] + 0.5 * repro[i]
# save the result, PyTables again
h5Ptr = tb.openFile(os.path.join(self.racDirectory,
'spatial_components.h5'),
mode="w",
title='RAICAR Component')
bicar = h5Ptr.createGroup(h5Ptr.root, 'bicar', 'RAICAR Component')
h5Ptr.createArray(bicar, 'sources', self.spatialSources, "S")
h5Ptr.createArray(bicar, 'mixing', self.spatialMixing, "A")
h5Ptr.close()
# this can just be pickled - it's not that large
fPtr = open(os.path.join(self.racDirectory, 'reproducibility.db'),
'wb')
cPickle.dump(self.reproducibility, fPtr, protocol=-1)
fPtr.close()
from Create_rnd_Net_initializeALL import Create_rnd_Net
#%%initialize
trn_acc_2rnd = np.zeros((num_epochs, num_trials))
vld_acc_2rnd = np.zeros((num_epochs, num_trials))
trn_acc_1Bsg = np.zeros((num_epochs, num_trials, num_Fuse))
vld_acc_1Bsg = np.zeros((num_epochs, num_trials, num_Fuse))
trn_acc_1rnd = np.zeros((num_epochs, num_trials, num_Fuse))
vld_acc_1rnd = np.zeros((num_epochs, num_trials, num_Fuse))
#%% Prepare Data
num_classes = 10
# # Loading train set and test set
x_train, x_test, label_train, label_test = get_train_test()
x_train, x_test = standardize(x_train, x_test)
# One-hot encoding:
y_train = to_categorical(label_train)
y_test = to_categorical(label_test)
x_train = x_train.astype('float32')
x_test = x_test.astype('float32')
#z-score
mean = np.mean(x_train, axis=(0, 1, 2))
std = np.std(x_train, axis=(0, 1, 2))
x_train = (x_train - mean) / (std + 1e-7)
x_test = (x_test - mean) / (std + 1e-7)
from build_234layer_1dconv_graph_he import build_234layer_1dconv_graph
else:
from build_234layer_1dconv_graph_rnd import build_234layer_1dconv_graph
par = {'batch_size':batch_size,'seq_len':seq_len,'lrn_rate':lrn_rate,'epochs':epochs,
'krnl_sz':krnl_sz,'krnl_sz_Bsg':krnl_sz_Bsg,'L':L,'K':K,'n_classes':n_classes,'n_channels':n_channels
,'n_outchannel':N,'Spool':Spool,'Sconv':Sconv,'num_trials':num_trials,'act_func':act_func}
#%% Prepare data
X_train, labels_train, list_ch_train = read_data(data_path="data/", split="train") # train
X_test, labels_test, list_ch_test = read_data(data_path="data/", split="test") # test
assert list_ch_train == list_ch_test, "Mistmatch in channels!"
# Normalize?
X_train, X_test = standardize(X_train, X_test)
# Train/Validation Split
X_tr, X_vld, lab_tr, lab_vld = train_test_split(X_train, labels_train,
stratify = labels_train, random_state = 123)
# One-hot encoding:
y_tr = one_hot(lab_tr)
y_vld = one_hot(lab_vld)
y_test = one_hot(labels_test)
X_tr = X_tr[:,:,0:n_channels]
X_test = X_test[:,:,0:n_channels]
X_vld = X_vld[:,:,0:n_channels]
#%% initialization: