def calculate_loss_tensor(filename,
Total_Windows,
W,
signals_models,
signals=None,
noisy_index=None):
n_windows = 250
windows = np.arange(int(Total_Windows / n_windows))
N_Windows = len(windows)
N_Models = len(signals_models)
Total_Windows = int(N_Windows * n_windows)
N_Signals = len(signals)
loss_tensor = np.zeros((N_Models, N_Signals, Total_Windows))
X_matrix = np.zeros((N_Signals, Total_Windows, W))
Y_matrix = np.zeros((N_Signals, Total_Windows, W))
i = 0
first_test_index = int(len(signals[0]) * 0.33)
for signal in signals:
signal_test = segment_signal(signal[first_test_index:], 256, 0.33)
X_matrix[i, :, :], Y_matrix[i, :, :] = randomize_batch(
signal_test[0], signal_test[1], Total_Windows)
i += 1
print("Loading model...")
model_info = signals_models[0]
signal2Model = Signal2Model(model_info.dataset_name,
model_info.directory,
signal_dim=model_info.Sd,
hidden_dim=model_info.Hd,
mini_batch_size=n_windows)
model = DeepLibphys.models.LibphysMBGRU.LibphysMBGRU(signal2Model)
for m in range(len(signals_models)):
model_info = signals_models[m]
model.model_name = model_info.dataset_name
model.load(dir_name=model_info.directory)
print("Processing Model " + model_info.name)
for s in range(N_Signals):
print("Calculating loss for ECG " + str(s + 1), end=';\n ')
for w in windows:
index = w * n_windows
x_test = X_matrix[s, index:index + n_windows, :]
y_test = Y_matrix[s, index:index + n_windows, :]
loss_tensor[m, s, index:index + n_windows] = np.asarray(
model.calculate_mse_vector(x_test, y_test))
np.savez(filename + ".npz",
loss_tensor=loss_tensor,
signals_models=signals_models)
return loss_tensor
def start(s):
signal_dim = 32
hidden_dim = 32
mini_batch_size = 8
batch_size = 128
window_size = 256
save_interval = 250
mean_tol = 0.4
std_tol = 0.4
# signal_directory = 'SIGNAL_ECG_RR_1024_256'.format(256, window_size)
signal_directory = 'SIGNAL_ECG_RR'.format(256, window_size)
raw_filenames, Ns, core_names = get_processing_variables()
# process_and_save_signals(raw_filenames, core_names, Ns, indexes2process=np.array([3]))#np.arange(45, len(raw_filenames)))
# exit()
processed_filenames = np.array([
'../data/processed/MIT/{0}[256].npz'.format(core_name)
for core_name in core_names
])
ind = [3]
for i, filename in enumerate(processed_filenames[ind]):
signal, core_name = np.load(filename)["signal"], np.load(
filename)["core_name"]
running_ok = False
signal2model = Signal2Model(core_name,
signal_directory,
signal_dim=signal_dim,
number_of_epochs=10000,
hidden_dim=hidden_dim,
learning_rate_val=0.01,
batch_size=batch_size,
mini_batch_size=mini_batch_size,
window_size=window_size + 1,
save_interval=save_interval,
lower_error=1e-10,
count_to_break_max=5,
n_signals=2)
[x_train,
y_train], [xRR,
yRR] = prepare_for_RR_data([signal[:int(len(signal) / 3)]],
signal2model,
overlap=0.11)
indexes = range(len(x_train) - (len(x_train) % mini_batch_size))
signal2model.batch_size = len(indexes)
signal2model.window_size = window_size
print("Compiling Model {0}".format(signal2model.model_name))
model = GRU.LibphysMultisignalGRU(signal2model)
# model.load(signal2model.signal_directory, model.get_file_tag(-5, -5))
returned = model.train(x_train[indexes], y_train[indexes],
xRR[indexes], yRR[indexes])
def train_fantasia(hidden_dim, mini_batch_size, batch_size, window_size, signal_directory, indexes, signals,save_interval,signal_dim):
models = db.resp_64_models
for i, signal, model_info in zip(indexes, signals, models):
name = 'resp_' + str(i)
signal2model = Signal2Model(name, signal_directory, signal_dim=signal_dim, hidden_dim=hidden_dim, batch_size=batch_size,
mini_batch_size=mini_batch_size, window_size=window_size,
save_interval=save_interval)
print("Compiling Model {0}".format(name))
model = DeepLibphys.models.LibphysMBGRU.LibphysMBGRU(signal2model)
print("Initiating training... ")
model.load(dir_name=model_info.directory)
model.train(signal, signal2model)
def try_calculate_loss_tensor(filename,
Total_Windows,
W,
signals_models,
signals=None,
noisy_index=None):
print("Loading model...")
n_windows = 250
modelx_info = signals_models[0]
signal2Model = Signal2Model(modelx_info.dataset_name,
modelx_info.directory,
signal_dim=modelx_info.Sd,
hidden_dim=modelx_info.Hd,
mini_batch_size=n_windows)
model = DeepLibphys.models.LibphysMBGRU.LibphysMBGRU(signal2Model)
windows = np.arange(int(Total_Windows / n_windows))
N_Windows = len(windows)
N_Models = len(signals_models)
Total_Windows = int(N_Windows * n_windows)
N_Signals = len(signals)
loss_tensor = np.zeros((N_Models, N_Signals, Total_Windows))
X_matrix = np.zeros((N_Signals, Total_Windows, W))
Y_matrix = np.zeros((N_Signals, Total_Windows, W))
i = 0
first_test_index = int(len(signals[0]) * 0.33)
for signal in signals:
signal_test = segment_signal(signal[first_test_index:], 256, 0.33)
X_matrix[i, :, :], Y_matrix[i, :, :] = randomize_batch(
signal_test[0], signal_test[1], Total_Windows)
i += 1
pool = Pool(8)
pool.starmap(
interate_loss_calculus,
zip(signals_models, range(len(signals_models)), repeat(model),
repeat(X_matrix), repeat(Y_matrix), repeat(n_windows),
repeat(windows), repeat(N_Signals), repeat(loss_tensor)))
if not os.path.isdir(os.path.dirname(filename + ".npz")):
os.mkdir(os.path.dirname(filename + ".npz"))
np.savez(filename + ".npz",
loss_tensor=loss_tensor,
signals_models=signals_models)
return loss_tensor
def create_history(subject):
model_info = db.ecg_64_models[subject]
signal2Model = Signal2Model(model_info.dataset_name,
model_info.directory,
signal_dim=model_info.Sd,
hidden_dim=model_info.Hd)
model = GRU.LibphysSGDGRU(signal2Model)
# plt.ion()
# font = {'family': 'lato',
# 'weight': 'bold',
# 'size': 40}
#
# matplotlib.rc('font', **font)
x = [0]
i = 0
window_size = 512
fz = 250
N = 3000
titles = []
signals = []
for epoch in [-1, 20, 30, 50, 80, 410]:
signal = [np.random.randint(0, 63, size=1)[0]]
if epoch < 0:
file_tag = model.get_file_tag(epoch, epoch)
else:
file_tag = model.get_file_tag(0, epoch)
model.load(file_tag=file_tag, dir_name=model_info.directory)
print("Processing epoch " + str(epoch))
for i in range(N):
# print(i)
sample, _ = model.generate_online_predicted_signal(
signal, window_size, uncertaintly=0.01)
signal.append(sample)
x.append((i + 1) / fz)
titles.append("Epoch {0}".format(epoch))
signals.append(np.array(signal))
np.savez("../data/history_of_{0}.npz".format(subject),
titles=titles,
signals=signals,
t=x)
return [titles, signals, x]
def synthetize(models,
uncertaintly=0.01,
filename="synthesized_signals_1024.npz"):
mini_bs = 256
plt.ion()
signals = []
signal2Model = Signal2Model(models[0].dataset_name,
models[0].directory,
signal_dim=models[0].Sd,
hidden_dim=models[0].Hd,
mini_batch_size=mini_bs)
model = GRU.LibphysMBGRU(signal2Model)
for model_info in models:
model.model_name = model_info.dataset_name
model.load(dir_name=model_info.directory)
print("Processing " + model_info.name)
# plt.ion()
# font = {'family': 'lato',
# 'weight': 'bold',
# 'size': 40}
#
# matplotlib.rc('font', **font)
x = [0]
i = 0
signal = np.random.randint(0, 63, size=(mini_bs, 1), dtype=np.int32)
window_size = 512
fz = 250
N = 512
# while True:
for i in range(N):
# print(i)
y = model.generate_online_predicted_vector(
signal, window_size, uncertaintly=uncertaintly)
signal.append(y)
plt.clf()
plt.plot(signal)
plt.pause(0.05)
signals.append(np.array(signal))
np.savez("img/" + model_info.dataset_name + ".npz",
synth_signal=signal,
probability=prob)
np.savez(filename, synth_signals=signals, probabilities=probabilities)
def train_FMH(hidden_dim, mini_batch_size, batch_size, window_size,
signal_directory, indexes, signals, save_interval, signal_dim):
for i, signal in zip(indexes, signals[indexes]):
name = 'emg_' + str(i + 1)
signal2model = Signal2Model(name,
signal_directory,
signal_dim=signal_dim,
hidden_dim=hidden_dim,
batch_size=batch_size,
mini_batch_size=mini_batch_size,
window_size=window_size,
save_interval=save_interval,
tolerance=1e-12)
returned = False
while not returned:
print("Compiling Model {0}".format(name))
model = DeepLibphys.models.LibphysMBGRU.LibphysMBGRU(signal2model)
print("Initiating training... ")
returned = model.train(signal, signal2model, overlap=0.05)
def calculate_loss_tensor(filename,
signals_models=[],
test_signals=None,
labels=None):
X_Windows = test_signals[:, :-1]
Y_Windows = test_signals[:, 1:]
N_Signals = np.shape(X_Windows)[0]
n_windows = np.shape(X_Windows)[0]
print("Loading model...")
model_info = signals_models[0]
signal2Model = Signal2Model(model_info.dataset_name,
model_info.directory,
signal_dim=model_info.Sd,
hidden_dim=model_info.Hd,
mini_batch_size=n_windows)
model = GRU.LibphysMBGRU(signal2Model)
loss_tensor = np.zeros((len(signals_models), N_Signals))
for m in range(len(signals_models)):
model_info = signals_models[m]
model.model_name = model_info.dataset_name
model.load(dir_name=model_info.directory)
print("Processing " + model_info.name)
# for s in range(N_Signals):
# if labels is not None:
# print("Calculating loss for " + labels[s], end=';\n ')
# else:
# print("Calculating loss for " + signals_models[s].name, end=';\n ')
loss_tensor[m, :] = np.asarray(
model.calculate_mse_vector(X_Windows, Y_Windows))
print(np.mean(loss_tensor[m, :]))
np.savez(filename + ".npz",
loss_tensor=loss_tensor,
signals_models=signals_models)
return loss_tensor
# print("Saving signals...")
#
# np.savez(noise_filename,
# processed_noise_array=processed_noise_array, SNRs = SNRs)
print("Loading signals...")
# noise_filename = "../data/ecg_noisy_signals.npz"
processed_noise_array = load_noisy_fantasia_signals(SNRx=[6, 9])
# processed_noise_array, SNRs = npzfile["processed_noise_array"], npzfile["SNRs"]
# snr1, snr2 = 7, 5
# SNRs = [8]#, SNRs[5]]
signal2model = Signal2Model("",
signal_directory,
signal_dim=signal_dim,
hidden_dim=hidden_dim,
batch_size=batch_size,
mini_batch_size=mini_batch_size,
window_size=window_size)
model = DeepLibphys.models.LibphysMBGRU.LibphysMBGRU(signal2model)
SNRs = ["RAW"] + [str(i) for i in range(9, 6, -1)]
for SNR, signals_with_noise in zip(SNRs, processed_noise_array):
for i, signal in zip(range(len(signals_with_noise)),
signals_with_noise):
running_ok = False
if SNR is not "RAW":
u = i + 1
# if np.logical_and(SNR == 9, u > 4) and np.logical_and(SNR == 9, u < 21):
name = 'ecg_' + str(u) + '_SNR_' + str(SNR)
print(name)
def train_fantasia(hidden_dim, mini_batch_size, batch_size, window_size,
signal_directory, indexes, signals, save_interval,
signal_dim):
for i, signal in zip(indexes, signals[indexes]):
name = 'ecg_' + str(i + 1)
signal2model = Signal2Model(name,
signal_directory,
signal_dim=signal_dim,
hidden_dim=hidden_dim,
batch_size=batch_size,
mini_batch_size=mini_batch_size,
window_size=window_size,
save_interval=save_interval,
lower_error=3e-5,
lower_learning_rate=1e-4,
count_to_break_max=30)
print("Compiling Model {0}".format(name))
last_index = int(len(signal) * 0.33)
x_train, y_train = prepare_test_data([signal[22500:last_index]],
signal2model,
mean_tol=0.9,
std_tol=0.5)
# fig, ax = plt.subplots()
# plt.subplots_adjust(bottom=0.2)
# l, = plt.plot(x_train[0], lw=2)
#
# class BooleanSwitcher(object):
# indexes = []
# ind = 0
#
# def yes(self, event):
# if self.ind < len(x_train):
# self.indexes.append(self.ind)
# self.ind += 1
# if self.ind < len(x_train):
# l.set_ydata(x_train[self.ind])
# plt.draw()
# else:
# self.crop()
# plt.close()
#
# def no(self, event):
# self.ind += 1
# if self.ind < len(x_train):
# l.set_ydata(x_train[self.ind])
# plt.draw()
# else:
# self.crop()
# plt.close()
#
# def crop(self):
# c = len(self.indexes) % 16
# self.indexes = self.indexes[:(len(self.indexes) - c)]
# callback = BooleanSwitcher()
# axprev = plt.axes([0.7, 0.05, 0.1, 0.075])
# axnext = plt.axes([0.81, 0.05, 0.1, 0.075])
# by = Button(axnext, 'Yes')
# by.on_clicked(callback.yes)
# bn = Button(axprev, 'No')
# bn.on_clicked(callback.no)
# plt.show()
model = GRU.LibphysMBGRU(signal2model)
# try:
#
# # if i < 20:
# # old_directory = "CLEAN_ECG_BIOMETRY[128.1024]"
# # old_name = 'clean_ecg' + str(i+1)
# # else:
# old_directory = "BIOMETRY[256.1024]"
# old_name = name
#
# old_tag= 'GRU_{0}[{1}.{2}.{3}.{4}.{5}]'. \
# format(old_name, signal_dim, hidden_dim, -1, -5, -5)
# model.load(old_tag, old_directory)
# except:
# pass
print("Initiating training... ")
model.model_name = 'ecg_' + str(i + 1)
model.start_time = time.time()
# returned = model.train_model(x_train[callback.indexes], y_train[callback.indexes], signal2model)
model.load(model.get_file_tag(), signal_directory)
returned = model.train_model(x_train, y_train, signal2model)
if returned:
model.save(signal2model.signal_directory,
model.get_file_tag(-5, -5))
mini_batch_size = 16
window_size = 256
# number_of_epochs = 1000000
#Noisy signals
for noisy_index in [2]:#range(3,5):
signals_tests = db.ecg_noisy_signals[noisy_index]
signals_models = db.signal_models
#
# # Load signals from database
signals = get_signals_tests(signals_tests, signals_models[0].Sd, type="ecg noise", noisy_index=noisy_index)
# train each signal from fantasia
for i in range(9, 19):
name = 'bio_noise_'+str(noisy_index)+'_ecg_' + str(i)
signal = Signal2Model(name, signal_directory, batch_size=batch_size)
model = GRU.LibPhys_GRU(signal_dim=signal_dim, hidden_dim=hidden_dim, signal_name=name, n_windows=mini_batch_size)
model.save(signal_directory, model.get_file_tag(-1, -1))
model.train_signals(signals[0][i], signals[1][i], signal, decay=0.95, track_loss=False)
# Normal + noisy ECGs
signal_dim = 64
hidden_dim = 256
signal_directory = 'BIOMETRIC_ECGs_[20.256]'
n_for_each = 16
mini_batch_size = n_for_each
signals_models = db.signal_models
signals_with_noise = [get_signals_tests(db.ecg_noisy_signals[noisy_index-1], signals_models[0].Sd, type="ecg noise",
noisy_index=noisy_index) for noisy_index in range(1,5)]
def calculate_loss_tensor(filename,
Total_Windows,
W,
signals_models,
signals=None,
overlap=0.33):
if Total_Windows > 256 * 4:
n_windows = 256
elif Total_Windows > 256:
n_windows = int(Total_Windows / 4)
else:
n_windows = Total_Windows
windows = np.arange(0, Total_Windows - n_windows + 1, n_windows)
N_Windows = len(windows)
N_Models = len(signals_models)
Total_Windows = int(N_Windows * n_windows)
N_Signals = len(signals)
loss_tensor = np.zeros((N_Models, N_Signals, Total_Windows))
X_matrix = np.zeros((N_Signals, Total_Windows, W))
Y_matrix = np.zeros((N_Signals, Total_Windows, W))
print("mini_batch: {0}, Total: {1}".format(n_windows, Total_Windows))
i = 0
for signal in signals:
signal_test = segment_signal(signal[int(len(signal) * 0.33):], W,
overlap)
X_matrix[i, :, :], Y_matrix[i, :, :] = randomize_batch(
signal_test[0], signal_test[1], Total_Windows)
i += 1
print("Loading model...")
model_info = signals_models[0]
signal2Model = Signal2Model(model_info.dataset_name,
model_info.directory,
signal_dim=model_info.Sd,
hidden_dim=model_info.Hd,
mini_batch_size=n_windows)
model = DeepLibphys.models.LibphysMBGRU.LibphysMBGRU(signal2Model)
for m, model_info in zip(range(len(signals_models)), signals_models):
model.model_name = model_info.dataset_name
model.load(dir_name=model_info.directory)
print("Processing Model " + model_info.name)
for s in range(N_Signals):
print("Calculating loss for Signal " + str(s + 1), end=';\n ')
for w in windows:
# index = w * n_windows
x_test = X_matrix[s, w:w + n_windows, :]
y_test = Y_matrix[s, w:w + n_windows, :]
loss_tensor[m, s, w:w + n_windows] = np.asarray(
model.calculate_mse_vector(x_test, y_test))
# loss_tensor[m, s, w:w + n_windows] = np.asarray(model.calculate_loss_vector(x_test, y_test))
# if not os.path.isdir(os.path.dirname(filename + ".npz")):
# os.mkdir(os.path.dirname(filename + ".npz"))
np.savez(filename + ".npz",
loss_tensor=loss_tensor,
signals_models=signals_models)
return loss_tensor
# example_index = [7]
hidden_dim = 1048
signal_name = "ecg_7"
signal_directory = 'ECGs_FANTASIA_[256.256]'
batch_size = 32
mini_batch_size = 8
window_size = 256
# number_of_epochs = 1000000
signals_tests = db.signal_tests
signals_models = db.signal_models
signal2model = Signal2Model(signal_name,
signal_directory,
hidden_dim=hidden_dim,
mini_batch_size=mini_batch_size,
window_size=window_size,
batch_size=batch_size,
save_interval=10000,
learning_rate_val=0.01)
model = RGRU.LibPhys_RGRU("RGRU_",
hidden_dim=hidden_dim,
mini_batch_dim=mini_batch_size)
signals = get_signals_tests(signals_tests,
signals_models[0].Sd,
index=7,
regression=True)
plt.plot(signals[0][0])
plt.show()
model.train_signal(signals[0][0],
signals[1][0],
# for i in range(2, 19):
# name = 'biometry_with_noise_' + str(i)
# signal2model = Signal2Model(name, signal_directory, mini_batch_size=mini_batch_size)
# model = GRU.LibphysMBGRU(signal2model)
# if i==2:
# model.load(dir_name=signal_directory, file_tag=model.get_file_tag(0,1000))
#
# model.train_block(signals[i], signal2model, n_for_each=n_for_each, loss_interval=1)
signal_dim = 64
hidden_dim = 256
mini_batch_size = 15
batch_size = 150
n_for_each = 150 / 5
signal_directory = 'NOISE_ECGs_[{0}.{1}]'.format(batch_size, window_size)
noise_filename = "../data/ecg_noisy_signals.npz"
npzfile = np.load(noise_filename)
processed_noise_array, SNRs = npzfile["processed_noise_array"], npzfile["SNRs"]
ecgs = np.load("signals_without_noise.npz")['signals_without_noise']
z = 0
for i, ecg in zip(range(z, len(ecgs)), ecgs[z:]):
name = 'noisy_ecg_' + str(i + 1)
signals = [ecg] + [pna[i] for pna in processed_noise_array]
signal2model = Signal2Model(name,
signal_directory,
mini_batch_size=mini_batch_size)
model = GRU.LibphysMBGRU(signal2model)
model.train_block(signals, signal2model, n_for_each=n_for_each)
# train(ecg, signal2model)
def filter_loss_tensor(signals,
loss_tensor,
all_models_info,
W,
min_windows=512,
overlap=0.33,
max_tol=0.8,
std_tol=0.5,
batch_percentage=0,
already_cut=False):
model_indexes = []
list_of_windows_indexes = []
number_of_windows = []
do_it = True
try:
signals[0][0][0]
except:
do_it = False
for i, model_info in enumerate(all_models_info):
signal2model = Signal2Model(model_info.dataset_name,
"",
signal_dim=model_info.Sd,
hidden_dim=model_info.Hd,
batch_size=np.shape(loss_tensor)[2],
window_size=W)
if do_it:
s = signals[i]
else:
s = [signals[i]]
indexes, _, _, _, _ = get_clean_indexes(s,
signal2model,
overlap=overlap,
max_tol=max_tol,
std_tol=std_tol,
already_cut=already_cut)
indexes = np.array(indexes)
if batch_percentage > 0:
all_indexes, _, _, _, _ = get_clean_indexes(signals[i],
signal2model,
overlap=overlap,
max_tol=0.7)
all_indexes = all_indexes[int(len(all_indexes) *
batch_percentage):]
new_indexes = []
for ind in indexes[indexes >= all_indexes[0]]:
if np.any(all_indexes == ind):
new_indexes.append(ind)
indexes = new_indexes
if len(indexes) >= min_windows + 20:
model_indexes.append(i)
list_of_windows_indexes.append(indexes)
number_of_windows.append(len(indexes))
li = 0
new_models_indexes = []
indexes_2_remove = []
Total = min(number_of_windows)
loss_list = np.zeros((len(model_indexes), len(model_indexes), Total - 1))
for i, inds in zip(model_indexes, list_of_windows_indexes):
first_index = 0
if len(inds) > Total + 20:
first_index = np.random.random_integers(0,
len(inds) - Total - 20,
1)[0]
end_index = first_index + Total - 1
if inds[end_index] > np.shape(loss_tensor)[2]:
first_index = 0
end_index = first_index + Total - 1
if inds[end_index] < np.shape(loss_tensor)[2]:
L = loss_tensor[i][model_indexes][:, inds[first_index:end_index]]
if np.shape(L)[1] > 0:
loss_list[li] = L
new_models_indexes.append(i)
if np.alltrue(loss_list[li] == np.zeros_like(loss_list[li])):
indexes_2_remove.append(li)
li += 1
if indexes_2_remove != []:
mask = mask_without_indexes(loss_list, indexes_2_remove)
loss_list = loss_list[mask][:, mask]
model_indexes = new_models_indexes
print("Rejected {0} people, total of windows = {1}".format(
len(all_models_info) - len(model_indexes), Total))
if model_indexes != []:
return loss_list, np.array(all_models_info)[np.array(model_indexes)]
else:
print("All models were rejected")
return [], []
batch_size = 128
window_size = 512
save_interval = 10000
signal_directory = "ECG_CLUSTER[256.512]"
print("Loading signals...")
mit_sinus = np.load('../data/processed/biometry_mit_sinus[256].npz')['signals']
mit_long_term = np.load(
'../data/processed/biometry_mit_long_term[256].npz')['signals']
cibhi_1, cibhi_2 = np.load('../data/processed/biometry_cybhi[256].npz')['train_signals'], \
np.load('../data/processed/biometry_cybhi[256].npz')['test_signals']
fantasia = np.load("../data/processed/FANTASIA_ECG[256].npz")['x_train']
signal2model = Signal2Model('ecg_26_SNR_12',
signal_directory,
signal_dim=signal_dim,
hidden_dim=hidden_dim,
batch_size=batch_size,
mini_batch_size=mini_batch_size,
window_size=window_size,
save_interval=save_interval)
model = GRU.LibphysMBGRU(signal2model)
model.load(model.get_file_tag(), signal_directory)
model.model_name = "ecg_abstraction"
model.train_block(mit_sinus.tolist() + mit_long_term.tolist() +
cibhi_1.tolist() + cibhi_2.tolist(),
signal2model,
n_for_each=mini_batch_size)
# model.start_time = time.time()
# returned = model.train_model(x_train, y_train, signal2model)
# # if i == 16:
# # model.load(dir_name=signal2model.signal_directory, file_tag=model.get_file_tag(0, 1000))
# if returned:
# model.save(signal2model.signal_directory, model.get_file_tag(-5, -5))
# running_ok = returned
x_trains, y_trains, signals_2_models = [], [], []
for i, signal, core_name in zip(indexes, signals[indexes],
core_names[indexes]):
signal2model = Signal2Model(core_name,
signal_directory,
signal_dim=signal_dim,
hidden_dim=hidden_dim,
batch_size=batch_size + 5,
mini_batch_size=mini_batch_size,
window_size=window_size,
save_interval=save_interval,
lower_error=1e-6,
count_to_break_max=15)
# last_index = int(len(signal) * 0.33)
# x_train, y_train = prepare_test_data([signal[:last_index]], signal2model, mean_tol=0.9, std_tol=0.2)
# x_train, y_train = windows_selection(x_train, y_train)
# x_trains.append(x_train)
# y_trains.append(y_train)
signals_2_models.append(signal2model)
# np.savez("x_trains__.npz", x_trains=x_trains, y_trains=y_trains)
file = np.load("x_trains__.npz")
x_trains, y_trains = file["x_trains"], file["y_trains"]
signal_directory = 'BIO_ACC_[{0}.{1}]'.format(window_size, batch_size)
signals_tests = db.signal_tests
signals_models = db.signal_models
for i in range(178, 300):
try:
SIGNAL_BASE_NAME = "biometric_acc_x_"
X_train, Y_train, X_test, Y_test = get_signals_tests(
signals_tests, signals_models[0].Sd, type="biometric", index=i)
signal_name = SIGNAL_BASE_NAME + str(i)
signal_info = Signal2Model(signal_name,
signal_directory,
signal_dim=signal_dim,
hidden_dim=hidden_dim,
learning_rate_val=0.05,
batch_size=batch_size,
window_size=window_size,
number_of_epochs=number_of_epochs,
mini_batch_size=mini_batch_size)
model = GRU.LibPhys_GRU(signal_dim=signal_dim,
hidden_dim=hidden_dim,
signal_name=signal_name,
n_windows=mini_batch_size)
model.save(signal_directory, model.get_file_tag(-1, -1))
model.train_signals(X_train[1],
Y_train[1],
signal_info,
decay=0.95,
# signals_models = db.signal_models
# Load signals from rr database
all_signals = get_signals_tests(signals_tests, signal_dim)
# i = 1
hidden_dim_array = [16, 32, 64, 128, 256]
for i, hidden_dim in zip(range(1,
len(hidden_dim_array) + 1), hidden_dim_array):
signal_directory = 'DAY_HRV_HF_[' + str(hidden_dim) + '.' + str(
window_size) + ']'
for group_signals in all_signals:
model_name = 'day_hrv_hf_{0}'.format(i)
signal2model = Signal2Model(model_name,
signal_directory,
signal_dim=signal_dim,
window_size=window_size,
hidden_dim=hidden_dim,
mini_batch_size=mini_batch_size,
learning_rate_val=0.05,
save_interval=500)
model = GRU.LibphysMBGRU(signal2model)
# model.load(dir_name=signal_directory, file_tag=model.get_file_tag(-5,-5))
model.train_block(group_signals,
signal2model,
n_for_each=n_for_each,
random_training=True)
#
# signal_dim = 64
# hidden_dim = 256
# batch_size = 128
# mini_batch_size = 16
signal_dim = model_info.Sd
hidden_dim = model_info.Hd
mini_batch_size = 32
batch_size = 256
window_size = 1024
save_interval = 250
signal_directory = "ECG_CLUSTER[128.1024]"
signal_paths = []
for file in os.listdir("../data/processed/ALL/"):
if file.endswith(".npz") and file != "history.npz":
signal_paths.append(file)
signal2model = Signal2Model("generic_ecg", signal_directory, signal_dim=signal_dim, hidden_dim=hidden_dim,
batch_size=batch_size,
mini_batch_size=mini_batch_size, window_size=window_size,
save_interval=save_interval, number_of_epochs=2000, lower_error=1e-9,
count_to_break_max=15, learning_rate_val=0.01)
model = GRU.LibphysMBGRU(signal2model)
limit = 3000
# model.start_time = time.time()
print(1)
i0 = 999
# history_of_indexes = {}
model.load(dir_name=signal_directory, file_tag=model.get_file_tag(i0, 0))
# history_of_indexes = np.load("../data/processed/ALL/history.npz")["history_of_indexes"]
#, history_of_indexes=history_of_indexes)
for i in range(i0, limit):
returned = False
while not returned:
def start(s):
signal_dim = 256
hidden_dim = 256
mini_batch_size = 8
batch_size = 256
window_size = 1024
save_interval = 250
signal_directory = 'ECG_BIOMETRY[MIT]'.format(256, window_size)
raw_filenames, Ns, core_names = get_processing_variables()
# ind = np.array([0, 2, 10, 15, 29, 33, 48, 50, 54, 57, 58, 59, 64, 68])
# ind = np.array([51, 55, 59, 64, 0] + [10, 11, 29, 33])
# process_and_save_signals_2(raw_filenames, core_names, Ns, indexes2process=[ind[s]])#np.arange(45, len(raw_filenames)))
ind = np.array([29, 59, 64])
# exit()
processed_filenames = np.array([
'../data/processed/MIT/{0}[256].npz'.format(core_name)
for core_name in core_names
])
x_trains, y_trains, signals_2_models = [], [], []
# indexes = np.array([1, 7, 11, 12, 20, 30, 32, 33, 42] + list(range(Ns[0] + 2, sum(Ns) + 1))) - 1
# s = indexes.tolist().index(29)
# ind = np.arange(0, len(processed_filenames))
# s = 2
step = 1
e = s * step + step
ind = ind[s * step:e]
# indexes = [48, 49]
# indexes = np.array([0, 6, 11, 17, 26, 36, 37, 38])#, 41, 51, 55])
# ind = np.array([indexes[s]])
print(str(np.arange(s * step, e)) + " - " + str(ind))
for i, filename in enumerate(processed_filenames[ind]):
signal, core_name = np.load(filename)["signal"], np.load(
filename)["core_name"]
running_ok = False
signal2model = Signal2Model(core_name,
signal_directory,
signal_dim=signal_dim,
hidden_dim=hidden_dim,
batch_size=batch_size,
mini_batch_size=mini_batch_size,
window_size=window_size,
save_interval=save_interval,
lower_error=1e-10,
count_to_break_max=15)
last_index = int(len(signal) * 0.33)
std_tol = 0.1
mean_tol = 0.02
n_runs = 0
plt.plot(signal[:last_index])
plt.figure()
plt.plot(signal[last_index:])
plt.figure()
x_train, y_train = prepare_several_special_data([signal[:last_index]],
signal2model,
overlap=0.11,
mean_tol=mean_tol,
std_tol=std_tol)
while not running_ok:
print("Initiating training... ")
# x_train, y_train = prepare_test_data([signal[:last_index]], signal2model, mean_tol=mean_tol, std_tol=std_tol)
# x_train, y_train = prepare_special_data([signal[:last_index]], signal2model, mean_tol=mean_tol, std_tol=std_tol)
print("done")
print("Compiling Model {0}".format(signal2model.model_name))
# if n_runs < 1:
# model = try_to_load(core_name, signal2model)
# else:
model = GRU.LibphysMBGRU(signal2model)
if model:
indexes = range(
len(x_train) - (len(x_train) % mini_batch_size))
returned = model.train_model(x_train[indexes],
y_train[indexes], signal2model)
# if i == 16:
if returned:
model.save(signal2model.signal_directory,
model.get_file_tag(-5, -5))
else:
std_tol += 0.05
running_ok = returned
n_runs += 1
else:
running_ok = True
import DeepLibphys.utils.functions.libphys_GRU as GRU
from DeepLibphys.utils.functions.common import get_fantasia_dataset
from DeepLibphys.utils.functions.signal2model import Signal2Model
import theano
fantasia_list = [1, 2, 3, 4, 5, 6, 7, 8, 9, 10]
signals = [Signal2Model("ecg_"+str(i), "FANTASIA[128.256]_", save_interval=1000, number_of_batches=1, batch_size=256) for i in fantasia_list]
X_train, Y_train = get_fantasia_dataset(signals[0].signal_dim, fantasia_list, 'Fantasia/ECG/mat/', peak_into_data=False)
signal2model = Signal2Model("ecg_old_fantasia", "FANTASIA[32x10.256]", save_interval=100, hidden_dim=256, batch_size=len(X_train)*32)
model = GRU.LibPhys_GRU(signal2model.signal_dim, hidden_dim=signal2model.hidden_dim, signal_name=signal2model.signal_name)
model.save(signal2model.signal_directory, model.get_file_tag(-1, -1))
model.train_signals(X_train, Y_train, signal2model)
# for i in [0, 1, 2, 3, 4, 5, 7]:
# model = GRU.LibPhys_GRU(signals[i].signal_dim, hidden_dim=signals[i].hidden_dim, signal_name=signals[i].signal_name)
# model.save(signals[i].signal_directory, model.get_file_tag(-1, -1))
# model.train_signal(X_train[i], Y_train[i], signals[i], track_loss=False, save_distance=100)
quantize_signal(original_signals[i], 256)
for i in range(original_signals.shape[0])
])
filtered_signals = np.array([
quantize_signal(filtered_signals[i], 256)
for i in range(filtered_signals.shape[0])
])
print("Quantization Finished")
save_directory = "bento"
for i in range(original_signals.shape[0]):
sig_model = Signal2Model("GRU_Filter_all_" + str(i),
save_directory,
signal_dim=256,
mini_batch_size=16,
window_size=512)
train_block([original_signals[i]], [filtered_signals[i]],
n_for_each=128,
signal2model=sig_model)
# test_signals = np.array([loadmat(get_fantasia_full_paths()[i])['val'][0][30000:60000] for i in range(len(get_fantasia_full_paths()))])
# model = LibphysSGDGRU.LibphysSGDGRU(sig_model)
# model.load(model.get_file_tag(-5, -5), sig_model.signal_directory)
# predicted = generate_predicted_signal(sig_model, starting_signal=test_signals[0],window_seen_by_GRU_size=1024)
# Predict? Do we have to do save and load?
# plt.subplot(311)
# plt.plot(original_signals[0])
# plt.subplot(312)
# all signals -> np.array([loadmat(get_fantasia_full_paths()[i])['val'][0] for i in range(len(get_fantasia_full_paths()))])
print(original_signals.shape)
start = time.time()
filtered_signals = remove_noise(original_signals)
end = time.time()
print("Time Required to remove noise:", end - start)
original_signals = quantize_signal(original_signals, 256)
filtered_signals = quantize_signal(filtered_signals, 256)
save_directory = "bento"
sig_model = Signal2Model("GRU_Filter_128",
save_directory,
signal_dim=256,
mini_batch_size=32,
window_size=512)
# maybe change mbs to 16, tune overlap
#print("Training...")
#train_block([original_signals], [filtered_signals], overlap=0.2, n_for_each=128, signal2model=sig_model)
test_signals = loadmat(get_fantasia_full_paths()[0])['val'][0][30000:60000]
test_signals = quantize_signal(test_signals, 256)
model = LibphysSGDGRU.LibphysSGDGRU(sig_model)
model.load(model.get_file_tag(-5, -5), sig_model.signal_directory)
predicted = generate_predicted_signal(model,
3000,
starting_signal=test_signals,
import DeepLibphys.utils.functions.libphys_GRU as GRU
from DeepLibphys.utils.functions.common import get_fantasia_dataset
from DeepLibphys.utils.functions.signal2model import Signal2Model
fantasia_list = [2, 3, 4, 5, 6, 7, 8, 9, 10]
signals = [
Signal2Model("resp_" + str(i),
"RESP_FANTASIA[128.256]",
save_interval=1000,
number_of_batches=1,
batch_size=256) for i in fantasia_list
]
X_train, Y_train = get_fantasia_dataset(signals[0].signal_dim,
fantasia_list,
'Fantasia/RESP/mat/',
peak_into_data=False)
signal2model = Signal2Model("resp_[1.2.3.4.5.6.7.8.9.10]_old_fantasia",
"RESP_FANTASIA[1000.256]",
save_interval=1000,
hidden_dim=256,
batch_size=500)
model = GRU.LibPhys_GRU(signal2model.signal_dim,
hidden_dim=signal2model.hidden_dim,
signal_name=signal2model.signal_name)
model.save(signal2model.signal_directory, model.get_file_tag(-1, -1))
model.train_signals(X_train, Y_train, signal2model)
for i in [0, 1, 2, 3, 4, 5, 7]:
import numpy as np
def exists(where, what):
try:
where.index(what)
return True
except ValueError:
return False
signals_models = db.ecg_1024_256_RAW + db.cybhi_512_M1 + db.cybhi_512_M2 + db.mit_1024
model_info = signals_models[0]
signal2Model = Signal2Model(model_info.dataset_name,
model_info.directory,
signal_dim=model_info.Sd,
hidden_dim=model_info.Hd)
model = GRU.LibphysMBGRU(signal2Model)
times_in_hours = [[], [], [], [], []]
for model_info in signals_models:
try:
model.model_name = model_info.dataset_name
dirx = model_info.directory
model.load(dir_name=dirx)
hours = model.train_time / (3.6 * 10**6)
times_in_hours[0].append(
"cybhi" if exists(model.model_name, "cybhi") else
"mit" if exists(model.model_name, "mit") else "fantasia")
times_in_hours[1].append(256 if exists(dirx, "256") else
1024 if exists(dirx, "1024") else 12)
def calculate_loss_tensor(Total_Windows,
W,
signals_models,
signals,
mean_tol=10000,
overlap=0.33,
batch_percentage=0,
mini_batch=256,
std_tol=10000,
X_matrix=None,
Y_matrix=None,
min_windows=100):
if X_matrix is None and Y_matrix is None:
prepare_data = True
X_matrix = []
Y_matrix = []
else:
prepare_data = False
sizes = []
removex = []
for signal, model_info, i in zip(signals, signals_models,
range(len(signals))):
signal2model = Signal2Model(model_info.dataset_name,
"",
signal_dim=model_info.Sd,
hidden_dim=model_info.Hd,
batch_size=Total_Windows,
window_size=W)
if type(signal[0]) is np.int64 or type(signal[0]) is np.float64:
signal = [signal]
if prepare_data:
X_list, Y_list = prepare_test_data(
signal,
signal2model,
overlap=overlap,
batch_percentage=batch_percentage,
mean_tol=mean_tol,
std_tol=std_tol,
randomize=False)
if np.shape(X_list)[0] >= min_windows:
X_matrix.append(X_list)
Y_matrix.append(Y_list)
sizes.append(np.shape(X_list)[0])
else:
removex.append(i)
else:
print(np.shape(X_matrix[i]))
sizes.append(np.shape(X_matrix[i])[0])
removex.sort(reverse=True)
[signals_models.pop(rem) for rem in removex]
print(np.shape(X_matrix))
max_windows = np.min(np.array(sizes))
for t, test_signal in enumerate(X_matrix):
X_matrix[t] = test_signal[:max_windows]
Y_matrix[t] = Y_matrix[t][:max_windows]
print(np.shape(X_matrix))
X_matrix, Y_matrix = np.array(X_matrix), np.array(Y_matrix)
max_windows = max_windows - (max_windows % mini_batch)
print("Number of Windows: {0} of {1}".format(max_windows,
np.max(np.array(sizes))))
windows = np.arange(0, max_windows, mini_batch)
print(windows)
N_Models = len(signals_models)
N_Signals = len(X_matrix)
loss_tensor = np.zeros((N_Models, N_Signals, max_windows))
print("Loading model...")
model_info = signals_models[0]
signal2Model = Signal2Model(model_info.dataset_name,
model_info.directory,
signal_dim=model_info.Sd,
hidden_dim=model_info.Hd,
mini_batch_size=mini_batch)
model = GRU.LibphysMBGRU(signal2Model)
times = []
for m, model_info in zip(range(len(signals_models)), signals_models):
model.model_name = model_info.dataset_name
model.load(dir_name=model_info.directory)
print("Processing Model " + model_info.name + " - time: " +
str(model.train_time))
for s in range(N_Signals):
print("Calculating loss for ECG " + str(s + 1), end=';\n ')
for w in windows:
x_test = X_matrix[s, w:w + mini_batch, :]
y_test = Y_matrix[s, w:w + mini_batch, :]
tic = time.time()
loss_tensor[m, s, w:w + mini_batch] = np.asarray(
model.calculate_mse_vector(x_test, y_test))
times.append(time.time() - tic)
times = np.array(times)
print(np.size(loss_tensor, 2))
print(
"Statistics: \n Mean time: {0}; \n Std time: {1}; Max time: {2}; Min Time: {3}"
.format(np.mean(times), np.std(times), np.max(times), np.min(times)))
return loss_tensor