def __init__(self, train_file, validation_file, test_file, ECG_N):
self.ECG_N = ECG_N
self.ecg_features = ECGFeatures(fs=FS_ECG)
self.data_train = self.readData(pd.read_csv(train_file))
self.data_val = self.readData(pd.read_csv(validation_file))
self.data_test = self.readData(pd.read_csv(test_file))
self.ECG_N = ECG_N
self.train_n = len(self.data_train)
self.val_n = len(self.data_val)
self.test_n = len(self.data_test)
def __init__(self, gps_file, ecg_file, mask_file, ecg_n=45, split_time = 45, stride=0.2):
self.gps_file = gps_file
self.ecg_file = ecg_file
self.mask_file = mask_file
self.ecg_n = ecg_n
self.stride = stride
self.featuresExct = ECGFeatures(FS_ECG)
# normalization ecg features
self.ecg_mean = np.array([8.69006926e+02, 5.36894839e+01, 5.13280806e+01, 3.60941889e+01,
5.12644394e+01, 3.50184864e+01, 7.05304536e+01, 4.00354951e+00,
1.93190808e+01, 2.37990616e+01, 4.74128624e+01, 1.79100193e+01,
3.61127269e+01, 8.89897875e+00, 1.83440847e+01, 9.47694202e-03,
2.99294977e-02, 8.32468992e-02, 2.66982160e-01, 1.05872866e+02,
8.24120855e+02, 1.61071678e+03, 2.27217251e+03, 3.79534713e+00,
2.62838417e+01, 3.42443566e+01, 3.56764545e+01, 3.36520073e+00,
5.63147720e+00, 6.03246072e+00, 6.01429266e+00, 5.03771520e+01,
4.96228480e+01, 1.88115762e+00, 1.67060736e+00, 1.47731287e-01,
3.62288239e+01, 6.01129089e+01, 2.21647750e+00, 1.17607048e+04,
4.32725792e-01])
self.ecg_std = np.array([1.12849622e+02, 7.22749812e+00, 4.13239686e+01, 3.76508218e+01,
6.06763869e+01, 4.94652050e+01, 9.08751400e+00, 2.91083561e+00,
1.74801557e+01, 1.09281612e+01, 2.35429989e+01, 1.10034329e+01,
2.36269451e+01, 9.38428850e+00, 2.01946583e+01, 1.47242980e-03,
1.22646518e-02, 2.66829649e-02, 7.71102801e-02, 8.09024660e+02,
3.84167641e+03, 1.11956783e+04, 1.61020832e+04, 3.85171123e+00,
1.77196093e+01, 1.81367242e+01, 2.12639285e+01, 1.62165833e+00,
1.37502961e+00, 1.34862368e+00, 1.51030700e+00, 2.27359887e+01,
2.27359887e+01, 2.64173948e+00, 5.57174922e-01, 8.24678134e-02,
4.28983320e+01, 4.14642962e+01, 9.86755656e-01, 4.09682862e+04,
3.83966021e-01])
self.data_set = self.readData()
self.test_n = len(self.data_set)
def __init__(self, train_file=None, validation_file=None, test_file=None, ECG_N=None, KD=False, teacher=False, ECG=False, training=True, high_only=False):
utils_path = "D:\\usr\\pras\\project\\ValenceArousal\\KnowledgeDistillation\\Utils\\"
self.max = np.load(utils_path+"max.npy")
self.mean = np.load(utils_path+"mean.npy")
self.std = np.load(utils_path+"std.npy")
self.KD = KD
self.teacher = teacher
self.ECG_N = ECG_N
self.ECG = ECG
self.high_only = high_only
self.w = 0
self.j =0
self.ecg_features = ECGFeatures(fs=FS_ECG)
#normalization ecg features
self.ecg_mean = np.array([2.18785670e+02, 5.34106162e+01, 1.22772848e+01, 8.87240641e+00,
1.23045575e+01, 8.19622448e+00, 2.80084568e+02, 1.51193876e+01,
3.36927105e+01, 6.63072895e+01, 7.52327656e-01, 1.85165308e+00,
1.42787092e-01])
self.ecg_std = np.array([28.6904681 , 7.2190369 , 8.96941273, 8.57895833, 13.34906982,
10.67710367, 36.68525696, 9.31097392, 21.09139643, 21.09139643,
0.88959446, 0.48770451, 0.08282199])
self.ECG_N = ECG_N
if training == True:
self.data_train = self.readData(pd.read_csv(train_file), KD, True)
self.data_val = self.readData(pd.read_csv(validation_file), KD)
self.train_n = len(self.data_train)
self.val_n = len(self.data_val)
self.data_test = self.readData(pd.read_csv(test_file), KD)
self.data_val = self.readData(pd.read_csv(validation_file), KD)
self.test_n = len(self.data_test)
self.val_n = len(self.data_val)
import glob
from Conf.Settings import FS_ECG, FS_GSR, FS_RESP, DATASET_PATH
from ECG.ECGFeatures import ECGFeatures
from GSR.GSRFeatures import PPGFeatures, EDAFeatures
from Resp.RespFeatures import RespFeatures
import pandas as pd
data_path = "D:\\usr\\pras\\data\\YAMAHA\\Yamaha-Experiment (2020-10-26 - 2020-11-06)\\data\\*"
ecg_file = "\\ECG\\"
gsr_file = "\\GSR\\"
resp_file = "\\Resp\\"
ecg_features = ECGFeatures(fs=FS_ECG)
ppg_features = PPGFeatures(fs=FS_GSR)
eda_features = EDAFeatures(fs=FS_GSR)
resp_features = RespFeatures(fs=FS_RESP)
for folder in glob.glob(DATASET_PATH + "*"):
for subject in glob.glob(folder + "\\*-2020-*"):
print(subject)
try:
# filter ecg iPhone
# ecg = pd.read_csv(glob.glob(subject+ecg_file+"*.csv")[0])
# ecg_filtered = ecg_features.filterECG(ecg["ecg"].values)
# ecg["ecg"] = ecg_filtered
# ecg.to_csv(subject+ecg_file+"filtered_ecg.csv", index=False)
#
# # filter PPG and EDA
# gsr = pd.read_csv(glob.glob(subject + gsr_file + "*.csv")[0], header=[0,1])
# time = gsr.iloc[:, 0].values
# eda = gsr.iloc[:, 5].values
path_results = path + 'results\\ECG\\'
data = pd.read_csv(path_ECGdata)
data_EmotionTest = pd.read_csv(path_EmotionTest)
# start = "2020-04-29 15:40:21"
# end = "2020-04-29 15:54:56"
#
# data = data.loc[(data.timestamp >= start) & (data.timestamp <= end)]
# convert timestamp to int
data.loc[:, 'timestamp'] = data.loc[:, 'timestamp'].apply(timeToInt)
data_EmotionTest.loc[:, 'Time_Start'] = data_EmotionTest.loc[:, 'Time_Start'].apply(timeToInt)
data_EmotionTest.loc[:, 'Time_End'] = data_EmotionTest.loc[:, 'Time_End'].apply(timeToInt)
# features extractor
featuresExct = ECGFeatures(set.FS_ECG)
emotionTestResult = pd.DataFrame(columns=["Idx", "Start", "End", "Valence", "Arousal", "Emotion", "Status"])
split_time = 45
idx = 0
for i in range(len(data_EmotionTest)):
tdelta = data_EmotionTest.iloc[i]["Time_End"] - data_EmotionTest.iloc[i]["Time_Start"]
time_end = data_EmotionTest.iloc[i]["Time_End"]
valence = data_EmotionTest.iloc[i]["Valence"]
arousal = data_EmotionTest.iloc[i]["Arousal"]
emotion = data_EmotionTest.iloc[i]["Emotion"]
for j in np.arange(0, (tdelta // split_time), 0.4):
end = time_end - (j * split_time)
start = time_end - ((j + 1) * split_time)
ecg = data[(data["timestamp"].values >= start) & (data["timestamp"].values <= end)]
status = 0
import pandas as pd
from Libs.Utils import utcToTimeStamp, timeToInt, butterBandpassFilter, windowFilter
from matplotlib import pyplot as plt
from ECG.ECGFeatures import ECGFeatures
path = "D:\\usr\\pras\\data\\EmotionTestVR\\Watanabe\\"
save_path = "D:\\usr\\pras\\result\\ValenceArousal\\ECG\\"
game_results_file = path + "Watanabe_M_2020_7_27_11_25_52_gameResults.csv"
ecg_file = path + "20200727_112502_148_HB_PW.csv"
ecg_data = pd.read_csv(ecg_file)
ecg_data["timestamp"] = ecg_data["timestamp"].apply(timeToInt)
game_results = pd.read_csv(game_results_file)
featuresExct = ECGFeatures()
for i in (0, 1, 7, 9):
time_start = timeToInt(game_results.iloc[i]["Time_Start"])
time_end = timeToInt(game_results.iloc[i]["Time_End"])
ecg = ecg_data[(ecg_data["timestamp"].values >= time_start)
& (ecg_data["timestamp"].values <= time_end)]["ecg"].values
ts, hb = featuresExct.computeHeartBeat(ecg, 256.)
plt.plot(ts, hb)
plt.ylim([50, 120])
plt.ylabel("Heart beat (bmp)")
plt.savefig(save_path + str(i) + "_ecg.png")
plt.show()
y_val.append(features_list.iloc[i]["Valence"])
else:
print(subject_path + "_" + str(i))
#load road test features
subject_road_path = ROAD_TEST_PATH + "TS001 E6\\20201119_135342_615_HB_PW.csv"
ecg_road_data = pd.read_csv(subject_road_path)
# convert timestamp to int
ecg_road_data.loc[:, 'timestamp'] = ecg_road_data.loc[:, 'timestamp'].apply(
timeToInt)
time_start = np.min(ecg_road_data["timestamp"].values)
time_end = np.max(ecg_road_data["timestamp"].values)
tdelta = time_end - time_start
#features extractor
featuresExct = ECGFeatures(200)
#road test features
road_test_features = []
for j in np.arange(0, (tdelta // SPLIT_TIME), 0.2):
end = time_end - ((j) * SPLIT_TIME)
start = time_end - ((j + 1) * SPLIT_TIME) - EXTENTION_TIME
ecg = ecg_road_data[(ecg_road_data["timestamp"].values >= start)
& (ecg_road_data["timestamp"].values <= end)]
ecg_values = featuresExct.filterECG(ecg['ecg'].values)
ecg_values = signal.resample(ecg_values,
int((len(ecg_values) / FS_ECG) * 200))
time_domain = featuresExct.extractTimeDomain(ecg_values)
freq_domain = featuresExct.extractFrequencyDomain(ecg_values)
from Resp.RespFeatures import RespFeatures
from ECG.ECGFeatures import ECGFeatures
import pandas as pd
import glob
from Libs.Utils import timeToInt, arToLabels
from Conf.Settings import FS_RESP, SPLIT_TIME, STRIDE, EXTENTION_TIME, RESP_RAW_PATH, ECG_RAW_RESP_PATH, RESP_PATH, ECG_RESP_PATH, DATASET_PATH, ECG_RR_PATH
import numpy as np
import os
data_path = "D:\\usr\\pras\\data\\YAMAHA\\Yamaha-Experiment (2020-10-26 - 2020-11-06)\\data\\*"
resp_file = "\\Resp\\"
game_result = "\\*_gameResults.csv"
resp_features_exct = RespFeatures(FS_RESP)
ecg_features_exct = ECGFeatures(FS_RESP)
min_len = FS_RESP * (SPLIT_TIME + 1)
for folder in glob.glob(DATASET_PATH + "*"):
for subject in glob.glob(folder + "\\*-2020-*"):
print(subject)
try:
data_EmotionTest = pd.read_csv(glob.glob(subject + game_result)[0])
resp_data = pd.read_csv(subject + resp_file + "filtered_resp.csv")
ecg_resp_data = pd.read_csv(subject + resp_file +
"filtered_ecg_resp.csv")
resp_data.iloc[:, 0] = resp_data.iloc[:, 0].apply(timeToInt)
ecg_resp_data.iloc[:, 0] = ecg_resp_data.iloc[:,
0].apply(timeToInt)
data_EmotionTest["Time_Start"] = data_EmotionTest[
"Time_Start"].apply(timeToInt)
data_EmotionTest["Time_End"] = data_EmotionTest["Time_End"].apply(
downsample_eeg_len = SPLIT_TIME * 200
#EDA Features extractor
eda_features_exct = EDAFeatures(FS_GSR)
ppg_features_exct = PPGFeatures(FS_GSR)
min_eda_len = (FS_GSR * SPLIT_TIME) - 50
#EEG features extractor
eeg_filter = SpaceLapFilter()
eeg_features_exct = EEGFeatures(FS_EEG)
#resp features extractor
resp_features_exct = RespFeatures(FS_RESP)
ecg_resp_features_exct = ECGRespFeatures(FS_RESP)
#ECG features exract
ecg_features_exct = ECGFeatures(FS_ECG)
for folder in glob.glob(DATASET_PATH + "*"):
for subject in glob.glob(folder + "\\*-2020-10-27"):
if not path.exists(subject + ALL_FEATURES_PATH):
os.mkdir(subject + ALL_FEATURES_PATH)
try:
data_EmotionTest = pd.read_csv(glob.glob(subject + game_result)[0])
#open ECG
ecg_data = pd.read_csv(glob.glob(subject + ecg_file + "*.csv")[0])
ecg_data.loc[:, 'timestamp'] = ecg_data.loc[:, 'timestamp'].apply(
timeToInt)
#open GSR
eda_data = pd.read_csv(subject + gsr_file + "filtered_eda.csv")
ppg_data = pd.read_csv(subject + gsr_file + "filtered_ppg.csv")
class DataFetchRoad:
def __init__(self, gps_file, ecg_file, mask_file, ecg_n=45, split_time = 45, stride=0.2):
self.gps_file = gps_file
self.ecg_file = ecg_file
self.mask_file = mask_file
self.ecg_n = ecg_n
self.stride = stride
self.featuresExct = ECGFeatures(FS_ECG)
# normalization ecg features
self.ecg_mean = np.array([8.69006926e+02, 5.36894839e+01, 5.13280806e+01, 3.60941889e+01,
5.12644394e+01, 3.50184864e+01, 7.05304536e+01, 4.00354951e+00,
1.93190808e+01, 2.37990616e+01, 4.74128624e+01, 1.79100193e+01,
3.61127269e+01, 8.89897875e+00, 1.83440847e+01, 9.47694202e-03,
2.99294977e-02, 8.32468992e-02, 2.66982160e-01, 1.05872866e+02,
8.24120855e+02, 1.61071678e+03, 2.27217251e+03, 3.79534713e+00,
2.62838417e+01, 3.42443566e+01, 3.56764545e+01, 3.36520073e+00,
5.63147720e+00, 6.03246072e+00, 6.01429266e+00, 5.03771520e+01,
4.96228480e+01, 1.88115762e+00, 1.67060736e+00, 1.47731287e-01,
3.62288239e+01, 6.01129089e+01, 2.21647750e+00, 1.17607048e+04,
4.32725792e-01])
self.ecg_std = np.array([1.12849622e+02, 7.22749812e+00, 4.13239686e+01, 3.76508218e+01,
6.06763869e+01, 4.94652050e+01, 9.08751400e+00, 2.91083561e+00,
1.74801557e+01, 1.09281612e+01, 2.35429989e+01, 1.10034329e+01,
2.36269451e+01, 9.38428850e+00, 2.01946583e+01, 1.47242980e-03,
1.22646518e-02, 2.66829649e-02, 7.71102801e-02, 8.09024660e+02,
3.84167641e+03, 1.11956783e+04, 1.61020832e+04, 3.85171123e+00,
1.77196093e+01, 1.81367242e+01, 2.12639285e+01, 1.62165833e+00,
1.37502961e+00, 1.34862368e+00, 1.51030700e+00, 2.27359887e+01,
2.27359887e+01, 2.64173948e+00, 5.57174922e-01, 8.24678134e-02,
4.28983320e+01, 4.14642962e+01, 9.86755656e-01, 4.09682862e+04,
3.83966021e-01])
self.data_set = self.readData()
self.test_n = len(self.data_set)
def fetch(self):
i = 0
while i < len(self.data_set):
data_i = self.data_set[i]
yield data_i
i+=1
def readData(self):
data_set = []
gps_data = pd.read_csv(self.gps_file)
ecg_data = pd.read_csv(self.ecg_file)
ecg_data.loc[:, 'timestamp'] = ecg_data.loc[:, 'timestamp'].apply(timeToInt)
gps_data.loc[:, 'timestamp'] = gps_data.loc[:, 'timestamp'].apply(timeToInt)
for j in range(1, len(gps_data)):
start = gps_data.loc[j]["timestamp"]
end = start + (SPLIT_TIME+4)
ecg = ecg_data[(ecg_data["timestamp"].values >= start) & (ecg_data["timestamp"].values <= end)]["ecg"].values
# print(len(ecg))
if len(ecg) >= self.ecg_n:
ecg = ecg[:self.ecg_n]
#extract ECG features
time_domain = self.featuresExct.extractTimeDomain(ecg)
freq_domain = self.featuresExct.extractFrequencyDomain(ecg)
nonlinear_domain = self.featuresExct.extractNonLinearDomain(ecg)
concatenate_features = (np.concatenate([time_domain, freq_domain, nonlinear_domain]) - self.ecg_mean) / self.ecg_std
data_set.append(concatenate_features)
#raw ecg
# ecg = (ecg - 2140.397356669409) / 370.95493558685325
# ecg = (ecg - 2048.485947046843) / 156.5629266658633
# ecg = ecg / (4095 - 0)
# ecg = signal.resample(ecg, 200 * SPLIT_TIME)
# data_set.append(ecg)
# print(ecg)
return data_set
from Resp.RespFeatures import RespFeatures
from Libs.Utils import timeToInt
import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
data_path = "D:\\usr\\pras\\data\\YAMAHA\\Yamaha-Experiment (2020-10-26 - 2020-11-06)\\data\\*"
ecg_file = "\\ECG\\filtered_ecg.csv"
eda_file = "\\GSR\\filtered_eda.csv"
ppg_file = "\\GSR\\filtered_ppg.csv"
resp_file = "\\Resp\\filtered_resp.csv"
ecg_resp_file = "\\Resp\\filtered_ecg_resp.csv"
game_result = "\\*_gameResults.csv"
plot_result = "\\plot\\"
ecg_features = ECGFeatures(fs=FS_ECG)
ppg_features = PPGFeatures(fs=FS_GSR)
eda_features = EDAFeatures(fs=FS_GSR)
resp_features = RespFeatures(fs=FS_RESP)
min_len = FS_RESP * (SPLIT_TIME + 1)
for folder in glob.glob(data_path):
for subject in glob.glob(folder + "\\*A6-2020-*"):
# try:
data_EmotionTest = pd.read_csv(glob.glob(subject + game_result)[0])
data_EmotionTest["Time_Start"] = data_EmotionTest["Time_Start"].apply(
timeToInt)
data_EmotionTest["Time_End"] = data_EmotionTest["Time_End"].apply(
timeToInt)
# ecg
ecg = pd.read_csv(subject + ecg_file)