def extract_random_segments_for_given_patient_during_warning(self,segment_label, patient_no): #during warning related to AR(l)__autocorrelation lag
current_patient_ = patient_list[patient_no]
patient_ann_ = current_patient_[:-4] + '-nsrr.xml'
ann_, onset_, duration_ = extract_anns(TEST_ANN_PATH + patient_ann_)
eeg_dict_, info_dict_ = extract_data(TEST_DATA_PATH + current_patient_, ann_, onset_)
return eeg_dict_[segment_label][np.random.choice(len(eeg_dict[segment_label]))]
def extract_test_segments_for_given_patient(self, patient_no): #helper
current_patient = patient_list[patient_no]
patient_ann = current_patient[:-4] + '-nsrr.xml'
ann, onset, duration = extract_anns(TEST_ANN_PATH + patient_ann)
eeg_dict, info_dict = extract_data(TEST_DATA_PATH + current_patient, ann, onset,duration[-1])
len_dict = {}
for i in eeg_dict.keys():
len_dict[i] = len(eeg_dict[i])
#print(f"eeg_dict{i} is {eeg_dict[i]}")
#random.shuffle(eeg_dict[i])
#print(len_dict)
selected_tuples=[]
for _ in range(excess_segments_needed_per_patient):
for i in eeg_dict.keys():
if i==4:
continue
if len_dict[i]!=0:
selected_tuples.append((int(i),eeg_dict[i][np.random.choice(len(eeg_dict[i]))]))
print(f"RAM: {resource.getrusage(resource.RUSAGE_SELF).ru_maxrss/1024} MB")
for t in selected_tuples:
yield t
def get_ref_dicts(self):
#indices of fit dicts, found beforehand, can use FitDictFinder class to get desired dicts
fit_ref_list = [3, 10, 19, 21, 22, 26, 28, 33, 36, 38, 40, 42, 43, 51, 52, 53, 55, 56, 57, 60, 62,\
64, 70, 72, 73, 82, 83, 85, 87, 89, 90, 93, 95, 101, 103, 107, 109, 113, 115, 116,\
117, 123, 130, 131, 135, 138, 139, 141, 148, 155, 156, 157, 159]
selected_ref_indices = list(
np.random.choice(fit_ref_list,
size=self.num_patients,
replace=False))
for i in selected_ref_indices:
ref = patient_list[i]
ann_ref = ref[:-4] + '-nsrr.xml'
ann, onset, duration = extract_anns(TRAIN_ANN_PATH + ann_ref)
preprocess = 'std'
eeg_dict = extract_data(TRAIN_DATA_PATH + ref,
ann,
onset,
duration[-1],
preprocess=preprocess)
len_dict = {}
for i in eeg_dict.keys():
len_dict[i] = len(eeg_dict[i])
print(len_dict)
print(ref)
print(ann_ref)
yield eeg_dict
def extract_random_segments_for_given_patient_during_warning(
self, segment_label,
patient_no): #during warning related to AR(l)__autocorrelation lag
current_patient_ = self.patient_list[patient_no]
print(f'\nCurrent Patient: {current_patient_}')
patient_ann_ = current_patient_[:-4] + '-nsrr.xml'
ann_, onset_, duration_ = extract_anns(self.ann_path + patient_ann_)
eeg_dict_, info_dict_ = extract_data(self.data_path + current_patient_,
ann_, onset_, duration_[-1])
#print(np.random.choice(len(eeg_dict_[segment_label])-1), len(eeg_dict_[segment_label]))
return (int(segment_label), eeg_dict_[segment_label][np.random.choice(
len(eeg_dict_[segment_label]) - 1)])
def extract_random_segments_for_given_patient(
self, patient_no, num_segs_chosen_per_patient): #helper
current_patient = self.patient_list[patient_no]
print(f'\nCurrent Patient: {current_patient}')
patient_ann = current_patient[:-4] + '-nsrr.xml'
ann, onset, duration = extract_anns(self.ann_path + patient_ann)
eeg_dict, info_dict = extract_data(self.data_path + current_patient,
ann, onset, duration[-1])
len_dict = {}
for i in eeg_dict.keys():
len_dict[i] = len(eeg_dict[i])
print(len_dict)
tuples = [] #all (label, segment)
for label in eeg_dict.keys():
# flag = (label == 0 or label == 2)
for seg in range(len_dict[label]):
tuples.append((int(label), eeg_dict[label][seg]))
# if flag and np.random.random() < 0.3:
# tuples.append((int(label), eeg_dict[label][seg]))
# if not flag:
# tuples.append((int(label), eeg_dict[label][seg]))
# l = []
# for t in tuples:
# l.append(t[0])
# print(f"tuples: {np.unique(l, return_counts=True)}")
random.shuffle(tuples)
selected_tuples = []
for i in range(num_segs_chosen_per_patient):
selected_tuples.append(tuples.pop())
# t = tuples.pop()
# if t[0] == self.ref_label:
# selected_tuples.append(t) #popping after shuffling equivalent to sampling randomly
# if t[0] != self.ref_label and np.random.random()<0.2:
# selected_tuples.append(t)
del tuples
# l = []
# for t in selected_tuples:
# l.append(t[0])
# print(f"selected_tuples: {np.unique(l, return_counts=True)}")
for t in selected_tuples:
yield t
def specific_patient_dataset(patient_no):
current_patient = patient_list[patient_no]
patient_ann = current_patient[:-4] + '-nsrr.xml'
ann, onset, duration = extract_anns(TRAIN_ANN_PATH + patient_ann)
eeg_dict, info_dict = extract_data(TRAIN_DATA_PATH + current_patient, ann,
onset)
len_dict = {}
eeg_dict_list = []
for i in eeg_dict.keys():
len_dict[i] = len(eeg_dict[i])
if i == 4 and len_dict[i] != 0:
print(
f"{len_dict[i]} keys of segment 4 in patient no: {patient_no}")
random.shuffle(eeg_dict[i])
eeg_dict_list.append(eeg_dict)
return eeg_dict_list, len_dict
def extract_random_segments_for_given_patient(self, patient_no,
num_segs_chosen): #helper
current_patient = patient_list[patient_no]
patient_ann = current_patient[:-4] + '-nsrr.xml'
ann, onset, duration = extract_anns(TRAIN_ANN_PATH + patient_ann)
eeg_dict, info_dict = extract_data(TRAIN_DATA_PATH + current_patient,
ann, onset, duration[-1])
len_dict = {}
labels_available = []
for i in eeg_dict.keys():
len_dict[i] = len(eeg_dict[i])
if len_dict[i] != 0:
labels_available.append(i)
#print(f"eeg_dict{i} is {eeg_dict[i]}")
#random.shuffle(eeg_dict[i])
#print(len_dict)
tuples = [] #all (label, segment)
for label in eeg_dict.keys():
for seg in range(len_dict[label]):
tuples.append((int(label), eeg_dict[label][seg]))
random.shuffle(tuples)
selected_tuples = []
for i in range(num_segs_chosen):
selected_tuples.append(tuples.pop(
)) #popping after shuffling equivalent to sampling randomly
print(
f"{i}th segment chosen of label {selected_tuples[i][0]} from patient {patient_no}"
)
#print(f"RAM: {resource.getrusage(resource.RUSAGE_SELF).ru_maxrss/1024} MB")
del tuples
print(
f"RAM: {resource.getrusage(resource.RUSAGE_SELF).ru_maxrss/1024} MB"
)
process = psutil.Process(os.getpid())
print("Gen RAM Free: " + humanize.naturalsize( psutil.virtual_memory().available ), \
" | Proc size: " + humanize.naturalsize( process.memory_info().rss))
for t in selected_tuples:
yield t
def get_fit_dict_indices(self):
fit_dict_indices = []
for i in range(len(patient_list)):
current_patient = patient_list[i]
patient_ann = current_patient[:-4] + '-nsrr.xml'
ann, onset, duration = extract_anns(TRAIN_ANN_PATH + patient_ann)
preprocess = 'std'
eeg_dict, info_dict = extract_data(TRAIN_DATA_PATH +
current_patient,
ann,
onset,
duration[-1],
preprocess=preprocess)
flag = self.is_dict_fit(eeg_dict)
print(i, flag)
if flag:
fit_dict_indices.append(i)
print(fit_dict_indices)
def extract_random_segments_for_given_patient_during_warning(
self, segment_label,
patient_no): #during warning related to AR(l)__autocorrelation lag
current_patient_ = patient_list[patient_no]
patient_ann_ = current_patient_[:-4] + '-nsrr.xml'
ann_, onset_, duration_ = extract_anns(TRAIN_ANN_PATH + patient_ann_)
eeg_dict_, info_dict_ = extract_data(
TRAIN_DATA_PATH + current_patient_, ann_, onset_, duration_[-1])
len_dict = {}
labels_available = []
for i in eeg_dict_.keys():
len_dict[i] = len(eeg_dict_[i])
if len_dict[i] != 0:
labels_available.append(i)
segment_label = np.random.choice(labels_available)
return (int(segment_label),
eeg_dict_[segment_label][(len(eeg_dict_[segment_label]) - 1) //
2])
def extract_test_segments_for_given_patient(self, patient_no): #helper
current_patient = patient_list[patient_no]
patient_ann = current_patient[:-4] + '-nsrr.xml'
ann, onset, duration = extract_anns(TEST_ANN_PATH + patient_ann)
preprocess = None #no-preprocessing
eeg_dict, stat = extract_data(TEST_DATA_PATH + current_patient,
ann,
onset,
duration[-1],
preprocess=preprocess,
return_stats=True)
len_dict = {}
for i in eeg_dict.keys():
len_dict[i] = len(eeg_dict[i])
selected_tuples = []
for i in eeg_dict.keys():
if len_dict[i] != 0:
#print(f"Label: {i}: {len_dict[i]}")
if i == 1:
seg_indices = np.random.choice(len(eeg_dict[i]),
min(11, len(eeg_dict[i])),
replace=False)
for j in seg_indices:
selected_tuples.append((int(i), eeg_dict[i][j]))
else:
seg_indices = np.random.choice(len(eeg_dict[i]),
min(10, len(eeg_dict[i])),
replace=False)
for j in seg_indices:
selected_tuples.append((int(i), eeg_dict[i][j]))
#print(seg_indices)
for t in selected_tuples:
yield t, stat
def extract_random_segments_for_given_patient(self, patient_no, num_segs_chosen_per_patient): #helper
current_patient = self.patient_list[patient_no]
patient_ann = current_patient[:-4] + '-nsrr.xml'
ann, onset, duration = extract_anns(self.ann_path + patient_ann)
preprocess = None #getting un-preprocessed segments
eeg_dict, stat = extract_data(self.data_path + current_patient, ann, onset, duration[-1], preprocess=preprocess, return_stats=True)
self.stats.append(stat)
len_dict = {}
for i in eeg_dict.keys():
len_dict[i] = len(eeg_dict[i])
selected_tuples = []
labels = []
tuples = [] #all (label, segment)
for label in [1]:
for seg in range(len_dict[label]):
selected_tuples.append((int(label), eeg_dict[label][seg]))
labels.append(label)
for label in [0,2,3,4]:
for seg in range(len_dict[label]):
tuples.append((int(label), eeg_dict[label][seg]))
random.shuffle(tuples)
for _ in range(num_segs_chosen_per_patient-len(selected_tuples)):
t = tuples.pop()
selected_tuples.append(t)
labels.append(t[0])
del tuples
self.segs_global.extend(labels)
self.data_list.extend(selected_tuples)
del selected_tuples
