def extract_random_segments_for_given_patient_during_warning(self,segment_label, patient_no): #during warning related to AR(l)__autocorrelation lag current_patient_ = patient_list[patient_no] patient_ann_ = current_patient_[:-4] + '-nsrr.xml' ann_, onset_, duration_ = extract_anns(TEST_ANN_PATH + patient_ann_) eeg_dict_, info_dict_ = extract_data(TEST_DATA_PATH + current_patient_, ann_, onset_) return eeg_dict_[segment_label][np.random.choice(len(eeg_dict[segment_label]))]
def extract_test_segments_for_given_patient(self, patient_no): #helper current_patient = patient_list[patient_no] patient_ann = current_patient[:-4] + '-nsrr.xml' ann, onset, duration = extract_anns(TEST_ANN_PATH + patient_ann) eeg_dict, info_dict = extract_data(TEST_DATA_PATH + current_patient, ann, onset,duration[-1]) len_dict = {} for i in eeg_dict.keys(): len_dict[i] = len(eeg_dict[i]) #print(f"eeg_dict{i} is {eeg_dict[i]}") #random.shuffle(eeg_dict[i]) #print(len_dict) selected_tuples=[] for _ in range(excess_segments_needed_per_patient): for i in eeg_dict.keys(): if i==4: continue if len_dict[i]!=0: selected_tuples.append((int(i),eeg_dict[i][np.random.choice(len(eeg_dict[i]))])) print(f"RAM: {resource.getrusage(resource.RUSAGE_SELF).ru_maxrss/1024} MB") for t in selected_tuples: yield t
def get_ref_dicts(self): #indices of fit dicts, found beforehand, can use FitDictFinder class to get desired dicts fit_ref_list = [3, 10, 19, 21, 22, 26, 28, 33, 36, 38, 40, 42, 43, 51, 52, 53, 55, 56, 57, 60, 62,\ 64, 70, 72, 73, 82, 83, 85, 87, 89, 90, 93, 95, 101, 103, 107, 109, 113, 115, 116,\ 117, 123, 130, 131, 135, 138, 139, 141, 148, 155, 156, 157, 159] selected_ref_indices = list( np.random.choice(fit_ref_list, size=self.num_patients, replace=False)) for i in selected_ref_indices: ref = patient_list[i] ann_ref = ref[:-4] + '-nsrr.xml' ann, onset, duration = extract_anns(TRAIN_ANN_PATH + ann_ref) preprocess = 'std' eeg_dict = extract_data(TRAIN_DATA_PATH + ref, ann, onset, duration[-1], preprocess=preprocess) len_dict = {} for i in eeg_dict.keys(): len_dict[i] = len(eeg_dict[i]) print(len_dict) print(ref) print(ann_ref) yield eeg_dict
def extract_random_segments_for_given_patient_during_warning( self, segment_label, patient_no): #during warning related to AR(l)__autocorrelation lag current_patient_ = self.patient_list[patient_no] print(f'\nCurrent Patient: {current_patient_}') patient_ann_ = current_patient_[:-4] + '-nsrr.xml' ann_, onset_, duration_ = extract_anns(self.ann_path + patient_ann_) eeg_dict_, info_dict_ = extract_data(self.data_path + current_patient_, ann_, onset_, duration_[-1]) #print(np.random.choice(len(eeg_dict_[segment_label])-1), len(eeg_dict_[segment_label])) return (int(segment_label), eeg_dict_[segment_label][np.random.choice( len(eeg_dict_[segment_label]) - 1)])
def extract_random_segments_for_given_patient( self, patient_no, num_segs_chosen_per_patient): #helper current_patient = self.patient_list[patient_no] print(f'\nCurrent Patient: {current_patient}') patient_ann = current_patient[:-4] + '-nsrr.xml' ann, onset, duration = extract_anns(self.ann_path + patient_ann) eeg_dict, info_dict = extract_data(self.data_path + current_patient, ann, onset, duration[-1]) len_dict = {} for i in eeg_dict.keys(): len_dict[i] = len(eeg_dict[i]) print(len_dict) tuples = [] #all (label, segment) for label in eeg_dict.keys(): # flag = (label == 0 or label == 2) for seg in range(len_dict[label]): tuples.append((int(label), eeg_dict[label][seg])) # if flag and np.random.random() < 0.3: # tuples.append((int(label), eeg_dict[label][seg])) # if not flag: # tuples.append((int(label), eeg_dict[label][seg])) # l = [] # for t in tuples: # l.append(t[0]) # print(f"tuples: {np.unique(l, return_counts=True)}") random.shuffle(tuples) selected_tuples = [] for i in range(num_segs_chosen_per_patient): selected_tuples.append(tuples.pop()) # t = tuples.pop() # if t[0] == self.ref_label: # selected_tuples.append(t) #popping after shuffling equivalent to sampling randomly # if t[0] != self.ref_label and np.random.random()<0.2: # selected_tuples.append(t) del tuples # l = [] # for t in selected_tuples: # l.append(t[0]) # print(f"selected_tuples: {np.unique(l, return_counts=True)}") for t in selected_tuples: yield t
def specific_patient_dataset(patient_no): current_patient = patient_list[patient_no] patient_ann = current_patient[:-4] + '-nsrr.xml' ann, onset, duration = extract_anns(TRAIN_ANN_PATH + patient_ann) eeg_dict, info_dict = extract_data(TRAIN_DATA_PATH + current_patient, ann, onset) len_dict = {} eeg_dict_list = [] for i in eeg_dict.keys(): len_dict[i] = len(eeg_dict[i]) if i == 4 and len_dict[i] != 0: print( f"{len_dict[i]} keys of segment 4 in patient no: {patient_no}") random.shuffle(eeg_dict[i]) eeg_dict_list.append(eeg_dict) return eeg_dict_list, len_dict
def extract_random_segments_for_given_patient(self, patient_no, num_segs_chosen): #helper current_patient = patient_list[patient_no] patient_ann = current_patient[:-4] + '-nsrr.xml' ann, onset, duration = extract_anns(TRAIN_ANN_PATH + patient_ann) eeg_dict, info_dict = extract_data(TRAIN_DATA_PATH + current_patient, ann, onset, duration[-1]) len_dict = {} labels_available = [] for i in eeg_dict.keys(): len_dict[i] = len(eeg_dict[i]) if len_dict[i] != 0: labels_available.append(i) #print(f"eeg_dict{i} is {eeg_dict[i]}") #random.shuffle(eeg_dict[i]) #print(len_dict) tuples = [] #all (label, segment) for label in eeg_dict.keys(): for seg in range(len_dict[label]): tuples.append((int(label), eeg_dict[label][seg])) random.shuffle(tuples) selected_tuples = [] for i in range(num_segs_chosen): selected_tuples.append(tuples.pop( )) #popping after shuffling equivalent to sampling randomly print( f"{i}th segment chosen of label {selected_tuples[i][0]} from patient {patient_no}" ) #print(f"RAM: {resource.getrusage(resource.RUSAGE_SELF).ru_maxrss/1024} MB") del tuples print( f"RAM: {resource.getrusage(resource.RUSAGE_SELF).ru_maxrss/1024} MB" ) process = psutil.Process(os.getpid()) print("Gen RAM Free: " + humanize.naturalsize( psutil.virtual_memory().available ), \ " | Proc size: " + humanize.naturalsize( process.memory_info().rss)) for t in selected_tuples: yield t
def get_fit_dict_indices(self): fit_dict_indices = [] for i in range(len(patient_list)): current_patient = patient_list[i] patient_ann = current_patient[:-4] + '-nsrr.xml' ann, onset, duration = extract_anns(TRAIN_ANN_PATH + patient_ann) preprocess = 'std' eeg_dict, info_dict = extract_data(TRAIN_DATA_PATH + current_patient, ann, onset, duration[-1], preprocess=preprocess) flag = self.is_dict_fit(eeg_dict) print(i, flag) if flag: fit_dict_indices.append(i) print(fit_dict_indices)
def extract_random_segments_for_given_patient_during_warning( self, segment_label, patient_no): #during warning related to AR(l)__autocorrelation lag current_patient_ = patient_list[patient_no] patient_ann_ = current_patient_[:-4] + '-nsrr.xml' ann_, onset_, duration_ = extract_anns(TRAIN_ANN_PATH + patient_ann_) eeg_dict_, info_dict_ = extract_data( TRAIN_DATA_PATH + current_patient_, ann_, onset_, duration_[-1]) len_dict = {} labels_available = [] for i in eeg_dict_.keys(): len_dict[i] = len(eeg_dict_[i]) if len_dict[i] != 0: labels_available.append(i) segment_label = np.random.choice(labels_available) return (int(segment_label), eeg_dict_[segment_label][(len(eeg_dict_[segment_label]) - 1) // 2])
def extract_test_segments_for_given_patient(self, patient_no): #helper current_patient = patient_list[patient_no] patient_ann = current_patient[:-4] + '-nsrr.xml' ann, onset, duration = extract_anns(TEST_ANN_PATH + patient_ann) preprocess = None #no-preprocessing eeg_dict, stat = extract_data(TEST_DATA_PATH + current_patient, ann, onset, duration[-1], preprocess=preprocess, return_stats=True) len_dict = {} for i in eeg_dict.keys(): len_dict[i] = len(eeg_dict[i]) selected_tuples = [] for i in eeg_dict.keys(): if len_dict[i] != 0: #print(f"Label: {i}: {len_dict[i]}") if i == 1: seg_indices = np.random.choice(len(eeg_dict[i]), min(11, len(eeg_dict[i])), replace=False) for j in seg_indices: selected_tuples.append((int(i), eeg_dict[i][j])) else: seg_indices = np.random.choice(len(eeg_dict[i]), min(10, len(eeg_dict[i])), replace=False) for j in seg_indices: selected_tuples.append((int(i), eeg_dict[i][j])) #print(seg_indices) for t in selected_tuples: yield t, stat
def extract_random_segments_for_given_patient(self, patient_no, num_segs_chosen_per_patient): #helper current_patient = self.patient_list[patient_no] patient_ann = current_patient[:-4] + '-nsrr.xml' ann, onset, duration = extract_anns(self.ann_path + patient_ann) preprocess = None #getting un-preprocessed segments eeg_dict, stat = extract_data(self.data_path + current_patient, ann, onset, duration[-1], preprocess=preprocess, return_stats=True) self.stats.append(stat) len_dict = {} for i in eeg_dict.keys(): len_dict[i] = len(eeg_dict[i]) selected_tuples = [] labels = [] tuples = [] #all (label, segment) for label in [1]: for seg in range(len_dict[label]): selected_tuples.append((int(label), eeg_dict[label][seg])) labels.append(label) for label in [0,2,3,4]: for seg in range(len_dict[label]): tuples.append((int(label), eeg_dict[label][seg])) random.shuffle(tuples) for _ in range(num_segs_chosen_per_patient-len(selected_tuples)): t = tuples.pop() selected_tuples.append(t) labels.append(t[0]) del tuples self.segs_global.extend(labels) self.data_list.extend(selected_tuples) del selected_tuples