def keep(self): self.parent.statusbar.showMessage("Keep Columns in process...") self.columns_to_keep = [] i = 0 while self.model.item(i): if self.model.item(i).checkState() == 2: self.columns_to_keep.append(self.model.item(i).text()) i += 1 query = "Are you sure you want to proceed?" reply = QMessageBox.question(self, 'Message',query, QMessageBox.Yes | QMessageBox.No, QMessageBox.No) if reply == QMessageBox.No: self.columns_to_keep = [] return self.parent.statusbar.showMessage("Processing...") self.parent.ui.logOutput.append("KEPT:") print("KEPT:") #per file first retrieve data then filter columnarly for file in self.batch_files: list_colindx = [] datablob = helpers.clean_header(file) for item in self.columns_to_keep: list_colindx.append(datablob[0].index(item)) for i in range(len(datablob)): datablob[i][:] = [x for i, x in enumerate(datablob[i]) if i in list_colindx] split_name = file.split('.') now = time.strftime('%Y%m%d_%H%M%S', time.localtime(time.time())) output_file = split_name[0] + "_keep_columns_" + now + "." + split_name[1] helpers.write_out_to_file(output_file,datablob) self.parent.ui.logOutput.append(" " + str(file.split('/')[-1])) print(" " + str(file.split('/')[-1])) helpers.job_complete_message(self) self.parent.statusbar.showMessage("Welcome Back!") self.close()
def stack(self): self.parent.statusbar.showMessage("Stack Datasets in process...") self.full_data_complement = {} self.columns = [] self.parent.ui.logOutput.append("STACKED:") print("STACKED:") for idx, file in enumerate(self.batch_files): datablob = helpers.clean_header(file) if idx == 0: self.columns = datablob[0] self.full_data_complement[idx] = datablob self.parent.ui.logOutput.append(" " + str(file.split('/')[-1])) print(" " + str(file.split('/')[-1])) csv.register_dialect('CSV', delimiter='\t', quoting=csv.QUOTE_NONE) now = time.strftime('%Y%m%d_%H%M%S', time.localtime(time.time())) output_file = os.getcwd() + "\\STACKED_DATASETS_" + now + ".txt" with open(output_file, 'w',newline='') as f: writer = csv.writer(f, 'CSV') writer.writerow(self.columns) for i in range(len(self.full_data_complement)): for j in range(len(self.full_data_complement[i])): if j > 0: writer.writerow(self.full_data_complement[i][j]) #after job has been completed! helpers.job_complete_message(self)
def AOI_aggregate(self, type): # update the status bar self.parent.statusbar.showMessage(type + " Aggregation in process...") self.parent.ui.logOutput.append(" " + str(type)) print(" " + str(type)) # final dictionary which will hold all the AOI data from each file on a per StimulusName per typeAOI pattern self.full_AOI_complement = {} # list of columns to use in the AOI aggregation algorithm (could possibly be shortened for efficiency) self.cols_to_keep = [ 'StudyName', \ 'Name', \ 'Age', \ 'Gender', \ 'StimulusName', \ type, \ 'Joy Evidence', \ 'Joy Intensity', \ 'Anger Evidence', \ 'Anger Intensity', \ 'Surprise Evidence', \ 'Surprise Intensity', \ 'Fear Evidence', \ 'Fear Intensity', \ 'Contempt Evidence', \ 'Contempt Intensity', \ 'Disgust Evidence', \ 'Disgust Intensity', \ 'Sadness Evidence', \ 'Sadness Intensity', \ 'Neutral Evidence', \ 'Neutral Intensity', \ 'Positive Evidence', \ 'Positive Intensity', \ 'Negative Evidence', \ 'Negative Intensity', \ 'Raw Internal ADC A13 (Shimmer Sensor)', \ 'Raw GSR (Shimmer Sensor)', \ 'Cal Internal ADC A13 (Shimmer Sensor)', \ 'Cal GSR (Shimmer Sensor)'] # algorithm builds line per line for idx, file in enumerate(self.batch_files): # pull all the data and clean the header in the process into a self.data variable self.data = helpers.clean_header(file) # rename variable for easier use datablob = self.data # get indexes of all the columns you want to keep (REMOVE UNUSED COLUMNS) self.indexed_cols = [] for item in self.cols_to_keep: self.indexed_cols.append(datablob[0].index(item)) self.indexed_cols = sorted(self.indexed_cols) # reset datablob[i]'th line to a new line with less columns for i in range(len(datablob)): row = [] for j in self.indexed_cols: try: row.append(datablob[i][j]) except IndexError: print("Error at index: (" + str(i) + "," + str(j) + ")") datablob[i] = row # set empty AOI fields to a 'Z'-Value (SETTING NON-AOI AREAS) self.stim_name_idx = datablob[0].index('StimulusName') self.aoi_idx = datablob[0].index(type) self.stim_name_package = [] for i in range(1,len(datablob)): if datablob[i][self.stim_name_idx] not in self.stim_name_package: self.stim_name_package.append(datablob[i][self.stim_name_idx]) self.little_dict = {} pattern_dict = self.little_dict for item in self.stim_name_package: self.AOIs = [] for i in range(len(datablob)): if datablob[i][self.stim_name_idx] == item and datablob[i][self.aoi_idx] not in self.AOIs: self.AOIs.append(datablob[i][self.aoi_idx]) pattern_dict[item] = self.AOIs for i in range(1,len(datablob)): if pattern_dict[str(datablob[i][self.stim_name_idx])] != [''] and datablob[i][self.aoi_idx] == '': datablob[i][self.aoi_idx] = 'Z' # get the index of the typeAOI column (REMOVE EMPTY AOI LINES) self.list_colindx = datablob[0].index(type) # cut dataset shorter by removing empty AOI fields datablob = [item for idx, item in enumerate(datablob) if datablob[idx][self.list_colindx] != ''] # begin the per StimulusName per typeAOI patterning (REPATTERN ALGORITHM) self.stim_name_idx = datablob[0].index('StimulusName') self.aoi_idx = datablob[0].index(type) self.stim_name_package = [] for i in range(1,len(datablob)): if datablob[i][self.stim_name_idx] not in self.stim_name_package: self.stim_name_package.append(datablob[i][self.stim_name_idx]) self.little_dict = {} pattern_dict = self.little_dict for item in self.stim_name_package: self.AOIs = [] for i in range(len(datablob)): if datablob[i][self.stim_name_idx] == item and datablob[i][self.aoi_idx] not in self.AOIs: self.AOIs.append(datablob[i][self.aoi_idx]) pattern_dict[item] = self.AOIs self.column_data = [] for i in range(len(self.cols_to_keep)): self.column_data.append(self.cols_to_keep[i]) final_blob = [self.column_data] final_blob[0].append('FACET Point Weight') final_blob[0].append('FACET Point Weight %') final_blob[0].append('GSR Point Weight') final_blob[0].append('GSR Point Weight %') for stimulus_name in pattern_dict: #point weight percentage counters weight_counter_facet = 0 weight_counter_gsr = 0 for aoi_field in pattern_dict[stimulus_name]: #FACET fields counter_facet = 0 joy1 = 0 joy2 = 0 ang1 = 0 ang2 = 0 sur1 = 0 sur2 = 0 fear1 = 0 fear2 = 0 cont1 = 0 cont2 = 0 disg1 = 0 disg2 = 0 sad1 = 0 sad2 = 0 neu1 = 0 neu2 = 0 pos1 = 0 pos2 = 0 neg1 = 0 neg2 = 0 #shimmer fields counter_gsr = 0 raw_int = 0 raw_gsr = 0 calc_int = 0 calc_gsr = 0 for k in range(len(datablob)): if datablob[k][self.stim_name_idx] == stimulus_name and datablob[k][self.aoi_idx] == aoi_field: if counter_gsr == 0: study_name = datablob[k][datablob[0].index('StudyName')] name = datablob[k][datablob[0].index('Name')] age = datablob[k][datablob[0].index('Age')] gender = datablob[k][datablob[0].index('Gender')] if datablob[k][datablob[0].index('Joy Evidence')] != '': joy1 += float(datablob[k][datablob[0].index('Joy Evidence')]) joy2 += float(datablob[k][datablob[0].index('Joy Intensity')]) ang1 += float(datablob[k][datablob[0].index('Anger Evidence')]) ang2 += float(datablob[k][datablob[0].index('Anger Intensity')]) sur1 += float(datablob[k][datablob[0].index('Surprise Evidence')]) sur2 += float(datablob[k][datablob[0].index('Surprise Intensity')]) fear1 += float(datablob[k][datablob[0].index('Fear Evidence')]) fear2 += float(datablob[k][datablob[0].index('Fear Intensity')]) cont1 += float(datablob[k][datablob[0].index('Contempt Evidence')]) cont2 += float(datablob[k][datablob[0].index('Contempt Intensity')]) disg1 += float(datablob[k][datablob[0].index('Disgust Evidence')]) disg2 += float(datablob[k][datablob[0].index('Disgust Intensity')]) sad1 += float(datablob[k][datablob[0].index('Sadness Evidence')]) sad2 += float(datablob[k][datablob[0].index('Sadness Intensity')]) neu1 += float(datablob[k][datablob[0].index('Neutral Evidence')]) neu2 += float(datablob[k][datablob[0].index('Neutral Intensity')]) pos1 += float(datablob[k][datablob[0].index('Positive Evidence')]) pos2 += float(datablob[k][datablob[0].index('Positive Intensity')]) neg1 += float(datablob[k][datablob[0].index('Negative Evidence')]) neg2 += float(datablob[k][datablob[0].index('Negative Intensity')]) counter_facet += 1 weight_counter_facet += 1 if datablob[k][datablob[0].index('Raw Internal ADC A13 (Shimmer Sensor)')] != '': raw_int += float(datablob[k][datablob[0].index('Raw Internal ADC A13 (Shimmer Sensor)')]) raw_gsr += float(datablob[k][datablob[0].index('Raw GSR (Shimmer Sensor)')]) calc_int += float(datablob[k][datablob[0].index('Cal Internal ADC A13 (Shimmer Sensor)')]) calc_gsr += float(datablob[k][datablob[0].index('Cal GSR (Shimmer Sensor)')]) counter_gsr += 1 weight_counter_gsr += 1 if counter_facet != 0: joy1 /= float(counter_facet) joy2 /= float(counter_facet) ang1 /= float(counter_facet) ang2 /= float(counter_facet) sur1 /= float(counter_facet) sur2 /= float(counter_facet) fear1 /= float(counter_facet) fear2 /= float(counter_facet) cont1 /= float(counter_facet) cont2 /= float(counter_facet) disg1 /= float(counter_facet) disg2 /= float(counter_facet) sad1 /= float(counter_facet) sad2 /= float(counter_facet) neu1 /= float(counter_facet) neu2 /= float(counter_facet) pos1 /= float(counter_facet) pos2 /= float(counter_facet) neg1 /= float(counter_facet) neg2 /= float(counter_facet) if counter_gsr != 0: raw_int /= float(counter_gsr) raw_gsr /= float(counter_gsr) calc_int /= float(counter_gsr) calc_gsr /= float(counter_gsr) line =[study_name,name,age,gender,stimulus_name,aoi_field,joy1,joy2,ang1,ang2,sur1,sur2,fear1,fear2, \ cont1,cont2,disg1,disg2,sad1,sad2,neu1,neu2,pos1,pos2,neg1,neg2,raw_int,raw_gsr,calc_int,calc_gsr, \ counter_facet,0,counter_gsr,0] final_blob.append(line) #point weight percentage calculation for i in range(len(final_blob)): if final_blob[i][4] == stimulus_name: final_blob[i][31] = (final_blob[i][30] / weight_counter_facet) * 100 final_blob[i][33] = (final_blob[i][32] / weight_counter_gsr) * 100 self.full_AOI_complement[idx] = final_blob self.parent.ui.logOutput.append(" " + str(file.split('/')[-1])) print(" " + str(file.split('/')[-1])) #data ready for output csv.register_dialect('CSV', delimiter='\t', quoting=csv.QUOTE_NONE) now = time.strftime('%Y%m%d_%H%M%S', time.localtime(time.time())) output_file = os.getcwd() + "\\" + type.upper() + "_aggregation_" + now + ".txt" #the write-out with open(output_file, 'w',newline='') as f: writer = csv.writer(f, 'CSV') writer.writerow(self.full_AOI_complement[0][0]) for i in range(len(self.full_AOI_complement)): for j in range(1,len(self.full_AOI_complement[i])): writer.writerow(self.full_AOI_complement[i][j])