def run(self, run_generators=True, run_noise=True, run_reference=True): smallest_p = create_scaled_data_structure([]) sig_p_count = create_scaled_data_structure(0) sig_p_count_interaction = create_scaled_data_structure(0) for simulation_number in range(self.number_of_simulations): if run_generators: if simulation_number == 0 or self.configuration == 'random': gen_conf_1 = self.condition_1_gen_conf() gen_conf_2 = self.condition_2_gen_conf(gen_conf_1) if self.configuration == 'uandk': gen_conf_1 = [{'depth': 7.0,\ 'magnitude': -1050.0,\ 'orientation': 0.0,\ 'orientation_phi': 0.0,\ 'phi': -0.65,\ 'theta': 0.6}] #print('Ran from here') [gen_conf_1, magnitude_stddev] =\ scale_generator_configuration(gen_conf_1, 3.5, 2) #print('STOP') #print('HUUUUUGE FAILLLL!') #print magnitude_stddev gen_conf_2 = self.condition_2_gen_conf(gen_conf_1) if simulation_number == 0 or self.configuration == 'random' or \ self.generator_magnitude_stddev != 0: [simulated_gen_1, gen_conf_1, simulated_gen_2, gen_conf_2, electrodes] = self.simulate_sources(gen_conf_1, gen_conf_2) self.electrodes = electrodes else: simulated_gen_1 = deepcopy(self.simulated_gen_1[simulation_number]) simulated_gen_2 = deepcopy(self.simulated_gen_2[simulation_number]) gen_conf_1 = deepcopy(self.gen_conf_1[simulation_number]) gen_conf_2 = deepcopy(self.gen_conf_2[simulation_number]) simulated_data_1 = deepcopy(simulated_gen_1) simulated_data_2 = deepcopy(simulated_gen_2) if run_noise: [normal_noise_1, topographic_noise_1, normal_noise_2, topographic_noise_2, constant_noise] =\ self.simulate_noise(simulated_gen_1.shape) else: normal_noise_1 = deepcopy(self.normal_noise_1[simulation_number]) normal_noise_2 = deepcopy(self.normal_noise_2[simulation_number]) topographic_noise_1 = deepcopy(self.topographic_noise_1[simulation_number]) topographic_noise_2 = deepcopy(self.topographic_noise_2[simulation_number]) constant_noise = deepcopy(self.constant_noise[simulation_number]) if normal_noise_1 != None and normal_noise_2 != None: simulated_data_1 += normal_noise_1 simulated_data_2 += normal_noise_2 if constant_noise != None: simulated_data_1 += constant_noise simulated_data_2 += constant_noise if topographic_noise_1 != None and topographic_noise_2 != None: simulated_data_1 += topographic_noise_1 simulated_data_2 += topographic_noise_2 if run_reference and self.reference != 'none': [simulated_data_1, simulated_data_2] =\ self.rereference(simulated_data_1, simulated_data_2, self.electrodes) electrode_indices = select_electrodes(self.hemispheres, self.sites, self.locations, self.electrodes)[4] simulated_data_1_scaled = perform_scaling(simulated_data_1, electrode_indices) simulated_data_2_scaled = perform_scaling(simulated_data_2, electrode_indices) self.electrode_indices = electrode_indices anova_results_scaled = \ create_scaled_data_structure(None) [anova_results, electrode_indices] = \ run_anova(self.hemispheres, self.sites, self.locations, simulated_data_1, simulated_data_2, self.electrodes, self.doElectrodesAsFactor, printResults=False, name='testu_sim') p = self.find_smallest_interaction_p(anova_results) smallest_p['Unrescaled'].append(p) if p < 0.05: found_interaction_on_unrescaled = True sig_p_count['Unrescaled'] += 1 else: found_interaction_on_unrescaled = False for key1 in anova_results_scaled.keys(): if key1 == 'Unrescaled': break for key2 in anova_results_scaled[key1].keys(): for key3 in anova_results_scaled[key1][key2].keys(): anova_results_scaled[key1][key2][key3] = \ run_anova(self.hemispheres, self.sites, self.locations, simulated_data_1_scaled[key1][key2][key3], simulated_data_2_scaled[key1][key2][key3], self.electrodes, self.doElectrodesAsFactor, printResults=False, name='testu_sim_scaled')[0] p = self.find_smallest_interaction_p\ (anova_results_scaled[key1][key2][key3]) smallest_p[key1][key2][key3].append(p) if p <0.05: sig_p_count[key1][key2][key3] += 1 if found_interaction_on_unrescaled: sig_p_count_interaction[key1][key2][key3] += 1 if self.collect: if len(self.gen_conf_1) == self.number_of_simulations: self.gen_conf_1[simulation_number] = gen_conf_1 self.gen_conf_2[simulation_number] = gen_conf_2 self.simulated_gen_1[simulation_number] = simulated_gen_1 self.simulated_gen_2[simulation_number] = simulated_gen_2 self.normal_noise_1[simulation_number] = normal_noise_1 self.normal_noise_2[simulation_number] = normal_noise_2 self.constant_noise[simulation_number] = constant_noise self.topographic_noise_1[simulation_number] = topographic_noise_1 self.topographic_noise_2[simulation_number] = topographic_noise_2 self.simulated_data_1[simulation_number] = simulated_data_1 self.simulated_data_2[simulation_number] = simulated_data_2 if self.electrodes == None: self.electrodes = electrodes self.sig_p_count[simulation_number] = sig_p_count self.sig_p_count_interaction[simulation_number] = sig_p_count_interaction if not (len(self.gen_conf_1) == self.number_of_simulations): self.gen_conf_1.append(gen_conf_1) self.gen_conf_2.append(gen_conf_2) self.simulated_gen_1.append(simulated_gen_1) self.simulated_gen_2.append(simulated_gen_2) self.normal_noise_1.append(normal_noise_1) self.normal_noise_2.append(normal_noise_2) self.constant_noise.append(constant_noise) self.topographic_noise_1.append(topographic_noise_1) self.topographic_noise_2.append(topographic_noise_2) self.simulated_data_1.append(simulated_data_1) self.simulated_data_2.append(simulated_data_2) if self.electrodes == None: self.electrodes = electrodes self.sig_p_count.append(sig_p_count) self.sig_p_count_interaction.append(sig_p_count_interaction) self.print_stats(sig_p_count, sig_p_count_interaction) return [simulated_data_1, simulated_data_2, simulated_data_1_scaled, simulated_data_2_scaled, gen_conf_1, gen_conf_2, smallest_p, sig_p_count, topographic_noise_1, topographic_noise_2, constant_noise]
[ERP_area_baseline["old"], electrodes_read_area[5]] = SPSS_query(file_SPSS_baseline, condition="11") [ERP_area_baseline["new"], electrodes_read_area[6]] = SPSS_query(file_SPSS_baseline, condition="12") # Preparing lists of electrodes if electrodes_read_avg[1] == electrodes_read_avg[2]: electrodes_avg = electrodes_read_avg[1] for key in electrodes_read_area.keys(): if electrodes_read_area[key] == electrodes_read_area[1]: electrodes_area = electrodes_read_area[1] else: electrodes = None print("Electrode mismatch in ERP data files") break electrode_indices = select_electrodes( parameters["hemispheres"], parameters["sites"], parameters["locations"], electrodes_area )[4] for key1 in ERP_area.keys(): for key2 in ERP_area[key1].keys(): # NOTE: Copy is done inside perform_scaling ERP_area_scaled[key1][key2] = perform_scaling(ERP_area[key1][key2], electrode_indices) # TODO: This should be done as a unit test probably if test_scaling(ERP_area_scaled[key1][key2], electrode_indices): print("A problem with scaling has been detected") if parameters["todo_test_assumptions_data"]: print("Performing assumption tests on ERP data") # This is temporary, but works
def test_assumptions(pickHemisphere, pickSite, pickLocation, dataCondition1, dataCondition2, electrodes, doElectrodesAsFactor = False, name = ''): """This function ... tests assumptions. Don't start me on how ugly it is, PLEASE FIX!!! """ # TODO: Clean up mess inside this function nSubjects = len(dataCondition1) [hemispherePicked, sitePicked, locationPicked, electrodesPicked, electrodeIndices] = select_electrodes(pickHemisphere, pickSite, pickLocation, electrodes) # Reading all relevant data into a single vector dataVector = [] for subject in range(nSubjects): dataVector = dataVector + list( dataCondition1[subject][electrodeIndices] ) dataVector = dataVector + list( dataCondition2[subject][electrodeIndices] ) # Converting data into an R matrix with rows corresponding to subjects and columns # corresponding to groups of levels of factors rDataVector = robjects.FloatVector(dataVector) robjects.globalenv["dataVector"] = rDataVector rData = robjects.r['matrix'](rDataVector, nrow = nSubjects, byrow = True) robjects.globalenv["data"] = rData # Representing factors as R variables rHemisphere = robjects.IntVector(2 * hemispherePicked) rSite = robjects.IntVector(2 * sitePicked) rLocation = robjects.IntVector(2 * locationPicked) rElectrodes = robjects.StrVector(2 * electrodesPicked) # The condition factor as an R variable rConditions = robjects.r.gl(2, len(electrodesPicked), 2*len(electrodesPicked), labels = ['old', 'new']) # Creating the variables in the R environment robjects.globalenv["conditions"] = rConditions robjects.globalenv["hemispheres"] = rHemisphere robjects.globalenv["locations"] = rLocation robjects.globalenv["sites"] = rSite robjects.globalenv["electrodes"] = rElectrodes # Creating R factors, currently can't do it via RPy2, need version 2.1 fCondition = robjects.r("condition <- factor(conditions)") fHemisphere = robjects.r("hemisphere <- factor(hemispheres)") fLocation = robjects.r("location <- factor(locations)") fSite = robjects.r("site <- factor(sites)") fElectrode = robjects.r("electrode <- factor(electrodes)") # Now we need to choose which factors will be included in the Anova. # We exclude all those that have only one level allFactors = '' if doElectrodesAsFactor == True: allFactors = ',electrode' else: if len(pickHemisphere)>1: allFactors += ',hemisphere' if len(pickLocation)>1: allFactors += ',location' if len(pickSite)>1 and not pickSite.count(0): allFactors += ',site' rFactors = robjects.r('allFactors <- data.frame(condition' + allFactors + ')') robjects.r("options(contrasts=c(\"contr.sum\", \"contr.poly\"))") anovaModel = robjects.r("model <- lm(data ~ 1)") robjects.r("library(car)") # Testing ANOVA assumptions for each electrode # Electrode factors for levene's test electrodesPicked1 = [] electrodesPicked2 = [] for electrode in electrodesPicked: electrodesPicked1.append('Old ' + electrode) electrodesPicked2.append('New ' + electrode) electrodesPickedLevene = electrodesPicked1 + electrodesPicked2 rElectrodesLevene = robjects.StrVector((len(rDataVector)/len(electrodesPickedLevene)) * electrodesPickedLevene) robjects.globalenv["electrodesLevene"] = rElectrodesLevene fElectrodeLevene = robjects.r("electrodeLevene <- factor(electrodesLevene)") # NORMALITY ASSUMPTION print('\n#\n# Testing ANOVA assumptions\n#') # Preparing to plot normal q-q plots dev_off = robjects.r('dev.off') nRows = len(pickLocation) if pickSite.count(0): nColumns = len(pickHemisphere) else: nColumns = len(pickHemisphere) * len(pickLocation) # Looping through all electrodes to do plots and the Shapiro-Wilk test print('\n# Shapiro-Wilk test for normality of noise at each electrode') for i in [1,2]: robjects.r.pdf('qqPlots' + name + str(i) + '.pdf') robjects.r('par(mfrow=c('+str(nRows)+','+str(nColumns) + '), pty="s", mar=c(2.5,2.5,1.5,0), mgp=c(1.5,0.1,0),tck=0.03,cex=0.8)') for j in range(len(electrodesPicked)): # Shapiro-Wilk test shapiro_test = robjects.r('shapiro.test') sw = shapiro_test(rData.rx(True,(i-1)*len(electrodesPicked)+j+1)) print('Shapiro-Wilk test for electrode ' + electrodesPicked[j] + ' in condition ' + str(i) + ', p = '+str(sw[1][0])) # Plotting q-q plots ylimits=robjects.FloatVector([-10,10]) if j == len(electrodesPicked)-1: robjects.r.qqnorm(rData.rx(True,(i-1)*len(electrodesPicked)+j+1),main=electrodesPicked[j].strip(),ylim=ylimits,col='blue', ylab='Scalp potential [uV]', xlab='Standard normal quantiles') robjects.r.qqline(rData.rx(True,(i-1)*len(electrodesPicked)+j+1), col='green4') else: #labels=False to turn off labels with numbers robjects.r.qqnorm(rData.rx(True,(i-1)*len(electrodesPicked)+j+1), ann=False, ylim=ylimits, tck=0.03, col='blue') robjects.r.qqline(rData.rx(True,(i-1)*len(electrodesPicked)+j+1), ann=False, ylim=ylimits, tck=0.03, col='green4') robjects.r.title(electrodesPicked[j].strip()) dev_off() # HOMOGENEITY OF VARIANCE ASSUMPTION # Levene's test print('\n# Levene\'s test for homogeneity of variance between electrodes in all conditions') levene_test = robjects.r('levene.test') lt = levene_test(rDataVector,fElectrodeLevene) print lt # Plot robjects.r.pdf('variancePlot' + name + '.pdf') robjects.r('par(mar=c(5,4,1,1), mgp=c(2,1,0),tck=0.01,cex=1, las=2)') robjects.r('boxplot(dataVector ~ electrodeLevene, range=0, col="green3", ylab=expression(paste("Scalp potential [",u,"V]")) )') dev_off() # Magnitude measurements print('\n#\n# Measuring the magnitude range of the data \n#') print('\n# Max - min magnitude for condition 1:') print(max(mean(dataCondition1,0)) - min(mean(dataCondition1,0))) print('\n# Max - min magnitude for condition 2:') print(max(mean(dataCondition2,0)) - min(mean(dataCondition2,0))) print('\n# Difference between maximal values in the two conditions:') print(max(mean(dataCondition1,0)) - max(mean(dataCondition2,0))) print('\n# The mean standard deviation for all electrodes for both conditions is: \n' + str(mean([mean(std(dataCondition1,0)), mean(std(dataCondition2,0))]))) print('\n# Max - min magnitude for difference between conditions:') print(max(mean(dataCondition1-dataCondition2,0)) - min(mean(dataCondition1-dataCondition2,0))) print('\n# The mean standard deviation for all electrodes for difference between conditions is: \n' + str(mean(std(dataCondition1-dataCondition2,0))))