def start_kernel(self): self.DTYPE = self._dtype self._G = getattr(np, self.DTYPE)(self._G) ne.set_num_threads(ne.detect_number_of_cores()) # Get const values self.MASS_LEN = len(self) self.SIM_DIM = len(self._mass_list[0]._r) # Allocate memory: Object parameters self.mass_r_array = np.zeros((self.MASS_LEN, self.SIM_DIM), dtype=self.DTYPE) self.mass_a_array = np.zeros((self.MASS_LEN, self.SIM_DIM), dtype=self.DTYPE) self.mass_m_array = np.zeros((self.MASS_LEN, ), dtype=self.DTYPE) # Copy const data into Numpy infrastructure for pm_index, pm in enumerate(self._mass_list): self.mass_m_array[pm_index] = pm._m # Allocate memory: Temporary variables self.relative_r = np.zeros((self.MASS_LEN - 1, self.SIM_DIM), dtype=self.DTYPE) self.distance_sq = np.zeros((self.MASS_LEN - 1, ), dtype=self.DTYPE) self.distance_sqv = np.zeros((self.MASS_LEN - 1, self.SIM_DIM), dtype=self.DTYPE) self.distance_inv = np.zeros((self.MASS_LEN - 1, ), dtype=self.DTYPE) self.a_factor = np.zeros((self.MASS_LEN - 1, ), dtype=self.DTYPE) self.a1 = np.zeros((self.MASS_LEN - 1, ), dtype=self.DTYPE) self.a1r = np.zeros((self.MASS_LEN - 1, self.SIM_DIM), dtype=self.DTYPE) self.a1v = np.zeros((self.SIM_DIM, ), dtype=self.DTYPE) self.a2 = np.zeros((self.MASS_LEN - 1, ), dtype=self.DTYPE) self.a2r = np.zeros((self.MASS_LEN - 1, self.SIM_DIM), dtype=self.DTYPE)
def set_numexpr_threads(n=None): # if we are using numexpr, set the threads to n # otherwise reset if _NUMEXPR_INSTALLED and _USE_NUMEXPR: if n is None: n = ne.detect_number_of_cores() ne.set_num_threads(n)
def calc_energy(data): """Calculate the energy of the entire system. :Parameters: **data** -- the standard python data dictionary """ #name relevant variables x = data['x'] y = data['y'] nv = data['nv'] rho = 1000 #density of water #initialize EK to zero l = nv.shape[0] area = np.zeros(l) for i in xrange(l): #first, calculate the area of the triangle xCoords = x[nv[i,:]] yCoords = y[nv[i,:]] #Compute two vectors for the area calculation. v1x = xCoords[1] - xCoords[0] v2x = xCoords[2] - xCoords[0] v1y = yCoords[1] - yCoords[0] v2y = yCoords[2] - yCoords[0] #calculate the area as the determinant area[i] = abs(v1x*v2y - v2x*v1y) #get a vector of speeds. sdata = calc_speed(data) speed = sdata['speed'] #calculate EK, use numexpr for speed (saves ~15 seconds) ne.set_num_threads(ne.detect_number_of_cores()) ek = ne.evaluate("sum(rho * area * speed * speed, 1)") ek = ek / 4 data['ek'] = ek return data
def __init__(self, complexity_threshold=2, multicore=True): if numexpr_ver is None or numexpr_ver<(2, 0, 0): raise ImportError("numexpr version 2.0.0 or better required.") self.complexity_threshold = complexity_threshold nc = numexpr.detect_number_of_cores() if multicore is True: multicore = nc elif multicore is False: multicore = 1 elif multicore<=0: multicore += nc numexpr.set_num_threads(multicore)
def calc_speed(data): """ Calculates the speed from ua and va :Parameters: **data** -- the standard python data dictionary .. note:: We use numexpr here because, with multiple threads, it is\ about 30 times faster than direct calculation. """ #name required variables ua = data['ua'] va = data['va'] #we can take advantage of multiple cores to do this calculation ne.set_num_threads(ne.detect_number_of_cores()) #calculate the speed at each point. data['speed'] = ne.evaluate("sqrt(ua*ua + va*va)") return data
nopt = 100000 price, strike, t = gen_data(nopt) call = np.zeros(nopt, dtype=np.float64) put = -np.ones(nopt, dtype=np.float64) def black_scholes(price, strike, t, rate, vol): mr = -rate sig_sig_two = vol * vol * 2 P = price S = strike T = t call = ne.evaluate( "P * (0.5 + 0.5 * erf((log(P / S) - T * mr + 0.25 * T * sig_sig_two) * 1/sqrt(T * sig_sig_two))) - S * exp(T * mr) * (0.5 + 0.5 * erf((log(P / S) - T * mr - 0.25 * T * sig_sig_two) * 1/sqrt(T * sig_sig_two))) " ) put = ne.evaluate("call - P + S * exp(T * mr) ") return call, put #ne.set_vml_num_threads(ne.detect_number_of_cores()) ne.set_num_threads(ne.detect_number_of_cores()) ne.set_vml_accuracy_mode('high') if __name__ == '__main__': black_scholes(price, strike, t, RISK_FREE, VOLATILITY)
import expert_finding.models.propagation_idne_model import expert_finding.models.pre_ane_model import expert_finding.models.post_ane_model import expert_finding.models.tadw import expert_finding.models.gvnrt import expert_finding.models.graph2gauss import expert_finding.models.idne import expert_finding.models.gvnrt_expert_model import numexpr import os, sys, resource import logging logger = logging.getLogger() numexpr.set_num_threads(numexpr.detect_number_of_cores()) def get_memory(): with open('/proc/meminfo', 'r') as mem: free_memory = 0 for i in mem: sline = i.split() if str(sline[0]) in ('MemFree:', 'Buffers:', 'Cached:'): free_memory += int(sline[1]) return free_memory def memory_limit(): soft, hard = resource.getrlimit(resource.RLIMIT_AS) resource.setrlimit(resource.RLIMIT_AS, (get_memory() * 1024 * 0.90, hard))
def implement(): x = ne.detect_number_of_cores() os.environ['NUMEXPR_MAX_THREADS'] = str(x) print(f"Number of cores used = {x}")
def main(N, T, dt, numNayLow, numNayHigh, L, eta, speed, numBins, burnInTime,saveCorrEvery,calculateCorrelations, showAnimation, resultsFolder): start = time.time() print(" #################################################################################################") print(" START ") print("N = ", N, " T = ", T, " dt = ", dt, " numNayLow =", numNayLow," numNayHigh = ", numNayHigh, " L = ", L, " eta = ", eta, " speed = ", speed, " numBins = ", numBins, "burnInTime = ", burnInTime," saveCorrEvery = ", saveCorrEvery,"calculateCorrelations" ,calculateCorrelations, " showAnimation = ",showAnimation) ################################################################ ########### initialise ############################################################# numexprNumCores = ne.detect_number_of_cores() ne.set_num_threads(numexprNumCores) resultsCompletePath = resultsFolder + '/N%s/n%s/alpha%s'% (str(N), str(numNayHigh-numNayLow),str(numNayLow)) positionsQ = np.zeros((N,2)) anglesQ = np.zeros(N) inRangeIndexQ = np.zeros((N, 1+numNayHigh - numNayLow),dtype=np.int) allNearAngles = np.zeros((N, 1+numNayHigh - numNayLow)) meanSinAllNearAngles = np.zeros(N) meanCosAllNearAngles = np.zeros(N) meanDirectionsQ = np.zeros(N) velocitiesQ = np.zeros((N,2)) velocities_uQ = np.zeros((N,2)) mean_velocityQ = np.zeros((1,2)) noisesQ = np.zeros(N) if calculateCorrelations == True: binMatrix = np.zeros((N,N),dtype=int) binMatrixVU = np.zeros((N,N),dtype=int) correlationList = np.zeros((0,2)) maxDist = np.sqrt(pow(L/2.0,2)+pow(L/2.0,2)) binWidth = maxDist/numBins distanceMatrix = np.zeros((N,N)) velocityDotProductMatrixVU = np.zeros((N,N)) correlationSumHistogramVU = np.zeros(numBins) correlationCountsHistogramVU = np.zeros(numBins) sqVelocityDotProductMatrixVU = np.zeros((N,N)) sqCorrelationSumHistogramVU = np.zeros(numBins) onesMatrix =np.ones(np.shape(velocityDotProductMatrixVU[np.triu_indices(N,1)])) runCount = int(1) sumVelocities_U = 0.0 dotProdVelocities_U = 0.0 sumDotProdVelocities_U = 0.0 orderParameter= 0.0 sumOrderParameter = 0.0 positionsQ = agents.initialiseRandomPositions(L,positionsQ) anglesQ = agents.initialiseRandomAngles(anglesQ) noisesQ = agents.updateRandomNoises(noisesQ,eta) velocitiesQ = agents.updateVelocities(velocitiesQ,anglesQ,speed) velocities_uQ = agents.updateVelocities_u(velocitiesQ,velocities_uQ) inRangeIndexQ = agents.initialise_inRangeIndex(inRangeIndexQ) if showAnimation == True: figVelocityAnimation = plt.figure(1,figsize=(6,6)) axVel = figVelocityAnimation.add_subplot(111) plt.ion() wframe = None figVelocityAnimation.set_visible(False) pause(0.00000000001) ################################################################ ########### simulation loop ############################################################# for i in range(T): if showAnimation == True: if i < burnInTime: pass elif i >= burnInTime: oldcol = wframe wframe = plottingFunctions.plot_grid(axVel,positionsQ, velocitiesQ) figVelocityAnimation.set_visible(True) ########### timestep inRangeIndexQ[:,1:] = agents.calculateParticleInRange(positionsQ, L, numNayLow, numNayHigh) anglesQ = agents.calculateAngles(anglesQ, inRangeIndexQ, allNearAngles, meanSinAllNearAngles, meanCosAllNearAngles, meanDirectionsQ, noisesQ) velocitiesQ = agents.updateVelocities(velocitiesQ, anglesQ, speed) positionsQ = agents.updatePositions(positionsQ, velocitiesQ, L, dt) velocities_uQ = agents.updateVelocities_u(velocitiesQ, velocities_uQ) noisesQ = agents.updateRandomNoises(noisesQ,eta) if i < burnInTime: pass elif i >= burnInTime: if showAnimation == True: if oldcol: axVel.collections.remove(oldcol) figVelocityAnimation.canvas.draw() axVel.autoscale(enable=True, axis='both', tight=True) axVel.set_xticks([]) axVel.set_yticks([]) plt.pause(0.00000000000001) ################################################################ ########### correlations ############################################################# if i < burnInTime: pass elif i >= burnInTime: if calculateCorrelations == True: ########### save correlations if i % saveCorrEvery==0: if np.sum(correlationSumHistogramVU) != 0.0: print("i ",i) correlations.saveArray(resultsCompletePath + '/Correlations/sums/Correlation Velocity_U Sum Histogram_run%s'% (str(runCount)) ,correlationSumHistogramVU,N, T, dt, numNayLow, numNayHigh, L, eta, speed, numBins, burnInTime, saveCorrEvery) correlations.saveArray(resultsCompletePath +'/Correlations/counts/Correlation Velocity_U Counts Histogram_run%s'% ( str(runCount)) ,correlationCountsHistogramVU,N, T, dt, numNayLow, numNayHigh, L, eta, speed, numBins, burnInTime, saveCorrEvery) correlations.saveArray(resultsCompletePath +'/Correlations/sqSums/Square Correlation Velocity_U Sum Histogram_run%s'% ( str(runCount)) ,sqCorrelationSumHistogramVU,N, T, dt, numNayLow, numNayHigh, L, eta, speed, numBins, burnInTime, saveCorrEvery) correlations.saveArray(resultsCompletePath +'/Correlations/sumvUdotvU/sum dot prod velocities_U_run%s'% ( str(runCount)) ,np.array([sumDotProdVelocities_U]), N, T, dt, numNayLow, numNayHigh, L, eta, speed, numBins, burnInTime, saveCorrEvery) correlations.saveArray(resultsCompletePath +'/OrderParameter/sum of OrderParameter_run%s'% ( str(runCount)) ,np.array([sumOrderParameter]), N, T, dt, numNayLow, numNayHigh, L, eta, speed, numBins,burnInTime, saveCorrEvery) print("saving correlationCountsHistogramVU", correlationCountsHistogramVU) print("total counts" , np.sum(correlationCountsHistogramVU)) ########### reset correlations arrays distanceMatrix = np.zeros((N,N)) velocityDotProductMatrixVU = np.zeros((N,N)) correlationSumHistogramVU = np.zeros(numBins) correlationCountsHistogramVU = np.zeros(numBins) sqVelocityDotProductMatrixVU = np.zeros((N,N)) sqCorrelationSumHistogramVU = np.zeros(numBins) sumDotProdVelocities_U = 0.0 binMatrixVU = np.zeros((N,N),dtype=int) upperTriangleIndicesMask = np.triu_indices(np.size(binMatrixVU, axis=0),1) runCount +=1 ########### calculate correlations distanceMatrix = correlations.calculateDistMatrixWithPeriodicBoundary(positionsQ, L) velocityDotProductMatrixVU = correlations.calculateMatrixDotProduct(velocities_uQ) sqVelocityDotProductMatrixVU = correlations.calcSquareMatrix(velocityDotProductMatrixVU) binMatrixVU = correlations.calculateBinMatrix(distanceMatrix, binWidth,binMatrixVU) upperTriangleIndicesMask = np.triu_indices(np.size(binMatrixVU, axis=0),1) correlationSumHistogramVU += correlations.calcCorrelationHistoSum(binMatrixVU, velocityDotProductMatrixVU,numBins,upperTriangleIndicesMask) correlationCountsHistogramVU += correlations.calcCorrelationHistoCounts(binMatrixVU, onesMatrix,numBins,upperTriangleIndicesMask) sqCorrelationSumHistogramVU += correlations.calcCorrelationHistoSum(binMatrixVU, sqVelocityDotProductMatrixVU,numBins,upperTriangleIndicesMask) ################################################################ ########### order parameter ############################################################# orderParameter= orderParameterStatistics.calculateOrderParameter(velocitiesQ,speed) sumOrderParameter += orderParameter sumVelocities_U = orderParameterStatistics.calculateSumVelocities_U(velocities_uQ) dotProdVelocities_U = orderParameterStatistics.calculateDotProductVelocities_U(velocities_uQ) sumDotProdVelocities_U += dotProdVelocities_U ################################################################ ########### correlations ############################################################# if calculateCorrelations==True: correlations.saveArray(resultsCompletePath +'/Correlations/sums/Correlation Velocity_U Sum Histogram_run%s'% ( str(runCount)) ,correlationSumHistogramVU,N, T, dt, numNayLow, numNayHigh, L, eta, speed, numBins, burnInTime, saveCorrEvery) correlations.saveArray(resultsCompletePath +'/Correlations/counts/Correlation Velocity_U Counts Histogram_run%s'% ( str(runCount)) ,correlationCountsHistogramVU,N, T, dt, numNayLow, numNayHigh, L, eta, speed, numBins, burnInTime, saveCorrEvery) correlations.saveArray(resultsCompletePath +'/Correlations/sqSums/Square Correlation Velocity_U Sum Histogram_run%s'% ( str(runCount)) ,sqCorrelationSumHistogramVU,N, T, dt, numNayLow, numNayHigh, L, eta, speed, numBins, burnInTime, saveCorrEvery) correlations.saveArray(resultsCompletePath +'/Correlations/sumvUdotvU/sum dot prod velocities_U_run%s'% ( str(runCount)) ,np.array([sumDotProdVelocities_U]), N, T, dt, numNayLow, numNayHigh, L, eta, speed, numBins, burnInTime, saveCorrEvery) correlations.saveArray(resultsCompletePath +'/OrderParameter/sum of OrderParameter_run%s'% ( str(runCount)) ,np.array([sumOrderParameter]), N, T, dt, numNayLow, numNayHigh, L, eta, speed, numBins,burnInTime, saveCorrEvery) end = time.time() meanOrderParameter = sumOrderParameter/(T-burnInTime) print(" #################################################################################################") print(" END ") print(" Simulation statistics") print('eta = ', eta) print('mean order parameter = ', meanOrderParameter) print('mean DotProdVelocities_U = ', sumDotProdVelocities_U/(T-burnInTime)) print ("time taken = ", end - start) return meanOrderParameter
# === # <codecell> #%%timeit sq = np.asarray(nums) ** 2 # <markdowncell> # Squaring example using NumExpr # === # <codecell> import numexpr as ne print ne.detect_number_of_cores() print ne.evaluate("nums ** 2")[:5] # <codecell> #%%timeit ne.evaluate("nums ** 2") # <codecell> ne.set_num_threads(1) # <codecell> #%%timeit ne.evaluate("nums**2")
"""The NIDDK SICR model for estimating the fraction infected with SARS-CoV-2""" import os import numexpr # numexpr.set_num_threads(numexpr.detect_number_of_cores()) ncpus = numexpr.detect_number_of_cores() if 'SLURM_CPUS_PER_TASK' in os.environ: try: ncpus = int(os.environ['SLURM_CPUS_PER_TASK']) except ValueError: ncpus = 2 elif 'SLURM_CPUS_ON_NODE' in os.environ: try: ncpus = int(os.environ['SLURM_CPUS_ON_NODE']) except ValueError: ncpus = 2 from .io import * from .stats import * from .analysis import * from .data import * from .prep import * from .prepV import *
def preprocess_eeg(id_num, random_seed=None): # Set important variables bids_path = BIDSPath(id_num, task=task, datatype=datatype, root=bids_root) plot_path = os.path.join(plotdir, "sub_{0}".format(id_num)) if os.path.exists(plot_path): shutil.rmtree(plot_path) os.mkdir(plot_path) if not random_seed: random_seed = int(binascii.b2a_hex(os.urandom(4)), 16) random.seed(random_seed) id_info = {"id": id_num, "random_seed": random_seed} ### Load and prepare EEG data ############################################# header = "### Processing sub-{0} (seed: {1}) ###".format( id_num, random_seed) print("\n" + "#" * len(header)) print(header) print("#" * len(header) + "\n") # Load EEG data raw = read_raw_bids(bids_path, verbose=True) # Check if recording is complete complete = len(raw.annotations) >= 600 # Add a montage to the data montage_kind = "standard_1005" montage = mne.channels.make_standard_montage(montage_kind) mne.datasets.eegbci.standardize(raw) raw.set_montage(montage) # Extract some info eeg_index = mne.pick_types(raw.info, eeg=True, eog=False, meg=False) ch_names = raw.info["ch_names"] ch_names_eeg = list(np.asarray(ch_names)[eeg_index]) sample_rate = raw.info["sfreq"] # Make a copy of the data raw_copy = raw.copy() raw_copy.load_data() # Trim duplicated data (only needed for sub-005) annot = raw_copy.annotations file_starts = [a for a in annot if a['description'] == "file start"] if len(file_starts): duplicate_start = file_starts[0]['onset'] raw_copy.crop(tmax=duplicate_start) # Make backup of EOG and EMG channels to re-append after PREP raw_other = raw_copy.copy() raw_other.pick_types(eog=True, emg=True, stim=False) # Prepare copy of raw data for PREP, dropping all non-EEG channels raw_copy.pick_types(eeg=True) # Plot data prior to any processing if complete: save_psd_plot(id_num, "psd_0_raw", plot_path, raw_copy) save_channel_plot(id_num, "ch_0_raw", plot_path, raw_copy, raw_other) ### Clean up events ####################################################### print("\n\n=== Processing Event Annotations... ===\n") event_names = [ "stim_on", "red_on", "trace_start", "trace_end", "accuracy_submit", "vividness_submit" ] doubled = [] wrong_label = [] new_onsets = [] new_durations = [] new_descriptions = [] # Find and flag any duplicate triggers annot = raw_copy.annotations trigger_count = len(annot) for i in range(1, trigger_count - 1): a = annot[i] on_last = i + 1 == trigger_count prev_trigger = annot[i - 1]['description'] next_onset = annot[i + 1]['onset'] if not on_last else a['onset'] + 100 # Determine whether duplicates are doubles or mislabeled if a['description'] == prev_trigger: if (next_onset - a['onset']) < 0.002: doubled.append(a) else: wrong_label.append(a) # Rename annotations to have meaningful names & fix duplicates for a in raw_copy.annotations: if a in doubled or a['description'] not in event_names: continue if a in wrong_label: index = event_names.index(a['description']) a['description'] = event_names[index + 1] new_onsets.append(a['onset']) new_durations.append(a['duration']) new_descriptions.append(a['description']) # Replace old annotations with new fixed ones if len(annot): new_annot = mne.Annotations( new_onsets, new_durations, new_descriptions, orig_time=raw_copy.annotations[0]['orig_time']) raw_copy.set_annotations(new_annot) # Check annotations to verify we have equal numbers of each orig_counts = Counter(annot.description) counts = Counter(raw_copy.annotations.description) print("Updated Annotation Counts:") for a in event_names: out = " - '{0}': {1} -> {2}" print(out.format(a, orig_counts[a], counts[a])) # Get info id_info['annot_doubled'] = len(doubled) id_info['annot_wrong'] = len(wrong_label) count_vals = [ n for n in counts.values() if n != counts['vividness_submit'] ] id_info['equal_triggers'] = all(x == count_vals[0] for x in count_vals) id_info['stim_on'] = counts['stim_on'] id_info['red_on'] = counts['red_on'] id_info['trace_start'] = counts['trace_start'] id_info['trace_end'] = counts['trace_end'] id_info['acc_submit'] = counts['accuracy_submit'] id_info['vivid_submit'] = counts['vividness_submit'] if not complete: remaining_info = { 'initial_bad': "NA", 'num_initial_bad': "NA", 'interpolated': "NA", 'num_interpolated': "NA", 'remaining_bad': "NA", 'num_remaining_bad': "NA" } id_info.update(remaining_info) e = "\n\n### Incomplete recording for sub-{0}, skipping... ###\n\n" print(e.format(id_num)) return id_info ### Run components of PREP manually ####################################### print("\n\n=== Performing CleanLine... ===") # Try to remove line noise using CleanLine approach n_threads = max(1, int(numexpr.detect_number_of_cores() / 2)) linenoise = np.arange(60, sample_rate / 2, 60) EEG_raw = raw_copy.get_data() * 1e6 EEG_new = removeTrend(EEG_raw, sample_rate=raw.info["sfreq"]) EEG_clean = mne.filter.notch_filter( EEG_new, Fs=raw.info["sfreq"], freqs=linenoise, filter_length="10s", method="spectrum_fit", mt_bandwidth=2, p_value=0.01, n_jobs=n_threads, # uses half of all avaliable cores ) EEG_final = EEG_raw - EEG_new + EEG_clean raw_copy._data = EEG_final * 1e-6 del linenoise, EEG_raw, EEG_new, EEG_clean, EEG_final # Plot data following cleanline save_psd_plot(id_num, "psd_1_cleanline", plot_path, raw_copy) save_channel_plot(id_num, "ch_1_cleanline", plot_path, raw_copy, raw_other) # Perform robust re-referencing prep_params = {"ref_chs": ch_names_eeg, "reref_chs": ch_names_eeg} reference = Reference(raw_copy, prep_params, ransac=True, random_state=random_seed) print("\n\n=== Performing Robust Re-referencing... ===\n") reference.perform_reference() # If not interpolating bad channels, use pre-interpolation channel data if not interpolate_bads: reference.raw._data = reference.EEG_before_interpolation * 1e-6 reference.interpolated_channels = [] reference.still_noisy_channels = reference.bad_before_interpolation reference.raw.info["bads"] = reference.bad_before_interpolation # Plot data following robust re-reference save_psd_plot(id_num, "psd_2_reref", plot_path, reference.raw) save_channel_plot(id_num, "ch_2_reref", plot_path, reference.raw, raw_other) # Re-append removed EMG/EOG/trigger channels raw_prepped = reference.raw.add_channels([raw_other]) # Get info initial_bad = reference.noisy_channels_original["bad_all"] id_info['initial_bad'] = " ".join(initial_bad) id_info['num_initial_bad'] = len(initial_bad) interpolated = reference.interpolated_channels id_info['interpolated'] = " ".join(interpolated) id_info['num_interpolated'] = len(interpolated) remaining_bad = reference.still_noisy_channels id_info['remaining_bad'] = " ".join(remaining_bad) id_info['num_remaining_bad'] = len(remaining_bad) # Print re-referencing info print("\nRe-Referencing Info:") print(" - Bad channels original: {0}".format(initial_bad)) if interpolate_bads: print(" - Bad channels after re-referencing: {0}".format(interpolated)) print(" - Bad channels after interpolation: {0}".format(remaining_bad)) else: print( " - Bad channels after re-referencing: {0}".format(remaining_bad)) # Check if too many channels were interpolated for the participant prop_interpolated = len( reference.interpolated_channels) / len(ch_names_eeg) e = "### NOTE: Too many interpolated channels for sub-{0} ({1}) ###" if max_interpolated < prop_interpolated: print("\n") print(e.format(id_num, len(reference.interpolated_channels))) print("\n") ### Filter data and apply ICA to remove blinks ############################ # Apply highpass & lowpass filters print("\n\n=== Applying Highpass & Lowpass Filters... ===") raw_prepped.filter(1.0, 50.0, fir_design='firwin', picks=['eeg']) # Plot data following frequency filters save_psd_plot(id_num, "psd_3_filtered", plot_path, raw_prepped) save_channel_plot(id_num, "ch_3_filtered", plot_path, raw_prepped) # Perform ICA using EOG data on eye blinks print("\n\n=== Removing Blinks Using ICA... ===\n") ica = ICA(n_components=15, random_state=random_seed, method='picard') ica.fit(raw_prepped, decim=5) eog_indices, eog_scores = ica.find_bads_eog(raw_prepped) ica.exclude = eog_indices if not len(ica.exclude): err = " - Encountered an ICA error for sub-{0}, skipping for now..." print("\n") print(err.format(id_num)) print("\n") save_bad_fif(raw_prepped, id_num, ica_err_dir) return id_info # Plot ICA info & diagnostics before removing from signal save_ica_plots(id_num, plot_path, raw_prepped, ica, eog_scores) # Remove eye blink independent components based on ICA ica.apply(raw_prepped) # Plot data following ICA save_psd_plot(id_num, "psd_4_ica", plot_path, raw_prepped) save_channel_plot(id_num, "ch_4_ica", plot_path, raw_prepped) ### Compute Current Source Density (CSD) estimates ######################## if perform_csd: print("\n") print("=== Computing Current Source Density (CSD) Estimates... ===\n") raw_prepped = mne.preprocessing.compute_current_source_density( raw_prepped.drop_channels(remaining_bad)) # Plot data following CSD save_psd_plot(id_num, "psd_5_csd", plot_path, raw_prepped) save_channel_plot(id_num, "ch_5_csd", plot_path, raw_prepped) ### Write preprocessed data to new EDF #################################### if max_interpolated < prop_interpolated: if not os.path.isdir(noisy_bad_dir): os.makedirs(noisy_bad_dir) outpath = os.path.join(noisy_bad_dir, outfile_fmt.format(id_num)) else: outpath = os.path.join(outdir_edf, outfile_fmt.format(id_num)) write_mne_edf(outpath, raw_prepped) print("\n\n### sub-{0} complete! ###\n\n".format(id_num)) return id_info