def chicalc( Keplerlc, analyticallc, sigma, T_dur ): #caculates the chi^2 and reduced chi^2 values whilst only using those data points that fall within a transit count = 0 for i in range( 0, len(Keplerlc['Foldtime']), 1 ): #loops over the kepler lc (and analytical lc) inorder to find each points chi^2 value if np.abs( Keplerlc['Foldtime'][i] ) <= T_dur: #if the data point is part of the required group (i.e. falls within the transit) count += 1 if not 'chi2' in locals( ): #for the first point, create a chi^2 variable chi2 = (((Keplerlc['Flux'][i] - analyticallc[1][i])**2) / sigma**2) else: #otherwise add the next points chi^2 value to the total chi^2 value chi2 = chi2 + (( (Keplerlc['Flux'][i] - analyticallc[1][i])**2) / sigma**2) else: continue if count == 0: return np.nan(), np.nan() print 'The chi^2 value is {} (intran)'.format(chi2) #Testing chi2reduced = chi2 / ( count + 3 ) #now calculate the reduced chi^2 value for 4 independent varaibles print 'The reduced chi^2 value is {} (intran)'.format( chi2reduced) #testing return chi2, chi2reduced #return both chi^2 values
def compute_radians(self, df, units): if units == 'degrees': df['lat_rad'] = np.radians(df[self.flds['y']]) df['lng_rad'] = np.radians(df[self.flds['x']]) elif units == 'radians': # No conversion necessary df['lat_rad'] = df[self.flds['y']] df['lng_rad'] = df[self.flds['x']] else: df['lat_rad'] = np.nan() df['lng_rad'] = np.nan() return df
def sliding_window_correct(R, window_size=10): morphOrder = R['morph'] lLick, rLick = np.nan(morphOrder.shape), np.nan(morphOrder.shape) lLick[np.where((R['first lick'] == 1) & (R['morph'] == 0))[0]] = 1. rLick[np.where((R['first lick'] == 2) & (R['morph'] == 1))[0]] = 1. boxcar = np.ones([ window_size, ]) lLick_smooth = astconv.convolve(lLick.ravel(), boxcar) rLick_smooth = astconv.convolve(rLick.ravel(), boxcar) return lLick_smooth, rLick_smooth
def align(movie_data, options, args, lrh): print 'pICA(scikit-learn)' nvoxel = movie_data.shape[0] nTR = movie_data.shape[1] nsubjs = movie_data.shape[2] align_algo = args.align_algo nfeature = args.nfeature randseed = args.randseed # zscore the data bX = np.nan((nsubjs*nvoxel, nTR)) for m in xrange(nsubjs): bX[m*nvoxel:(m+1)*nvoxel, :] = stats.zscore(movie_data[:, :, m].T, axis=0, ddof=1).T del movie_data np.random.seed(randseed) A = np.mat(np.random.random((nfeature, nfeature))) ica = FastICA(n_components=nfeature, max_iter=500, w_init=A, random_state=randseed) St = ica.fit_transform(bX.T) S = St.T #bW = ica.transform(bX.T) bW = ica.mixing_ print S.shape print bW.shape niter = 10 # initialization when first time run the algorithm np.savez_compressed(options['working_path']+align_algo+'_'+lrh+'_'+str(niter)+'.npz',\ bW = bW, niter=niter) return niter
def viterbi (self, y , suppx): """ Calculate the maximum a - posteriori assignment of x ’s . : param y : a sequence of words : param suppx : the support of x ( what values it can attain ) : param t : the transition distributions of the model : param e : the emission distributions of the model : return : xhat , the most likely sequence of hidden states ( parts of speech ). """ n = len(y) num_tags = len(self.tags) prob_v = np.zeros(n, num_tags) track_v = np.zeros(n, num_tags) prob_v[0, :] = self.q for i in range(1, n): for j in range(num_tags): temp = np.multiply(np.multiply(prob_v[i -1, :], self.t[:, j]), self.e[:, self.words_dict[y[i]]]) prob_v[i, j ] = np.max(temp) track_v[i, j] = np.argmax(temp) # calc route route = np.nan(n) route[n] = self.tags_dict[np.argmax(prob_v[n,:])] #track = track_v[np.max(prob_v[n,:])] for i in range(n - 1, 0, -1): route[i] = self.tags_dict[track_v[self.tags_dict[route[i+1]]]] return route
def __normalised_true_range(row: Se): try: # https://www.investopedia.com/terms/a/atr.asp - # we use the formula for TR and then divide by close return max(row[0] - row[1], abs(row[0] - row[2]), abs(row[1] - row[2])) / row[2] except RuntimeError: return np.nan()
def align(movie_data, options, args, lrh): print 'pPCA(scikit-learn)' nvoxel = movie_data.shape[0] nTR = movie_data.shape[1] nsubjs = movie_data.shape[2] align_algo = args.align_algo nfeature = args.nfeature # zscore the data bX = np.nan((nsubjs*nvoxel,nTR)) for m in xrange(nsubjs): bX[m*nvoxel:(m+1)*nvoxel,:] = stats.zscore(movie_data[:, :, m].T, axis=0, ddof=1).T del movie_data U, s, VT = np.linalg.svd(bX, full_matrices=False) bW = np.zeros((nsubjs*nvoxel,nfeature)) for m in xrange(nsubjs): bW[m*nvoxel:(m+1)*nvoxel,:] = U[m*nvoxel:(m+1)*nvoxel,:nfeature] niter = 10 # initialization when first time run the algorithm np.savez_compressed(options['working_path']+align_algo+'_'+lrh+'_'+str(niter)+'.npz',\ bW = bW, niter=niter) return niter
def Tsurv(nu12,nu23,masses,m0=1.,fudge=1,res=False): """ Main result from the paper. Return the survival time estimate as a function of the initial period ratios and masses (eq. 81). In units of of the innermost orbit Returns np.inf if separation wide enough that 3-body MMRs don't overlap. nu12, nu23 : Initial period ratios masses : planet masses m0 : star mass fudge : fudge factor to adjust the number of resonances taken into account for more than three planets. Usually 1. res : Boolean. True means that the exact distance to the closest resonance is used. However, since we do not take into account the actual shape of the two planet MMR the results are less good than by assuming a constant distance $\nu/2$ to the MMR. """ plsepov = get_plsep_ov(nu12,nu23,masses,m0)*fudge**.25 al12 = nu12**(2/3) al23 = nu23**(2/3) eta = nu12*(1-nu23)/(1-nu12*nu23) plsep = (1-al12)*(1-al23)/(2-al12-al23) Tnorm = 2**1.5/9*(plsep/plsepov)**6/(1-(plsep/plsepov)**4)*10**(-np.log(1-(plsep/plsepov)**4)) A = np.sqrt(38/pi) Mfac = get_Mfac(nu12,nu23,masses,m0) PrefacD = Mfac*nu12*A*np.sqrt(eta*(1-eta))*fudge**-2 if res: Deta,u0=distance_eta_diffusion_direction(nu12,nu23,masses) Detanorm = np.maximum(Deta,np.sum(masses)**(2/3)/plsep**(1/3))/plsep # the Deta is in unit of plsep since plsep already in Tnorm # Minimum distance is comparable to resonance size u0 = np.minimum(0.7,np.maximum(0.3,u0)) #Bounds to somewhat take into account the time spent inside the MMR Tsurv = ((Detanorm)**2/(PrefacD)*Tnorm*3/2*u0**2*(1-u0)**2) else: Tsurv = (3/2)**2/PrefacD*Tnorm*3/32 #Deta=3/2 in units of plsep return np.nan(Tsurv,nan=np.inf)
def house_data_extraction(house_url): """ Returns an numpy array containing the scrapped data from a Zoopla house URL Parameters: house_url(list): A string of the house URL Returns: data(array): A 1D numpy array containing the extracted data from each house """ # Open the house link to extract parameters html = urlopen(house_url) soup = BeautifulSoup(html, 'html.parser') pagesource = str(soup).split() try: # Number of beds bed_find = pagesource.index('num_beds:') beds = pagesource[bed_find + 1][0] # Number of baths bath_find = pagesource.index('num_baths:') baths = pagesource[bath_find + 1][0] # House price price_find = pagesource.index('price_actual:') price_rough = pagesource[price_find + 1] price = price_rough.split(',')[0] # Property type type_find = pagesource.index('property_type:') prop_type_rough = pagesource[type_find + 1] prop_type = prop_type_rough.split('"')[1] #Lattitude lat_find = pagesource.index('"latitude":') lat_rough = pagesource[lat_find + 1] lat = float(lat_rough.split(',')[0]) #Longtitude lon_find = pagesource.index('"longitude":') lon_rough = pagesource[lon_find + 1] lon = float(lon_rough.split(',')[0]) # First published date price_history = pagesource.index('class="dp-price-history__item">') year = pagesource[price_history + 4].split('<')[0] #Distance to nearsest attraction/train station station_find = pagesource.index('miles') station = float(pagesource[station_find - 1]) except: beds, baths, price, prop_type, lat, lon, year, station = np.nan() #Is there a mention of a Loft? if 'loft' in str(soup): loft = 1 else: loft = 0 #Is there a mention of a Garden? if 'garden' in str(soup): garden = 1 else: garden = 0 data_extract = np.array([ price, beds, baths, loft, garden, station, year, lat, lon, prop_type ]) return data_extract
def chicalc(Keplerlc, analyticallc, sigma, T_dur): #caculates the chi^2 and reduced chi^2 values whilst only using those data points that fall within a transit count = 0 for i in range(0, len(Keplerlc['Foldtime']), 1): #loops over the kepler lc (and analytical lc) inorder to find each points chi^2 value if np.abs(Keplerlc['Foldtime'][i]) <= T_dur: #if the data point is part of the required group (i.e. falls within the transit) count += 1 if not 'chi2' in locals(): #for the first point, create a chi^2 variable chi2 = (((Keplerlc['Flux'][i] - analyticallc[1][i])**2) / sigma**2) else: #otherwise add the next points chi^2 value to the total chi^2 value chi2 = chi2 + (((Keplerlc['Flux'][i] - analyticallc[1][i])**2) / sigma**2) else: continue if count == 0: return np.nan(), np.nan() print 'The chi^2 value is {} (intran)'.format(chi2) #Testing chi2reduced = chi2 / (count + 3) #now calculate the reduced chi^2 value for 4 independent varaibles print 'The reduced chi^2 value is {} (intran)'.format(chi2reduced) #testing return chi2, chi2reduced #return both chi^2 values
def safe_check(value): """Check for finite value and replace with np.nan if does not exist.""" try: if np.isfinite(value): return value else: return np.nan() except ValueError: return np.nan
def ncc(seg, width): # Local Variables: B, hash, y, i, siz, s, N, seg, width, S, cacheDir, u, x, Nu, xs, ys, uniq, o2, o1 # Function calls: load, hashMat, unique, false, floor, sum, nan, uint8, ceil, uniqfilt, length, save, ncc, find, bwlabel, size #% N = ncc( seg, w ) #% #% N(i,j) = number of connected components in the #% w-by-w window centered on location (i,j) of seg #% #% What do I mean by "number of connected components"? #% Here are some example segmentation patches: #% #% 1 1 2 2 1 1 #% 1 1 2 2 1 1 this patch has THREE segments, even #% 1 1 2 2 1 1 though unique() only returns [1;2]. #% 1 1 2 2 1 1 #% #% 3 3 7 7 7 7 #% 3 3 7 7 7 7 this patch has FOUR segments, even #% 7 7 3 3 3 3 though unique() is simply [3;7]. #% 7 7 3 3 3 3 #% #% THIS CODE IS VERY SLOW, hence all results are cached. cacheDir = '/home/ashishmenon/labpc/oef/cache/clust/nccCache/' hash = hashMat( np.array(np.vstack((np.hstack((seg.flatten(1))), np.hstack((width)))))) try: np.load(np.array(np.hstack((cacheDir, hash))), 'N') except: siz = matcompat.size(seg) o1 = np.floor(((width - 1.) / 2.)) o2 = np.ceil(((width - 1.) / 2.)) #% This takes ~80 seconds per image N = np.nan(siz) B = false(siz) B[int(o1 + 1.) - 1:0 - o2, int(o1 + 1.) - 1:0 - o2] = 1. Nu = uniqfilt(seg, o2) N[int(np.logical_and(B, Nu == 1.)) - 1] = 1. [ys, xs] = nonzero(np.logical_and(B, Nu > 1.)) for i in np.arange(1., (length(ys)) + 1): y = ys[int(i) - 1] x = xs[int(i) - 1] S = seg[int(y - o1) - 1:y + o2, int(x - o1) - 1:x + o2] uniq = np.unique(S) N[int(y) - 1, int(x) - 1] = 0. for s in uniq.flatten(0).conj(): u = np.unique(bwlabel((S == s), 4.)) N[int(y) - 1, int(x) - 1] = N[int(y) - 1, int(x) - 1] + np.sum((u != 0.)) #% convert to uint8 to save disk space #% (but this converts NaNs to 0s..) N = np.uint8(N) plt.save(np.array(np.hstack((cacheDir, hash))), 'N') return [N]
def _predict_single_sample(self, internal_ids, sample_name): try: model = self._model_dict[sample_name] pred = model.predict( self._peptide_df.learned_rt.values[internal_ids].reshape( *self._target_shape)) return pred.flatten() except KeyError: return np.nan()
def dataCleaner(dataframe): """ Removes the empty rows :arg: DataFrame dataframe :return: DataFrame dataframe """ dataframe = dataframe.dropna(how='all') for col in dataframe: dataframe[col] = dataframe[col].apply(lambda x : np.nan() if str(x).isspace() else x) dataframe[col] = dataframe[col].fillna(dataframe[col].mean()) return dataframe
def __init__(self, illuminance_transfer_matrix, hyp_parameters, num_points): self.num_user, self.num_LED = illuminance_transfer_matrix.shape self.illuminance = np.zeros((self.num_user, 1)) self.dimming = np.zeros((self.num_LED, 1)) self.hyp_parameters = hyp_parameters self.illuminance_set = self.feasible_illuminance(illuminance_transfer_matrix, num_points) self.acq_func = np.ones(self.illuminance_set.size, 1) / self.illuminance_set.size self.est_sat_fun = np.array([]) self.est_pref_illuminance = np.nan((self.num_user, 1)) self.observed_illuminance = np.array([]) self.observed_feedback = np.array([])
def quater(mn,yr): if mn in [1,2,3]: return "Q1-"+str(yr) elif mn in [4,5,6]: return "Q2-"+str(yr) elif mn in [7,8,9]: return "Q3-"+str(yr) elif mn in [10,11,12]: return "Q4-"+str(yr) else: return np.nan()
def get_data_for_EEMD(sym, column, per, perend): datasetname = 'cmcdataset' store = StoreDF.select_HDFstore(datasetname) [window, period, periodend] = mungeData.conv_win_2_block(0, per, perend) df = store.get(sym) df = df.loc[~df.index.duplicated(keep='first')] try: datablock = df[column].iloc[-(period + periodend):-periodend] except Exception as e: print 'Period outside of length of block ', e datablock = np.nan() return datablock
def getSimilarsByDistance(signal, tolerance=0, maxSimilarNumber=np.Inf): # Select elements by their closeness first, select required number of the closest ones in groups. groupedIndexes=[] groupedValues=[] while not np.all(np.isnan(signal)): elemIndex = np.nonzero(not np.isnan(signal))[0] elem = signal[elemIndex] indexes = np.nonzero(signal-elem <= tolerance) groupedValues.append(np.mean(signal[indexes])) signal[indexes] = np.nan(indexes.shape) groupedIndexes.append(indexes) return groupedValues, groupedIndexes
def get_dataset_metafeature_from_openml(task_id): task = openml.tasks.get_task(task_id) dataset = openml.datasets.get_dataset(task.dataset_id) features = [] for f in list_metafeatures: try: val = dataset.qualities[f] if not np.nan(val): features.append(val) else: features.append(0) except: features.append(0) return features
def costheta_wrt_reference(self,reference=None, DEBUG=False): #second3vector=(0,0,1),NewTriadFromLorentzVector=NewTriadFromLorentzVector): if reference is not None: _beta=self.beta_vector() # is an np.array _beta_u=u.versor(_beta) _reference=reference.beta_vector() # is an np.array _reference_u=u.versor(_reference) cos = contract_tuples(_beta_u, _reference_u, metric = None) if DEBUG: print('*******','costheta_wrt_reference','*******') print('cos',cos,' theta:',np.arccos(cos)) return cos else: print('The named paramter *reference* needs to be specified') return np.nan()
def get_current_season(date_): if isinstance(fecha_stock_actual_start, datetime.datetime): date_fisrt_season = datetime.datetime(2016, 1, 1) # delta_month = (date_.year - date_fisrt_season.year) * 12 + date_.month - date_fisrt_season.month delta_season = (date_.year - date_fisrt_season.year) * 2 if date_.month <= 6: season = delta_season + 1 else: season = delta_season + 2 else: print('Shoud be datetime') season = np.nan() return season
def phi_wrt_reference(self,reference=None,second3vector=(0,0,1), DEBUG=False,NewTriadFromLorentzVector=NewTriadFromLorentzVector): if reference is not None: newBasis=reference.NewTriadFromLorentzVector(second3vector=second3vector ) print('*******','phi_wrt_reference','*******') if DEBUG: print(newBasis['x2prime']) if DEBUG: print(newBasis['vectors']) newpl=self.Change3DBasis(newBasis['vectors']) newreference=reference.Change3DBasis(newBasis['vectors']) if DEBUG: print('newreference') if DEBUG: newreference.print_fv() return newpl.phi() else: print('The named paramter *reference* needs to be specified') return np.nan()
def calc_median_manual(self, loc=0.0, scale=1.0): shape = (self.a + self.b).shape i1 = np.ones((1,)) alpha = self.alpha * i1 beta = self.beta * i1 id = np.logical_and(self.alpha > 1.0, self.beta > 1.0) if np.any(id == False): self.logger.warning("No closed form of median for beta distribution for alpha or beta <= 1.0!" + "\nReturning nan!") median = np.nan((alpha+beta).shape) id = np.where(id)[0] median[id] = (alpha[id] - 1.0/3) / (alpha[id] + beta[id] - 2.0/3) return np.reshape(median, shape) + loc else: self.logger.warning("Approximate calculation for median of beta distribution!") return (self.alpha - 1.0/3) / (self.alpha + self.beta - 2.0/3) + loc
def _count_fullres_per_lowres_bead(multiscale_factor, lengths, ploidy, fullres_torm=None): """Count the number of full-res beads corresponding to each low-res bead. """ if multiscale_factor == 1: return None fullres_indices = _get_struct_indices( ploidy=ploidy, multiscale_factor=multiscale_factor, lengths=lengths).reshape(multiscale_factor, -1) if fullres_torm is not None and fullres_torm.sum() != 0: fullres_indices[fullres_indices == np.where(fullres_torm)[0]] = np.nan() return (~ np.isnan(fullres_indices)).sum(axis=0)
def sintheta_wrt_reference(self,reference=None, DEBUG=False): #second3vector=(0,0,1),NewTriadFromLorentzVector=NewTriadFromLorentzVector): if reference is not None: _beta=self.beta_vector() # is an np.array _beta_u=u.versor(_beta) _reference=reference.beta_vector() # is an np.array _reference_u=u.versor(_reference) _ort = np.cross(_beta_u, _reference_u) sin = np.sqrt( contract_tuples(_ort, _ort, metric = None) ) if DEBUG: print('*******','sintheta_wrt_reference','*******') print('sin',sin,' theta:',np.arcsin(sin)) return sin else: print('The named paramter *reference* needs to be specified') return np.nan()
def coeffs_line2(R): x1 = R[0] y1 = R[1] x2 = R[2] y2 = R[3] B = None if (x2 - x1) == 0: A = 1 B = 0 else: A = 1 / (x2 - x1) if y2 - y1 == 0: B = 1 A = 0 elif np.nan(B): B = -1 / (y2 - y1) C = y1 * (-B) - x1 * A coeffs = [A, B, C] return coeffs
def get_current_season(date_): ''' Return the season of the indicates date :param date_: date.time date :return: int number of the season ''' if isinstance(date_, datetime.datetime): date_fisrt_season = datetime.datetime(2016, 1, 1) delta_season = (date_.year - date_fisrt_season.year) * 2 if date_.month <= 6: season = delta_season + 1 else: season = delta_season + 2 else: print('Shoud be datetime') season = np.nan() return season
def get_delta_theta_e_sol(bpr): """ Appendix B.7 in [prEN 15316-2:2014] delta_theta_e_sol = 8K -- for medium window fraction or internal loads (e.g. residential) delta_theta_e_sol = 12K -- for large window fraction or internal loads (e.g. office) :param bpr: :return: """ if 0 <= bpr.architecture.win_wall < 0.5: # TODO fix criteria delta_theta_e_sol = 8 # (K) elif 0.5 <= bpr.architecture.win_wall < 1.0: delta_theta_e_sol = 12 # (K) else: delta_theta_e_sol = np.nan() print('Error! Unknown window to wall ratio') return delta_theta_e_sol
def binary_search(f, lower, upper, error): """ Perform binary search to find a root of f within the bounds [lower, upper] to the specified error. Parameters ---------- f : (float -> float) A continuous real-valued scalar function defined in the range [lower, upper]. lower: float Lower limit of binary search upper: float Upper limit of binary search error : float Maximum error of result Returns ------- float: root of f float: number of iterations """ a = lower b = upper count = 0 if sign(f(a)) == sign( f(b)): # This shouldn't happen if I choose my limits right. print("No root detected. Choose different limits.", file=sys.stderr) return np.nan(), 0 while b - a > error: m = (b + a) / 2 count += 1 if sign(f(a)) == sign( f(m) ): #If both f(a) and f(m) are the same sign, then the zero is betwen m and b a = m else: b = m return m, count
def extract_from_indices(self, idxs, check_bounds=False): ''' Helper function to extract the points referred to by the 3D indices 'idxs' If check_bound is true, then will check all the idxs to see if they are valid invalid idxs will get a nan returned (Could also make it so invalid idxs don't get anything returned...) ''' assert idxs.shape[1] == 3 if check_bounds: print "Warning - not tested this bit yet" # create output array, find which are valid idxs and look up their values output_array = np.nan(idxs.shape[0], 3) valid_idxs = self.find_valid_idx(idx) output_array[valid_idxs] = self.V[idxs[valid_idxs, 0], idxs[valid_idxs, 1], idxs[valid_idxs, 2]] return output_array else: return self.V[idxs[:, 0], idxs[:, 1], idxs[:, 2]]
def getEvents(events): ''' This function gets all the events from a half-time element and return them as dictonaries ''' eventList=[] for n in range(1,len(events)): kind = events[n].span.get('title') if events[n].div.get('style') == 'float:left': time = events[n].td.text hometeam = True elif events[n].div.get('style') == 'float:right': time = events[n].findAll('td', 'match-sum-wd-minute')[1].text hometeam = False else: hometeam = np.nan() if kind != 'Substitute in': name = events[n].div.text.strip() eventDict = {'time':time, 'kind':kind, 'hometeam':hometeam, 'player':name } else: playerIn = events[n].div.text.strip() playerOut = 'PLAYER OUT NOT DEFINED' if (len(events[n].findAll('a')) == 2): playerOut = events[n].findAll('a')[1].text.strip() eventDict = {'time':time, 'kind':kind, 'hometeam':hometeam, 'playerIn':playerIn, 'playerOut':playerOut } eventList.append(eventDict) return eventList
def extract_wc_over_semantic_classes(directory): for f in os.listdir(os.fsencode(directory)): if f.endswith(b'.csv'): file_name = f.decode('utf-8') df = pd.read_csv(directory + '/' + file_name) print(df.shape) try: df['wc_over_semantic_classes'] = df['WC'].values / df[ 'semantic_classes'].values print(np.nan(df['semantic_classes'])) except Exception as e: print(e) finally: df['wc_over_semantic_classes'].replace(np.inf, 0, inplace=True) df['wc_over_semantic_classes'].replace(np.nan, 0, inplace=True) print(df['WC'].head()) print(df['semantic_classes'].head()) print(df['wc_over_semantic_classes'].head()) print(df.shape) df.to_csv(directory + '/' + file_name, index=False)
def run_model(id): from patsy import dmatrix from pandas import Series import numpy as np import hddm dataHDDM = hddm.load_csv('DDM/dataHDDM_pmt.csv') dataHDDM["subj_idx"] = dataHDDM["participant"] del dataHDDM["participant"] dataHDDM["SAT"] = dataHDDM.apply(lambda row: 0 if row['SAT'] == "Accuracy" else 1, axis=1) dataHDDM["FC"] = dataHDDM.apply(lambda row: -0.5 if row['FC'] == "low" else 0.5, axis=1) dataHDDM["contrast"] = dataHDDM.contrast.replace([1,2,3,4,5,6], [-.5,-.3,-.1,.1,.3,.5]) dataHDDM["givenResp"] = dataHDDM["response"] dataHDDM["stim"] = dataHDDM.apply(lambda row: 1 if row['stim'] == 'Right' else 0, axis=1) dataHDDM["response"] = dataHDDM.apply(lambda row: 1 if row['givenResp'] == 'Right' else 0, axis=1) def v_link_func(x, data=dataHDDM): stim = (np.asarray(dmatrix('0 + C(s, [[1], [-1]])', {'s': data.stim.ix[x.index]}))) return x*stim if id < 4: ############## M1 LM = [{'model':'t ~ SAT + FC + contrast + SAT:FC + SAT:contrast + FC:contrast + SAT:FC:contrast', 'link_func': lambda x: x} , {'model':'v ~ contrast', 'link_func':v_link_func} , {'model':'a ~ FC + SAT + SAT:FC', 'link_func': lambda x: x} ] deps = {'sz' : 'SAT'} inc = ['sv','sz','st','z'] model_name = "Joint_t0" else : return np.nan() name = 'light_reg_PMT_%s' %str(id) m = hddm.HDDMRegressor(dataHDDM, LM , depends_on = deps, include=inc, group_only_nodes=['sv', 'sz','st', "sz_SAT"], group_only_regressors=False, keep_regressor_trace=True) m.find_starting_values() m.sample(iter=10000, burn=8500, thin=1, dbname='DDM/traces/db_%s'%name, db='pickle') m.save('DDM/Fits/%s'%name) return m
def labeling_vertebrae_T2( label, input_anat, input_centerline, input_surface, output_centerline_vertebra, output_surface_vertebra, surface_do ): command = "fslhd " + input_anat result = commands.getoutput(command) orientation = ( result[result.find("qform_xorient") + 15] + result[result.find("qform_yorient") + 15] + result[result.find("qform_zorient") + 15] ) if orientation != "ASR": print "\nReorient input volume to AP SI RL orientation..." sct.run(sct.fsloutput + "fslswapdim tmp.anat AP SI RL " + label.input_path + "/tmp.anat_orient") sct.run(sct.fsloutput + "fslswapdim tmp.centerline AP SI RL " + label.input_path + "/tmp.centerline_orient") # load_images anat_file = nibabel.load(label.input_path + "/tmp.anat_orient") anat = anat_file.get_data() hdr = anat_file.get_header() dims = hdr["dim"] scales = hdr["pixdim"] # if surface_do==1: # surface_file = nibabel.load(input_surface_reorient) # surface = surface_file.get_data() centerline_file = nibabel.load(label.input_path + "/tmp.centerline_orient") centerline = centerline_file.get_data() else: # loading images print "\nLoading Images..." anat_file = nibabel.load(input_anat) anat = anat_file.get_data() hdr = anat_file.get_header() dims = hdr["dim"] scales = hdr["pixdim"] # if surface_do==1: # surface_file = nibabel.load(input_surface) # surface = surface_file.get_data() centerline_file = nibabel.load(input_centerline) centerline = centerline_file.get_data() # ================================================== # Calculation of the profile intensity # ================================================== print "\nCalculation of the profile intensity..." shift_AP = label.shift_AP * scales[1] size_AP = label.size_AP * scales[1] size_RL = label.size_RL * scales[3] np.uint16(anat) X, Y, Z = np.where(centerline > 0) j = np.argsort(Y) y = Y[j] x = X[j] z = Z[j] # eliminating double in y index = 0 for i in range(len(y) - 1): if y[i] == y[i + 1]: if index == 0: index_double = i else: index_double = np.resize(index_double, index + 1) index_double[index] = i index = index + 1 mask = np.ones(len(y), dtype=bool) mask[index_double] = False y = y[mask] x = x[mask] z = z[mask] # shift the centerline to the spine of shift_AP x1 = np.round(x - shift_AP / scales[1]) # build intensity profile along the centerline I = np.zeros((len(y), 1)) for index in range(len(y)): lim_plus = index + 5 lim_minus = index - 5 if lim_minus < 0: lim_minus = 0 if lim_plus >= len(x1): lim_plus = len(x1) - 1 # normal vector of the orthogonal plane to the centerline i.e tangent vector to the centerline Vx = x1[lim_plus] - x1[lim_minus] Vz = z[lim_plus] - z[lim_minus] Vy = y[lim_plus] - y[lim_minus] d = Vx * x1[index] + Vy * y[index] + Vz * z[index] for i_slice_RL in range(2 * np.int(round(size_RL / scales[3]))): for i_slice_AP in range(2 * np.int(round(size_AP / scales[1]))): result = (d - Vx * (x1[index] + i_slice_AP - size_AP - 1) - Vz * z[index]) / Vy if result > anat.shape[1]: result = anat.shape[1] I[index] = ( I[index] + anat[ np.int(round(x1[index] + i_slice_AP - size_AP - 1)), np.int(round(result)), np.int(round(z[index] + i_slice_RL - size_RL - 1)), ] ) # Detrending Intensity print "\nDetrending Intensity..." start_centerline_y = y[0] X = np.where(I == 0) mask2 = np.ones((len(y), 1), dtype=bool) mask2[X, 0] = False # I = I[mask2] if label.verbose == 1: pl.plot(I) pl.xlabel("direction superior-inferior") pl.ylabel("intensity") pl.title("Intensity profile along the shifted spinal cord centerline") pl.show(block=False) frequency = scipy.fftpack.fftfreq(len(I[:, 0]), d=1) spectrum = np.abs(scipy.fftpack.fft(I[:, 0], n=None, axis=-1, overwrite_x=False)) Wn = np.amax(frequency) / 10 N = 5 # Order of the filter b, a = scipy.signal.iirfilter(N, Wn, rp=None, rs=None, btype="low", analog=False, ftype="bessel", output="ba") I_fit = scipy.signal.filtfilt(b, a, I[:, 0], axis=-1, padtype="constant", padlen=None) if label.verbose == 1: pl.plot(I[:, 0]) pl.plot(I_fit) pl.show() I_detrend = np.zeros((len(I[:, 0]), 1)) I_detrend[:, 0] = I[:, 0] - I_fit I_detrend = I_detrend / abs((np.amin(I_detrend))) if label.verbose == 1: pl.plot(I_detrend) pl.xlabel("direction superior-inferior") pl.ylabel("intensity") pl.title("Intensity profile along the shifted spinal cord centerline after detrending and basic normalization") pl.show(block=False) info_1 = input("Is the more rostral vertebrae the C1 or C2 one? if yes, enter 1 otherwise 0:") if info_1 == 0: level_start = input( "enter the level of the more rostral vertebra - choice of the more rostral vertebral level of the field of view:" ) else: level_start = 2 mean_distance_dict = scipy.io.loadmat(label.mean_distance_mat) mean_distance = (mean_distance_dict.values()[2]).T C1C2_distance = mean_distance[0:2] mean_distance = mean_distance[level_start + 1 : len(mean_distance)] space = np.linspace(-5 / scales[2], 5 / scales[2], round(11 / scales[2]), endpoint=True) pattern = (np.sinc((space * scales[2]) / 15)) ** (20) xmax_pattern = np.argmin(pattern) pixend = len(pattern) - xmax_pattern # ================================================== # step 1 : find the first peak # ================================================== print "\nFinding the First Peak..." pattern1 = np.concatenate((pattern, np.zeros(len(I_detrend[:, 0]) - len(pattern)))) corr_all = scipy.signal.correlate(I_detrend[:, 0], pattern1) loc_corr = np.arange(-np.round((len(corr_all) / 2)), np.round(len(corr_all) / 2) + 2) index_fp = 0 count = 0 for i in range(len(corr_all)): if corr_all[i] > 0.1: if i == 0: if corr_all[i] < corr_all[i + 1]: index_fp = i count = count + 1 elif i == (len(corr_all) - 1): if corr_all[i] < corr_all[i - 1]: index_fp = np.resize(index_fp, count + 1) index_fp[len(index_fp) - 1] = i else: if corr_all[i] < corr_all[i + 1]: index_fp = np.resize(index_fp, count + 1) index_fp[len(index_fp) - 1] = i count = count + 1 elif corr_all[i] < corr_all[i - 1]: index_fp = np.resize(index_fp, count + 1) index_fp[len(index_fp) - 1] = i count = count + 1 else: if i == 0: index_fp = i count = count + 1 else: index_fp = np.resize(index_fp, count + 1) index_fp[len(index_fp) - 1] = i count = count + 1 mask_fp = np.ones(len(corr_all), dtype=bool) mask_fp[index_fp] = False value = corr_all[mask_fp] loc_corr = loc_corr[mask_fp] loc_corr = loc_corr - I_detrend.shape[0] loc_first_peak = xmax_pattern - loc_corr[np.amax(np.where(value > 0.6))] Mcorr1 = value[np.amax(np.where(value > 0.6))] # building the pattern that has to be added at each iteration in step 2 if loc_first_peak >= 0: template_truncated = pattern[(loc_first_peak + 1) :] else: template_truncated = np.concatenate((np.zeros(abs(loc_first_peak)), pattern)) xend = len(template_truncated) if label.verbose == 1: pl.plot(template_truncated) pl.plot(I_detrend) pl.title("Detection of First Peak") pl.xlabel("direction anterior-posterior (mm)") pl.ylabel("intensity") pl.show(block=False) # smoothing the intensity curve---- I_detrend[:, 0] = scipy.ndimage.filters.gaussian_filter1d(I_detrend[:, 0], 10) loc_peak_I = np.arange(len(I_detrend[:, 0])) count = 0 index_p = 0 for i in range(len(I_detrend[:, 0])): if I_detrend[i] > 0.05: if i == 0: if I_detrend[i, 0] < I_detrend[i + 1, 0]: index_p = i count = count + 1 elif i == (len(I_detrend[:, 0]) - 1): if I_detrend[i, 0] < I_detrend[i - 1, 0]: index_p = np.resize(index_p, count + 1) index_p[len(index_p) - 1] = i else: if I_detrend[i, 0] < I_detrend[i + 1, 0]: index_p = np.resize(index_p, count + 1) index_p[len(index_p) - 1] = i count = count + 1 elif I_detrend[i, 0] < I_detrend[i - 1, 0]: index_p = np.resize(index_p, count + 1) index_p[len(index_p) - 1] = i count = count + 1 else: if i == 0: index_p = i count = count + 1 else: index_p = np.resize(index_p, count + 1) index_p[len(index_p) - 1] = i count = count + 1 mask_p = np.ones(len(I_detrend[:, 0]), dtype=bool) mask_p[index_p] = False value_I = I_detrend[mask_p] loc_peak_I = loc_peak_I[mask_p] count = 0 for i in range(len(loc_peak_I) - 1): if i == 0: if loc_peak_I[i + 1] - loc_peak_I[i] < round(10 / scales[1]): index = i count = count + 1 else: if (loc_peak_I[i + 1] - loc_peak_I[i]) < round(10 / scales[1]): index = np.resize(index, count + 1) index[len(index) - 1] = i count = count + 1 elif (loc_peak_I[i] - loc_peak_I[i - 1]) < round(10 / scales[1]): index = np.resize(index, count + 1) index[len(index) - 1] = i count = count + 1 mask_I = np.ones(len(value_I), dtype=bool) mask_I[index] = False value_I = -value_I[mask_I] loc_peak_I = loc_peak_I[mask_I] from scipy.interpolate import UnivariateSpline fit = UnivariateSpline(loc_peak_I, value_I) P = fit(np.arange(len(I_detrend))) if label.verbose == 1: pl.xlim(0, len(I_detrend) - 1) pl.plot(loc_peak_I, value_I) pl.plot(I_detrend) pl.plot(P) pl.title("Setting values of peaks at one by fitting a smoothing spline") pl.xlabel("direction superior-inferior (mm)") pl.ylabel("normalized intensity") pl.show(block=False) for i in range(len(I_detrend)): if P[i] > 0.1: I_detrend[i, 0] = I_detrend[i, 0] / abs(P[i]) # =================================================================================== # step 2 : Cross correlation between the adjusted template and the intensity profile # local moving of template's peak from the first peak already found # =================================================================================== mean_distance_new = mean_distance mean_ratio = np.zeros(len(mean_distance)) L = np.round(1.2 * max(mean_distance)) - np.round(0.8 * min(mean_distance)) corr_peak = np.nan(np.zeros((L, len(mean_distance)))) for i_peak in range(len(mean_distance)): scale_min = np.round(0.80 * mean_distance_new[i_peak]) - xmax_pattern - pixend if scale_min < 0: scale_min = 0 scale_max = np.round(1.2 * mean_distance_new[i_peak]) - xmax_pattern - pixend scale_peak = np.arange(scale_min, scale_max + 1) for i_scale in range(len(scale_peak)): template_resize_peak = np.concatenate([template_truncated, np.zeros(scale_peak[i_scale]), pattern]) if len(I_detrend[:, 0]) > len(template_resize_peak): template_resize_peak1 = np.concatenate( (template_resize_peak, np.zeros(len(I_detrend[:, 0]) - len(template_resize_peak))) ) corr_template = scipy.signal.correlate(I_detrend[:, 0], template_resize_peak) if len(I_detrend[:, 0]) > len(template_resize_peak): val = np.dot(I_detrend[:, 0], template_resize_peak1.T) else: I_detrend_2 = np.concatenate( (I_detrend[:, 0], np.zeros(len(template_resize_peak) - len(I_detrend[:, 0]))) ) val = np.dot(I_detrend_2, template_resize_peak.T) corr_peak[i_scale, i_peak] = val if label.verbose == 1: pl.xlim(0, len(I_detrend[:, 0])) pl.plot(I_detrend[:, 0]) pl.plot(template_resize_peak) pl.show(block=False) pl.plot(corr_peak[:, i_peak], marker="+", linestyle="None", color="r") pl.title("correlation value against the displacement of the peak (px)") pl.show(block=False) max_peak = np.amax(corr_peak[:, i_peak]) index_scale_peak = np.where(corr_peak[:, i_peak] == max_peak) good_scale_peak = scale_peak[index_scale_peak][0] Mcorr = Mcorr1 Mcorr = np.resize(Mcorr, i_peak + 2) Mcorr[i_peak + 1] = np.amax(corr_peak[:, 0 : (i_peak + 1)]) flag = 0 if i_peak > 0: if (Mcorr[i_peak + 1] - Mcorr[i_peak]) < 0.4 * np.mean(Mcorr[1 : i_peak + 2] - Mcorr[0 : i_peak + 1]): test = i_peak template_resize_peak = np.concatenate( (template_truncated, np.zeros(round(mean_distance[i_peak]) - xmax_pattern - pixend), pattern) ) good_scale_peak = np.round(mean_distance[i_peak]) - xmax_pattern - pixend flag = 1 if i_peak == 0: if (Mcorr[i_peak + 1] - Mcorr[i_peak]) < 0.4 * Mcorr[0]: template_resize_peak = np.concatenate( (template_truncated, np.zeros(round(mean_distance[i_peak]) - xmax_pattern - pixend), pattern) ) good_scale_peak = round(mean_distance[i_peak]) - xmax_pattern - pixend flag = 1 if flag == 0: template_resize_peak = np.concatenate((template_truncated, np.zeros(good_scale_peak), pattern)) mean_distance_new[i_peak] = good_scale_peak + xmax_pattern + pixend mean_ratio[i_peak] = np.mean(mean_distance_new[:, 0:i_peak] / mean_distance[:, 0:i_peak]) template_truncated = template_resize_peak if label.verbose == 1: pl.plot(I_detrend[:, 0]) pl.plot(template_truncated) pl.xlim(0, (len(I_detrend[:, 0]) - 1)) pl.show(block=False) minpeakvalue = 0.5 loc_disk = np.arange(len(template_truncated)) count = 0 index_disk = 0 for i in range(len(template_truncated)): if template_truncated[i] >= minpeakvalue: if i == 0: if template_truncated[i] < template_truncated[i + 1]: index_disk = i count = count + 1 elif i == (len(template_truncated) - 1): if template_truncated[i] < template_truncated[i - 1]: index_disk = np.resize(index_disk, count + 1) index_disk[len(index_disk) - 1] = i else: if template_truncated[i] < template_truncated[i + 1]: index_disk = np.resize(index_disk, count + 1) index_disk[len(index_disk) - 1] = i count = count + 1 elif template_truncated[i] < template_truncated[i - 1]: index_disk = np.resize(index_disk, count + 1) index_disk[len(index_disk) - 1] = i count = count + 1 else: if i == 0: index_disk = i count = count + 1 else: index_disk = np.resize(index_disk, count + 1) index_disk[len(index_disk) - 1] = i count = count + 1 mask_disk = np.ones(len(template_truncated), dtype=bool) mask_disk[index_disk] = False loc_disk = loc_disk[mask_disk] X1 = np.where(loc_disk > I_detrend.shape[0]) mask_disk1 = np.ones(len(loc_disk), dtype=bool) mask_disk1[X1] = False loc_disk = loc_disk[mask_disk1] loc_disk = loc_disk + start_centerline_y - 1 # ===================================================================== # Step 3: Building of labeled centerline and surface # ===================================================================== print "\nBuilding of Labeled Centerline and Surface..." for i in range(len(loc_disk)): Index = np.array(np.where(y == loc_disk[i])).T lim_plus = Index + 5 lim_minus = Index - 5 if lim_minus < 1: lim_minus = 1 if lim_plus > len(x): lim_plus = len(x) Vx = x[lim_plus] - x[lim_minus] Vz = z[lim_plus] - z[lim_minus] Vy = y[lim_plus] - y[lim_minus] d = Vx * x1[Index] + Vy * y[Index] + Vz * z[Index] intersection = np.ones(len(x)) for j in range(len(x)): intersection[j] = np.abs((Vx * x[j] + Vy * y[j] + Vz * z[j] - d)) min_intersection = np.amin(intersection) index_intersection = np.where(min_intersection == intersection) loc_disk[i] = y[index_intersection] center_disk = np.array(centerline) for i in range(len(loc_disk) - 1): tmp = center_disk[loc_disk[i] : loc_disk[i + 1]] tmp[np.where(tmp == 1)] = i + level_start center_disk[loc_disk[i] : loc_disk[i + 1]] = tmp center_disk[np.where(center_disk == 1)] = 0 if level_start == 2: center_disk[x[0], (int(round(loc_disk[0] - C1C2_distance[1])) - 1) : loc_disk[0], z[0]] = 2 center_disk[ x[0], (int(round(loc_disk[0] - C1C2_distance[0] - C1C2_distance[1])) - 1) : int( round(loc_disk[0] - C1C2_distance[1] - 1) ), z[0], ] = 1 # Write NIFTI volumes hdr.set_data_dtype("uint8") # set imagetype to uint8 print "\nWrite NIFTI volumes..." img = nibabel.Nifti1Image(center_disk, None, hdr) file_name = output_centerline_vertebra nibabel.save(img, file_name) print ".. File created:" + file_name
tup = (1, 2, 3, 4, 5) #%% #Numpy arrays import numpy as np a = np.array([1, 2, 3, 4]) a.mean a.mean() a.std() np.std(a) np.mean(a) a.append(6) c = np.array([1, 2, np.NAN, 3, 4]) c.ndim c.shape c[~np.nan(c)] c[~np.isnan(c)] np.mean(c) np.mean(c[~np.isnan(c)]) #%% #Fibonacci sequence def fib(n): '''Print a Fibonacci series up to n''' a, b = 0, 1 while a < n: print a, a, b = b, a+b fib(1000) #%%
def nestData(self, universalFirstDate, universalLastDate): frontLength = self.firstDate - universalFirstDate backLength = universalLastDate - self.lastDate self.allData = np.hstack([np.nan(1,frontLength), self.currentData, np.nan(1,backLength)])
def __init__(self): self.gamma = np.nan() # unit weight self.E = np.nan() #Young Modulus self.name = 'None' #name of material self.poisson_ratio = 0.25
def test_weird_data(): with raises(TypeError): binary_search([np.nan(), None, 0.1, 0.5], 0.5)