def get_beta(pval_read, raw_beta, beta_read, line_dict, header_dict, se, z_from_se, N, eff_type, se_inferred_zscores): if eff_type == 'BLUP': return raw_beta if z_from_se: assert se in header_dict, "SE was not specified in summary statistics provided, which is necessary when the 'z_from_se' flag is used." se_read = float(line_dict[header_dict[se]]) if se_read == 0 or not isfinite(se_read): return None se_inferred_zscores += 1 return get_beta_from_se(beta_read, se_read, eff_type, raw_beta, N) else: if pval_read == 0 or not isfinite(stats.norm.ppf(pval_read)): #Attempt to Parse SEs to infer Z-score if not se in header_dict: return None else: se_read = float(line_dict[header_dict[se]]) if se_read == 0 or not isfinite(se_read): return None se_inferred_zscores += 1 return get_beta_from_se(beta_read, se_read, eff_type, raw_beta, N) else: #return sp.sign(raw_beta) * stats.norm.ppf(pval_read / 2.0)/ sp.sqrt(N) return -1 * sp.sign(raw_beta) * stats.norm.ppf( pval_read / 2.0) / sp.sqrt(N)
def __init__(self, kind, dic_init): self.kind = kind if (self.kind == 'auto'): fname = dic_init['data_auto'] elif (self.kind == 'cross'): fname = dic_init['data_cross'] elif (self.kind == 'autoQSO'): fname = dic_init['data_autoQSO'] rmin = dic_init['rmin'] rmax = dic_init['rmax'] mumin = dic_init['mumin'] mumax = dic_init['mumax'] bin_size = dic_init['bin_size'] h = pyfits.open(fname) da = h[1].data.DA co = h[1].data.CO self.dm = h[1].data.DM self.rt = h[1].data.RT self.rp = h[1].data.RP self.z = h[1].data.Z self.da_all = copy.deepcopy(da) self.co_all = copy.deepcopy(co) ### Get the center of the bins from the regular grid bin_center_rt = np.zeros(self.rt.size) bin_center_rp = np.zeros(self.rp.size) for i in np.arange(-self.rt.size - 1, self.rt.size + 1, 1).astype('int'): bin_center_rt[sp.logical_and(self.rt >= bin_size * i, self.rt < bin_size * (i + 1.))] = bin_size * (i + 0.5) bin_center_rp[sp.logical_and(self.rp >= bin_size * i, self.rp < bin_size * (i + 1.))] = bin_size * (i + 0.5) r = np.sqrt(bin_center_rt**2 + bin_center_rp**2) mu = bin_center_rp / r cuts = (r > rmin) & (r < rmax) & (mu >= mumin) & (mu <= mumax) if sp.isfinite(dic_init['r_per_min']): cuts = cuts & (bin_center_rt > dic_init['r_per_min']) if sp.isfinite(dic_init['r_per_max']): cuts = cuts & (bin_center_rt < dic_init['r_per_max']) if sp.isfinite(dic_init['r_par_min']): cuts = cuts & (bin_center_rp > dic_init['r_par_min']) if sp.isfinite(dic_init['r_par_max']): cuts = cuts & (bin_center_rp < dic_init['r_par_max']) co = co[:, cuts] co = co[cuts, :] da = da[cuts] self.cuts = cuts self.da = da self.co = co self.ico = sp.linalg.inv(co)
def patch_holes(data_map): r""" Fills in any areas with a non finite value by taking a linear average of the nearest non-zero values along each axis """ # # getting coordinates of all valid data points data_vector = sp.ravel(data_map) inds = sp.where(sp.isfinite(data_vector))[0] points = sp.unravel_index(inds, data_map.shape) values = data_vector[inds] # # linearly interpolating data to fill gaps xi = sp.where(~sp.isfinite(data_vector))[0] msg = '\tattempting to fill %d values with a linear interpolation' logger.debug(msg, xi.size) xi = sp.unravel_index(xi, data_map.shape) intrp = griddata(points, values, xi, fill_value=sp.nan, method='linear') data_map[xi[0], xi[1]] = intrp # # performing a nearest interpolation any remaining regions data_vector = sp.ravel(data_map) xi = sp.where(~sp.isfinite(data_vector))[0] msg = '\tfilling %d remaining values with a nearest interpolation' logger.debug(msg, xi.size) xi = sp.unravel_index(xi, data_map.shape) intrp = griddata(points, values, xi, fill_value=0, method='nearest') data_map[xi[0], xi[1]] = intrp # return data_map
def normalize(series): ''' Returns the series demeaned and divided by its standard deviation. ''' mean = series[sp.isfinite(series)].mean() sdev = series[sp.isfinite(series)].std() return (series - mean) / sdev
def obs_d22(station, day, numdays=1, varlist=[]): ''' station: offshore station name as string day: datetime object (should be with 00 hours) returns dictionary with hourly time, Hs, Tp, Tm, FF, DD each of the returned variable is a 2d array with time and the number of available sensors ''' time, WMlist, WIlist = d22.read_d22(station,start=day, end=day+dt.timedelta(numdays-1)) datadict = {'time':time['1hr']} # organize each variable as 2d-arrays with time and sensor number as dimensions Hs, Tp, Tm02 ,Tm01, DDP, DDM, WMnames, FF, DD, WInames = [],[],[],[], [], [], [], [], [], [] for WM in WMlist: if sp.isfinite(WM['Hs']).sum()>1: Hs.append(WM['Hs']) Tp.append(WM['Tp']) Tm02.append(WM['Tm02']) Tm01.append(WM['Tm01']) DDP.append(WM['DDP']) DDM.append(WM['DDM']) WMnames.append(WM['name']) for WI in WIlist: if sp.isfinite(WI['FF']).sum()>1: FF.append(WI['FF']) DD.append(WI['DD']) WInames.append(WI['name']) datadict.update({'Hs':sp.array(Hs), 'Tp':sp.array(Tp), 'Tm02':sp.array(Tm02) , 'Tm01':sp.array(Tm01) , 'FF':sp.array(FF), 'DD':sp.array(DD),'DDM':sp.array(DDM), 'DDP':sp.array(DDP)}) datadict.update({'WMnames':WMnames, 'WInames':WInames}) # save sensor names return datadict
def emissive_radiance_old(emissivity, T, wl): """Radiance of a surface due to emission""" h = 6.62607004e-34 # m2 kg s-1 c = 299792458 # m s-1 numerator = 2.0 * h * (c**2) # m4 kg s-1 wl_m = wl * 1e-9 numerator_per_lam5 = numerator * pow(wl_m, -5) # kg s-1 m-1 k = 1.380648520 - 23 # Boltzmann constant, m2 kg s-2 K-1 denom = s.exp(h * c / (k * wl_m * T)) - 1.0 # dimensionless L = numerator_per_lam5 / denom # Watts per m3 cm2_per_m2, nm_per_m, uW_per_W = 10000, 1e9, 1e6 conversion = cm2_per_m2 * nm_per_m * uW_per_W / s.pi # -> uW nm-1 cm-2 sr-1 L = L * conversion ddenom_dT = s.exp(h * c / (k * wl_m * T)) * h * c * (-1.0) / (pow( k * wl_m * T, 2)) * k * wl_m dL_dT = -numerator_per_lam5 / pow(denom, 2.0) * ddenom_dT * conversion L = L * emissivity dL_dT = dL_dT * emissivity L[s.logical_not(s.isfinite(L))] = 0 dL_dT[s.logical_not(s.isfinite(dL_dT))] = 0 return L, dL_dT
def get_prf_data(pixel_values, pixel_stddev, pixel_offsets, error_threshold=0.1): """ Return the PRF measurements for (a subset of) the image. Args: pixel_values(2-D float array): The calibrated pixel responses from the image to include in the plot. pixel_stddev(2-D float array): The estimated standard deviation of `pixel_values`. pixel_offsets: The slice of the return value of find_pixel_offsets() corresponding to `pixel_values`. Returns: (2-D float array, 2-D float array, 2-D float array, 2-D float array): * The x-offsets of the points at which PRF measurements are available. * The y-offsets of the points at which PRF measurements are available. * The measured normalized PRF at the available offsets * estimated errors of the PRF measurements. """ prf_measurements = ( ( pixel_values - pixel_offsets['zero_point'] ) / pixel_offsets['norm'] ) prf_errors = ( pixel_stddev / pixel_offsets['norm'] ) #False positive #pylint: disable=assignment-from-no-return include = scipy.logical_and(scipy.isfinite(prf_measurements), scipy.isfinite(prf_errors)) include = scipy.logical_and(include, prf_errors < error_threshold) #pylint: enable=assignment-from-no-return return scipy.stack(( pixel_offsets['x_off'][include], pixel_offsets['y_off'][include], prf_measurements[include], prf_errors[include] ))
def bias(a,b): ''' bias ''' a,b = sp.array(a),sp.array(b) mask = sp.logical_and(sp.isfinite(a),sp.isfinite(b)) a, b = a[mask], b[mask] return a.mean()-b.mean()
def intercept(self, ray): """Solves for intersection point of surface and a ray or Beam Args: ray: Ray or Beam object It must be in the same coordinate space as the surface object. Returns: s: value of s [meters] which intercepts along norm, otherwise an empty tuple (for no intersection). Examples: Accepts all point and point-derived object inputs, though all data is stored as a python object. Generate an y direction Ray in cartesian coords using a Vec from (0,0,1):: cen = geometry.Center(flag=True) ydir = geometry.Vecx((0,1,0)) zpt = geometry.Point((0,0,1),cen) """ # Proceedure will be to generate if self._origin is ray._origin: try: rcopy = ray.copy() rcopy.redefine(self) intersect = _beam.interceptCyl(scipy.atleast_2d(rcopy.x()[:,-1]), scipy.atleast_2d(rcopy.norm.unit), scipy.array([self.sagi.s,self.sagi.s]), scipy.array([-self.norm.s,self.norm.s])) + rcopy.norm.s[-1] if not scipy.isfinite(intersect): #relies on r1 using arctan2 so that it sets the branch cut properly (-pi,pi] return None elif self.edgetest(intersect, (rcopy(intersect)).r1()): return intersect else: rcopy.norm.s[-1] = intersect intersect = _beam.interceptCyl(scipy.atleast_2d(rcopy.x()[:,-1]), scipy.atleast_2d(rcopy.norm.unit), scipy.array([self.sagi.s,self.sagi.s]), scipy.array([-self.norm.s,self.norm.s])) + rcopy.norm.s[-1] if not scipy.isfinite(intersect): #relies on r1 using arctan2 so that it sets the branch cut properly (-pi,pi] return None elif self.edgetest(intersect, (rcopy(intersect)).r1()): return None else: return None except AttributeError: raise ValueError('not a surface object') else: raise ValueError('not in same coordinate system, use redefine and try again')
def d22mean(sample): ''' return the mean if at least 3 of 6 10min values are finite Returns NaN only if more than 3 values missing, or if all are the same (std=0) ''' if (sum(sp.isfinite(sample)) > 2 and sp.std(sample) > 0.): average = sp.mean(sample[sp.isfinite(sample)]) else: average = sp.nan return average
def df(x): x_ = X0 x_[Ifilter_x] = x rv = gpr.LMLgrad(param_list_to_dict(x_,param_struct,skeys),*args,**kw_args) rv = param_dict_to_list(rv,skeys) if (~SP.isfinite(rv)).any(): idx = (~SP.isfinite(rv)) rv[idx] = 1E6 return rv[Ifilter_x]
def test_remove_boundary_conditions(self): alg = op.algorithms.GenericTransport(network=self.net, phase=self.phase) alg.set_value_BC(pores=self.net.pores('top'), values=1) alg.set_value_BC(pores=self.net.pores('bottom'), values=0) assert sp.sum(sp.isfinite(alg['pore.bc_value'])) > 0 alg.remove_BC(pores=self.net.pores('top')) assert sp.sum(sp.isfinite(alg['pore.bc_value'])) > 0 alg.remove_BC(pores=self.net.pores('bottom')) assert sp.sum(sp.isfinite(alg['pore.bc_value'])) == 0
def df(x): x_ = X0 x_[Ifilter_x] = x rv = gpr.LMLgrad(param_list_to_dict(x_, param_struct, skeys), *args, **kw_args) rv = param_dict_to_list(rv, skeys) # LG.debug("dL("+str(x_)+")=="+str(rv)) if not SP.isfinite(rv).all(): # SP.isnan(rv).any(): In = ~SP.isfinite(rv)#SP.isnan(rv) rv[In] = 1E6 return rv[Ifilter_x]
def f(x): x_ = X0 x_[Ifilter_x] = x lml = gpr.LML(param_list_to_dict(x_, param_struct, skeys)) if SP.isnan(lml): lml = 1E6 lml_grad = gpr.LMLgrad() lml_grad = param_dict_to_list(lml_grad, skeys) if (~SP.isfinite(lml_grad)).any(): idx = (~SP.isfinite(lml_grad)) lml_grad[idx] = 1E6 return lml, lml_grad[Ifilter_x]
def f(x): x_ = X0 x_[Ifilter_x] = x lml = gpr.LML(param_list_to_dict(x_,param_struct,skeys)) if SP.isnan(lml): lml=1E6 lml_grad = gpr.LMLgrad() lml_grad = param_dict_to_list(lml_grad,skeys) if (~SP.isfinite(lml_grad)).any(): idx = (~SP.isfinite(lml_grad)) lml_grad[idx] = 1E6 return lml, lml_grad[Ifilter_x]
def _box_cox_transform(values, standard=True): """ Performs the Box-Cox transformation, over different ranges, picking the optimal one w. respect to normality. """ a = sp.array(values) if standard: vals = (a - min(a)) + 0.1 * sp.var(a) else: vals = a sw_pvals = [] lambdas = sp.arange(-2.0, 2.1, 0.1) for l in lambdas: if l == 0: vs = sp.log(vals) else: vs = ((vals ** l) - 1) / l r = stats.shapiro(vs) if sp.isfinite(r[0]): pval = r[1] else: pval = 0.0 sw_pvals.append(pval) i = sp.argmax(sw_pvals) l = lambdas[i] if l == 0: vs = sp.log(vals) else: vs = ((vals ** l) - 1) / l return vs
def most_normal_transformation(self, pid, trans_types=['none', 'sqrt', 'log', 'sqr', 'exp', 'arcsin_sqrt'], perform_trans=True, verbose=False): """ Performs the transformation which results in most normal looking data, according to Shapiro-Wilk's test """ #raw_values = self.phen_dict[pid]['values'] from scipy import stats shapiro_pvals = [] for trans_type in trans_types: if trans_type != 'none': if not self.transform(pid, trans_type=trans_type): continue phen_vals = self.get_values(pid) #print 'sp.inf in phen_vals:', sp.inf in phen_vals if sp.inf in phen_vals: pval = 0.0 else: r = stats.shapiro(phen_vals) if sp.isfinite(r[0]): pval = r[1] else: pval = 0.0 shapiro_pvals.append(pval) #self.phen_dict[pid]['values'] = raw_values if trans_type != 'none': self.revert_to_raw_values(pid) argmin_i = sp.argmax(shapiro_pvals) trans_type = trans_types[argmin_i] shapiro_pval = shapiro_pvals[argmin_i] if perform_trans: self.transform(pid, trans_type=trans_type) if verbose: print "The most normal-looking transformation was %s, with a Shapiro-Wilk's p-value of %0.6f" % \ (trans_type, shapiro_pval) return trans_type, shapiro_pval
def PlotLc(id=None, dir=None, quarter=None, tset=None): if id != None: tset, status = GetLc(id=id, dir=dir, tr_out=True) if tset is None: return elif tset == None: print "no tset" return if quarter != None: tset.tables[1] = tset.tables[1].where(tset.tables[1].Q == quarter) if len(tset.tables[1].TIME) == 0: print "No data for Q%d" % quarter return time = tset.tables[1].TIME phase, inTr = TransitPhase(tset) col = ["r", "g", "b", "y", "c", "m", "grey"] npl, nobs = inTr.shape pylab.figure(1) pylab.clf() pylab.plot(time, tset.tables[1].PDCSAP_FLUX, "k-") for ipl in scipy.arange(npl): list = inTr[ipl, :].astype(bool) pylab.plot(time[list], tset.tables[1].PDCSAP_FLUX[list], ".", c=col[ipl]) l = scipy.isfinite(time) pylab.xlim(time[l].min(), time[l].max()) ttl = "KIC %d, P=" % (tset.tables[0].KID[0]) for i in scipy.arange(npl): ttl = "%s%.5f " % (ttl, tset.tables[0].Period[i]) if quarter != None: ttl += "Q%d" % quarter pylab.title(ttl) return
def TransitPhase(tset): lc = copy.deepcopy(tset.tables[1]) time = lc.TIME nobs = len(time) lg = scipy.isfinite(time) pl = tset.tables[0] npl = len(pl.Period) phase = scipy.zeros((npl, nobs)) inTr = scipy.zeros((npl, nobs), "int") for ipl in scipy.arange(npl): period = pl.Period[ipl] t0 = pl.t0[ipl] + BJDREF_t0 - BJDREF_lc dur = pl.Dur[ipl] / 24.0 / period counter = 0 while (time[lg] - t0).min() < 0: t0 -= period counter += 1 if counter > 1000: break ph = ((time - t0) % period) / period ph[ph < -0.5] += 1 ph[ph > 0.5] -= 1 phase[ipl, :] = ph inTr[ipl, :] = abs(ph) <= dur / 1.5 return phase, inTr
def dtw_path(s1, s2): l1 = s1.shape[0] l2 = s2.shape[0] cum_sum = sp.zeros((l1 + 1, l2 + 1)) cum_sum[1:, 0] = sp.inf cum_sum[0, 1:] = sp.inf predecessors = [([None] * l2) for i in range(l1)] for i in range(l1): for j in range(l2): if sp.isfinite(cum_sum[i + 1, j + 1]): dij = sp.linalg.norm(s1[i] - s2[j])**2 pred_list = [ cum_sum[i, j + 1], cum_sum[i + 1, j], cum_sum[i, j] ] argmin_pred = sp.argmin(pred_list) cum_sum[i + 1, j + 1] = pred_list[argmin_pred] + dij if i + j > 0: if argmin_pred == 0: predecessors[i][j] = (i - 1, j) elif argmin_pred == 1: predecessors[i][j] = (i, j - 1) else: predecessors[i][j] = (i - 1, j - 1) i = l1 - 1 j = l2 - 1 best_path = [(i, j)] while predecessors[i][j] is not None: i, j = predecessors[i][j] best_path.insert(0, (i, j)) return best_path
def _fitData(idx, xdata, ydata, bounds=None, method=None, tol=None): """ internal program to interface with scipy.optimize.minimize (BFGS_L) algo """ #ydata =data[idx] # need to subtract baseline from data if not bounds == None: inp, bounds = _assembleInit(xdata, ydata, bounds=bounds) bounds[1::3] *= bounds[3::3] * scipy.sqrt( scipy.pi) / 123. #same for these values else: inp = _assembleInit(xdata, ydata) #inp[0] *= .9 inp[0] /= 123. inp[1::3] *= inp[3::3] * scipy.sqrt( scipy.pi) / 123. #convert to integrated counts # the 123 comes from converting counts to actual photons for the energies observed by the SIF temp = scipy.optimize.minimize(interface, inp, args=(xdata, ydata / 123.), method=method, bounds=bounds, jac=True) output = temp.x if scipy.isfinite(temp.fun): output = scipy.append(output, [temp.fun]) else: output = scipy.append(output, [-1.]) return output
def parent_sample(catalog): cuts = {'upper_g': catalog.apogee.logg <= 2.2, 'nonnull_g': catalog.apogee.logg > 0., # there are a few values less than zero that are not null 'nonnull_k': catalog.apogee.k > 0, 'nonnull_bp_rp': sp.isfinite(catalog.gaia.bp_rp), 'nonnull_w1mpro': sp.isfinite(catalog.wise.w1mpro), 'nonnull_w2mpro': sp.isfinite(catalog.wise.w2mpro), 'nonvariable': catalog.gaia.phot_variable_flag != 'VARAIBLE', 'nonduplicate': ~catalog.tmass.tmass_id.duplicated(), 'photometry_jk': (catalog.apogee.j - catalog.apogee.k) < (.4 + .45*catalog.gaia.bp_rp), 'photometry_hw': (catalog.apogee.h - catalog.wise.w2mpro) > -.05, 'finite_jhk': catalog.apogee[['j', 'h', 'k']].gt(-100).all(1), 'positive_jhk_err': catalog.apogee[['j_err', 'h_err', 'k_err']].gt(0).all(1), 'finite_wise': catalog.wise[['w1mpro', 'w2mpro']].apply(sp.isfinite).all(1), 'positive_wise_err': catalog.wise[['w1mpro_error', 'w2mpro_error']].gt(0).all(1)} return tools.cut(catalog, cuts)
def leap_prob_recurse(self, Z_chain, C, active_idx): """ Recursively compute to cumulative probability of transitioning from the beginning of the chain Z_chain to the end of the chain Z_chain. """ if sp.isfinite(C[0,-1,0]): ## we've already visited this leaf cumu = C[0,-1,:].reshape((1,-1)) return cumu, C if len(Z_chain) == 2: ## the two states are one apart p_acc = self.leap_prob(Z_chain[0], Z_chain[1]) p_acc = p_acc[:,active_idx] C[0,-1,:] = p_acc.ravel() return p_acc, C cum_forward, Cl = self.leap_prob_recurse(Z_chain[:-1], C[:-1,:-1,:], active_idx) C[:-1,:-1,:] = Cl cum_reverse, Cl = self.leap_prob_recurse(Z_chain[:0:-1], C[:0:-1,:0:-1,:], active_idx) C[:0:-1,:0:-1,:] = Cl H0 = self.Eval_H(Z_chain[0]) H1 = self.Eval_H(Z_chain[-1]) Ediff = H0 - H1 Ediff = Ediff[:,active_idx] start_state_ratio = sp.exp(Ediff) prob =((sp.vstack ((1. - cum_forward, start_state_ratio*(1. - cum_reverse)))).min(axis=0)).reshape((1,-1)) cumu = cum_forward + prob C[0,-1,:] = cumu.ravel() return cumu, C
def _box_cox_transform(self, verbose=False, method='standard'): """ Performs the Box-Cox transformation, over different ranges, picking the optimal one w. respect to normality. """ from scipy import stats a = sp.array(self.values) if method == 'standard': vals = (a - min(a)) + 0.1 * sp.var(a) else: vals = a sw_pvals = [] lambdas = sp.arange(-2.0, 2.1, 0.1) for l in lambdas: if l == 0: vs = sp.log(vals) else: vs = ((vals**l) - 1) / l r = stats.shapiro(vs) if sp.isfinite(r[0]): pval = r[1] else: pval = 0.0 sw_pvals.append(pval) i = sp.argmax(sw_pvals) l = lambdas[i] if l == 0: vs = sp.log(vals) else: vs = ((vals**l) - 1) / l self._perform_transform(vs, "box_cox") log.debug('optimal lambda was %0.1f' % l) return True
def atEndPoint(self, val, bdcode): """val, bdcode -> Bool Determines whether val is at the endpoint specified by bdcode, to the precision of the interval's _abseps tolerance. bdcode can be one of 'lo', 'low', 0, 'hi', 'high', 1""" assert self.defined, 'Interval undefined' assert isinstance(val, float) or isinstance(val, int), \ 'Invalid value type' assert isfinite(val), "Can only test finite argument values" if bdcode in ['lo', 'low', 0]: if self.type == Float: return abs(val - self._loval) < self._abseps elif self.type == Int: return val == self._loval else: raise TypeError, "Unsupported value type" elif bdcode in ['hi', 'high', 1]: if self.type == Float: return abs(val - self._hival) < self._abseps elif self.type == Int: return val == self._hival else: raise TypeError, "Unsupported value type" else: raise ValueError, 'Invalid boundary spec code'
def lognpdf(x,mean,sig): if x<0 or not scipy.isfinite(x): pdf = 0 else: a = 1./(x*sig*scipy.sqrt(2*scipy.pi)) pdf = a*scipy.exp(-(scipy.log(x)-mean)**2/(2.*sig**2)) return pdf
def set(self, arg): if type(arg) in [list, tuple, Array, NArray] and len(arg)==2: if arg[0] == arg[1]: self.set(arg[0]) else: self.issingleton = False loval = arg[0] hival = arg[1] assert loval is not NaN and hival is not NaN, \ ("Cannot specify NaN as interval endpoint") if not loval < hival: print "set() was passed loval = ", loval, \ " and hival = ", hival raise PyDSTool_ValueError, ('Interval endpoints must be ' 'given in order of increasing size') self._intervalstr = '['+str(loval)+',' \ +str(hival)+']' self._loval = loval self._hival = hival self.defined = True elif type(arg) in [int, float]: assert isfinite(arg), \ "Singleton interval domain value must be finite" self.issingleton = True self._intervalstr = str(arg) self._loval = arg self._hival = arg self.defined = True else: print "Error in argument: ", arg, "of type", type(arg) raise PyDSTool_TypeError, \ 'Interval spec must be a numeric or a length-2 sequence type'
def PlotLc(id=None, dir = None, quarter=None, tset = None): if id != None: tset, status = GetLc(id = id, dir = dir, tr_out = True) if tset is None: return elif tset == None: print('no tset') return if quarter != None: tset.tables[1] = tset.tables[1].where(tset.tables[1].Q == quarter) if len(tset.tables[1].TIME) == 0: print('No data for Q%d' % quarter) return time = tset.tables[1].TIME phase, inTr = TransitPhase(tset) col = ['r','g','b','y','c','m','grey'] npl, nobs = inTr.shape pylab.figure(1) pylab.clf() pylab.plot(time, tset.tables[1].PDCSAP_FLUX, 'k-') for ipl in scipy.arange(npl): list = inTr[ipl,:].astype(bool) pylab.plot(time[list], tset.tables[1].PDCSAP_FLUX[list], '.', c = col[ipl]) l = scipy.isfinite(time) pylab.xlim(time[l].min(), time[l].max()) ttl = 'KIC %d, P=' % (tset.tables[0].KID[0]) for i in scipy.arange(npl): ttl = '%s%.5f ' % (ttl, tset.tables[0].Period[i]) if quarter != None: ttl += 'Q%d' % quarter pylab.title(ttl) return
def evaluate_using(data_item, using_list, vars_local, style, firsterror, verb_errors, lineunit, fileline, description): errcount = 0 for k in range(len(using_list)): value = gp_eval.gp_eval(using_list[k], vars_local, verbose=False) if (not SCIPY_ABSENT) and (not scipy.isfinite(value)): raise ValueError if (fabs(value) > gp_math.FLT_MAX): raise ValueError if (style[-5:] != "range"): if ((firsterror != None) and (k >= firsterror) and (value < 0.0)): # Check for negative error bars value = 0.0 if (verb_errors): gp_warning("Warning: Negative errorbar detected %s%s %s."%(lineunit,fileline,description)) ; errcount+=1 elif (style[0] == "y"): # yerrorbar styles... first quoted error is y-error if (k == 2) and (value > data_item[1]): value = data_item[1] if (verb_errors): gp_warning("Warning: y lower limit > x value %s%s %s."%(lineunit,fileline,description)) ; errcount+=1 if (k == 3) and (value < data_item[1]): value = data_item[1] if (verb_errors): gp_warning("Warning: y upper limit < x value %s%s %s."%(lineunit,fileline,description)) ; errcount+=1 else: # This gets executed for all kinds of error ranges ; make sure that error ranges are sensible if (k == 2) and (value > data_item[0]): value = data_item[0] if (verb_errors): gp_warning("Warning: x lower limit > x value %s%s %s."%(lineunit,fileline,description)) ; errcount+=1 if (k == 3) and (value < data_item[0]): value = data_item[0] if (verb_errors): gp_warning("Warning: x upper limit < x value %s%s %s."%(lineunit,fileline,description)) ; errcount+=1 if (k == 4) and (value > data_item[1]): value = data_item[1] if (verb_errors): gp_warning("Warning: y lower limit > x value %s%s %s."%(lineunit,fileline,description)) ; errcount+=1 if (k == 5) and (value < data_item[1]): value = data_item[1] if (verb_errors): gp_warning("Warning: y upper limit < y value %s%s %s."%(lineunit,fileline,description)) ; errcount+=1 data_item.append(value) return errcount
def set(self, arg): if type(arg) in [list, tuple, Array, NArray] and len(arg) == 2: if arg[0] == arg[1]: self.set(arg[0]) else: self.issingleton = False loval = arg[0] hival = arg[1] assert loval is not NaN and hival is not NaN, \ ("Cannot specify NaN as interval endpoint") if not loval < hival: print "set() was passed loval = ", loval, \ " and hival = ", hival raise PyDSTool_ValueError, ( 'Interval endpoints must be ' 'given in order of increasing size') self._intervalstr = '['+str(loval)+',' \ +str(hival)+']' self._loval = loval self._hival = hival self.defined = True elif type(arg) in [int, float]: assert isfinite(arg), \ "Singleton interval domain value must be finite" self.issingleton = True self._intervalstr = str(arg) self._loval = arg self._hival = arg self.defined = True else: print "Error in argument: ", arg, "of type", type(arg) raise PyDSTool_TypeError, \ 'Interval spec must be a numeric or a length-2 sequence type'
def calculate_sp_pval(phen_vals): r = stats.shapiro(phen_vals) if sp.isfinite(r[0]): sp_pval = r[1] else: sp_pval = 0.0 return sp_pval
def EBTransitPhase(tset, kid_x): ebp = atpy.Table('%s/eb_pars.txt' %dir, type = 'ascii') lc = tset.tables[1] time = lc.TIME flux = lc.PDCSAP_FLUX nobs = len(time) lg = scipy.isfinite(time) pylab.figure(52) pylab.clf() pylab.plot(time[lg], flux[lg]) npl = 2 phase = scipy.zeros((npl, nobs)) inTr = scipy.zeros((npl, nobs), 'int') period = ebp.P[ebp.KID == kid_x] for ipl in scipy.arange(npl): if ipl == 0: t0 = ebp.Ep1[ebp.KID == kid_x] if ipl == 1: t0 = ebp.Ep2[ebp.KID == kid_x] if ipl == 0: dur = ebp.Dur1[ebp.KID == kid_x] if ipl == 1: dur = ebp.Dur2[ebp.KID == kid_x] dur /= period counter = 0 while (time[lg] - t0).min() < 0: t0 -= period counter += 1 if counter > 1000: break ph = ((time - t0) % period) / period ph[ph < -0.5] += 1 ph[ph > 0.5] -= 1 phase[ipl,:] = ph inTr[ipl,:] = (abs(ph) <= dur/1.5) return phase, inTr
def _update_evd(self, hyperparams, covar_id): keys = [] if 'covar_%s' % covar_id in hyperparams: keys.append('covar_%s' % covar_id) if 'X_%s' % covar_id in hyperparams: keys.append('X_%s' % covar_id) if not (self._is_cached(hyperparams, keys=keys)): K = [] S = [] U = [] i = 0 for covar in getattr(self, 'covar_%s' % covar_id): temp_kernel = covar.K(hyperparams['covar_%s' % covar_id][i]) temp_kernel = temp_kernel + SP.eye( temp_kernel.shape[0], temp_kernel.shape[1] ) * 0.001 # Rosolving possible numerical instability temp_kernel[SP.isnan(temp_kernel)] = SP.finfo(SP.float32).eps temp_kernel[~SP.isfinite(temp_kernel)] = 1E6 K.append(temp_kernel) S_temp, U_temp = LA.eigh(temp_kernel) S_temp[S_temp <= 0] = SP.finfo( SP.float32).eps # Rosolving possible numerical instability S.append(S_temp) U.append(U_temp) i += 1 self._covar_cache['K_%s' % covar_id] = K self._covar_cache['U_%s' % covar_id] = U self._covar_cache['S_%s' % covar_id] = S
def _box_cox_transform(self, verbose=False, method='standard'): """ Performs the Box-Cox transformation, over different ranges, picking the optimal one w. respect to normality. """ from scipy import stats a = sp.array(self.values) if method == 'standard': vals = (a - min(a)) + 0.1 * sp.var(a) else: vals = a sw_pvals = [] lambdas = sp.arange(-2.0, 2.1, 0.1) for l in lambdas: if l == 0: vs = sp.log(vals) else: vs = ((vals ** l) - 1) / l r = stats.shapiro(vs) if sp.isfinite(r[0]): pval = r[1] else: pval = 0.0 sw_pvals.append(pval) i = sp.argmax(sw_pvals) l = lambdas[i] if l == 0: vs = sp.log(vals) else: vs = ((vals ** l) - 1) / l self._perform_transform(vs,"box_cox") log.debug('optimal lambda was %0.1f' % l) return True
def get_covariances(self,hyperparams): """ INPUT: hyperparams: dictionary OUTPUT: dictionary with the fields K: kernel Kinv: inverse of the kernel L: chol(K) alpha: solve(K,y) W: D*Kinv * alpha*alpha^T """ if self._is_cached(hyperparams): return self._covar_cache K = self.covar.K(hyperparams['covar']) if self.likelihood is not None: Knoise = self.likelihood.K(hyperparams['lik'],self.n) K += Knoise K[~SP.isfinite(K)] = SP.finfo(float).eps K[SP.isnan(K)] = SP.finfo(float).eps L = LA.cholesky(K).T# lower triangular alpha = LA.cho_solve((L,True),self.Y) Kinv = LA.cho_solve((L,True),SP.eye(L.shape[0])) W = self.t*Kinv - SP.dot(alpha,alpha.T) self._covar_cache = {} self._covar_cache['K'] = K self._covar_cache['Kinv'] = Kinv self._covar_cache['L'] = L self._covar_cache['alpha'] = alpha self._covar_cache['W'] = W self._covar_cache['hyperparams'] = copy.deepcopy(hyperparams) return self._covar_cache
def median_filter_bord(im, size=3): """The function performs a local median filter on a flat image. Border's pixels are processed. Args: im: the image to process size: the size in pixels of the local square window. Default value is 3. Returns: out: the filtered image """ ## Get the size of the image [nl, nc, d] = im.shape ## Get the size of the moving window s = (size - 1) / 2 ## Initialization of the output out = sp.empty((nl, nc, d)) temp = sp.empty((nl + 2 * s, nc + 2 * s, d)) # A temporary file is created temp[0:s, :] = sp.NaN temp[:, 0:s] = sp.NaN temp[-s:, :] = sp.NaN temp[:, -s:] = sp.NaN temp[s : s + nl, s : nc + s] = im ## Apply the max filter for i in range(s, nl + s): # Shift the origin to remove border effect for j in range(s, nc + s): for k in range(d): window = temp[i - s : i + 1 + s, j - s : j + s + 1, k] out[i - s, j - s, k] = sp.median(window[sp.isfinite(window)]) return out.astype(im.dtype.name)
def most_normal_transformation(self,trans_types=SUPPORTED_TRANSFORMATIONS, perform_trans=True, verbose=False): """ Performs the transformation which results in most normal looking data, according to Shapiro-Wilk's test """ from scipy import stats shapiro_pvals = [] for trans_type in trans_types: if trans_type == 'most_normal': continue if trans_type != 'none': if not self.transform(trans_type=trans_type): continue phen_vals = self.values #print 'sp.inf in phen_vals:', sp.inf in phen_vals if sp.inf in phen_vals: pval = 0.0 else: r = stats.shapiro(phen_vals) if sp.isfinite(r[0]): pval = r[1] else: pval = 0.0 shapiro_pvals.append(pval) if trans_type != 'none': self.revert_to_raw_values() argmin_i = sp.argmax(shapiro_pvals) trans_type = trans_types[argmin_i] shapiro_pval = shapiro_pvals[argmin_i] if perform_trans: self.transform(trans_type=trans_type) log.info("The most normal-looking transformation was %s, with a Shapiro-Wilk's p-value of %.2E" % \ (trans_type, shapiro_pval)) return trans_type, shapiro_pval
def _fitData(idx, xdata, ydata, bounds=None, method=None, tol=None): """ internal program to interface with scipy.optimize.minimize (BFGS_L) algo """ #ydata =data[idx] # need to subtract baseline from data if not bounds == None: inp, bounds = _assembleInit(xdata, ydata, bounds=bounds) else: inp = _assembleInit(xdata, ydata) #inp[0] *= .9 temp = scipy.optimize.minimize(interface, inp, args=(xdata,ydata), method=method, bounds=bounds, jac=True) output = temp.x if scipy.isfinite(temp.fun): output = scipy.append(output,[temp.fun]) else: output = scipy.append(output,[-1.]) return output
def lnprob(theta, time, rv, err): lp = lnprior(theta) if not sp.isfinite(lp): return -sp.inf #print(lp) #print(lp + lnlike(theta, time, rv, err)) return lp + lnlike(theta, time, rv, err)
def getBeamFluxSpline(beam, plasma, t, lim1, lim2, points=1000): """ generates a spline off of the beampath. Assumes that the change in flux is MONOTONIC""" lim = beam.norm.s beam.norm.s = scipy.linspace(0, lim[-1], points) h = time.time() psi = plasma.eq.rz2rmid(beam.r()[0], beam.r()[2], t) #evaluates all psi's at once print(time.time() - h) outspline = len(t) * [0] inspline = len(t) * [0] for i in range(t.size): temp = lim1 mask = scipy.logical_and(scipy.isfinite(psi[i]), psi[i] < lim2 + .02) try: minpos = scipy.argmin(psi[i][mask]) test = psi[i][mask][minpos] except ValueError: test = lim2 + .03 #plt.plot(beam.x()[0][mask],psi[i][mask]) #plt.show() sizer = psi[i][mask].size if not test > lim2: #plt.plot(beam.x()[0][mask][0:minpos],psi[i][mask][0:minpos],beam.x()[0][mask][minpos:],psi[i][mask][minpos:]) #plt.show() #limout = scipy.insert(lim,(2,2),(beam.norm.s[mask][minpos],beam.norm.s[mask][minpos])) # add minimum flux s for bound testing if lim1 < test: temp = test try: temp1 = scipy.clip( scipy.digitize((lim1, lim2), psi[i][mask][minpos::-1]), 0, minpos) outspline[i] = beam.norm.s[mask][minpos::-1][temp1] except ValueError: tempmask = (psi[i][mask] < lim2)[0] outspline[i] = scipy.array( [beam.norm.s[mask][minpos], beam.norm.s[mask][tempmask]]) try: temp2 = scipy.clip( scipy.digitize((lim1, lim2), psi[i][mask][minpos:]), 0, sizer - minpos - 1) inspline[i] = beam.norm.s[mask][minpos:][temp2] except ValueError: inspline[i] = scipy.array( [beam.norm.s[mask][minpos], beam.norm.s[mask][-1]]) else: outspline[i] = scipy.array([[], []]) inspline[i] = scipy.array([[], []]) return (outspline, inspline)
def nanmedian(x): """Find the median over the given axis ignoring nans. fixme: should be fixed to work along an axis. """ x = _asarray1d(x).copy() y = compress(isfinite(x), x) return median(y)
def last_index(X): timestamps_infinite = sp.all(~sp.isfinite(X), axis=1) # Are there NaNs padded after the TS? if sp.alltrue(~timestamps_infinite): idx = X.shape[0] else: # Yes? then remove them idx = sp.nonzero(timestamps_infinite)[0][0] return idx
def nmodelfit(data, p, model, weight=0): if p.ndim == 2: if p.shape[0] == 2: p = p.T mask = p[:, 1].copy() t = scipy.zeros(mask.sum()) static = scipy.zeros(mask.size - t.size) j = 0 k = 0 for i in range(mask.size): if mask[i] > 0: t[j] = p[i, 0] j += 1 else: static[k] = p[i, 0] k += 1 p = t.copy() else: mask = scipy.ones(p.size) static = scipy.zeros(0) if data.ndim == 1: x = scipy.arange(0., data.size, 1.) z = data.copy() else: x = data[:, 0] z = data[:, 1] good = scipy.isfinite(z) x = x[good] z = z[good] pars, cov, info, mesg, ier = optimize.leastsq(domodel, p, args=(x, z, mask, static, model, weight), maxfev=10000, full_output=True) chi2 = info['fvec'] chi2 = chi2 * chi2 if weight == 0: chi2 /= abs(z) chi2 = chi2.sum() p = scipy.zeros(mask.size) j = 0 k = 0 for i in range(mask.size): if mask[i] > 0: p[i] = pars[j] j += 1 else: p[i] = static[k] k += 1 if i % 3 == 0: p[i] = scipy.fabs(p[i]) return p, chi2
def selectArray(flux,wave=None,wmin=None,wmax=None): """ Select the sub array of a flux, given a wavelength range. If no range is given, return the full array. @param flux: The wavelength array @type flux: array @keyword wave: The wavelength array. If default, no wavelength selection is done. @type wave: array @keyword wmin: The minimum wavelength. If not given, the minimum wavelength is the first entry in the wave array (default: None) @type wmin: float @keyword wmin: The maximum wavelength. If not given, the maximum wavelength is the last entry in the wave array (default: None) @type wmax: float @return: The flux in given wavelengths @rtype: array """ flux = array(flux) fsel = flux[isfinite(flux)] if wave <> None: wave = array(wave) wsel = wave[isfinite(flux)] if wmin is None: wmin = wsel[0] if wmax is None: wmax = wsel[-1] wmin, wmax = float(wmin), float(wmax) fsel = fsel[(wsel>=wmin)*(wsel<=wmax)] return fsel
def IWLS(self, niter=100): """Perform Iterative Weighted Least Squares""" test = sp.zeros(self.weights.shape) while not sp.allclose(test,self.weights) and niter > 0: test = self.weights.copy() self.weights = sp.diag((1.0/self._residuals**2).flatten()) self.weights[~sp.isfinite(self.weights)]=0 self.calc() niter -= 1
def goodGIW(time, shot, name="c_W"): """extract values which are only valid, otherwise place in zeros""" temp = GIWData(shot, data=name) interp = scipy.interpolate.interp1d(temp.time, temp.data, bounds_error=False) output = interp(time) output[ scipy.logical_not(scipy.isfinite(output))] = 0. #force all bad values to zero return output
def replaceNansByMeans(self): """Replace all not-a-number entries in the dataset by the means of the corresponding column.""" for d in self.data.itervalues(): means = scipy.nansum(d[:self.getLength()], axis=0) / self.getLength() for i in xrange(self.getLength()): for j in xrange(ds.dim): if not scipy.isfinite(d[i, j]): d[i, j] = means[j]
def mean_absolute_errors(results): """Calculates the interreplicate pairwise mean absolute errors among measurements generated by ``measure_all``""" valid_locations = ~sp.isnan(results.sum(0)) & sp.isfinite(results.sum(0)) flat_results = results[:, valid_locations].reshape((4, -1)) maes = [] for r1, r2 in it.combinations(flat_results, 2): maes.append(sp.mean(sp.absolute(r1 - r2))) return maes
def trace(self, ray, limiter=0): try: # norm vector is modfied following the convention set in geometry.Origin if not ray._origin is self: ray.redefine(self) invesselflag = self.inVessel(ray.r()[...,-1]) intersect = _beam.interceptCyl(scipy.atleast_2d(ray.x()[:,-1]), scipy.atleast_2d(ray.norm.unit), self.meri.s, self.norm.s) + ray.norm.s[-1] if scipy.isfinite(intersect): ray.norm.s = scipy.append(ray.norm.s,intersect) intersect = _beam.interceptCyl(scipy.atleast_2d(ray.x()[:,-1]), scipy.atleast_2d(ray.norm.unit), self.meri.s, self.norm.s) + ray.norm.s[-1] if not invesselflag and scipy.isfinite(intersect): ray.norm.s = scipy.append(ray.norm.s,intersect) # This is used when the actual plasma vessel structure is not as described in the eqdsk # example being the Limiter on Alcator C-Mod, which then keys to neglect an intersection, # and look for the next as the true wall intersection. for i in xrange(limiter): intersect = _beam.interceptCyl(scipy.atleast_2d(ray.x()[:,-1]), scipy.atleast_2d(ray.norm.unit), self.meri.s, self.norm.s) + ray.norm.s[-1] if scipy.isfinite(intersect): ray.norm.s[-1] = intersect except AttributeError: for i in ray: self.trace(i, limiter=limiter) except ValueError: # for subBeams, reference main beam self.trace(ray.main)
def calc_var(quarter_var, kid = None, time_in = None, flux = None, period = None): ''' calculate 5-95th percentile of flux (i.e. amplitude) whole LC and in each period block ''' # Of whole LC... sort_flc= sorted(flux) fi_ind_flc = int(len(sort_flc) * 0.05) nifi_ind_flc = int(len(sort_flc) * 0.95) amp_flc = sort_flc[nifi_ind_flc] - sort_flc[fi_ind_flc] # Of each block... if period > 0: num = int(scipy.floor(len(time_in) / period)) var_arr = scipy.zeros(num) + scipy.nan per_cent = scipy.zeros(num) + scipy.nan for i in scipy.arange(num-1): t_block = time_in[(time_in-time_in.min() >= i*period) * (time_in-time_in.min() < (i+1)*period)] f_block = flux[(time_in-time_in.min() >= i*period) * (time_in-time_in.min() < (i+1)*period)] if len(t_block) < 2: continue sort_f_block = sorted(f_block) fi_ind = int(len(sort_f_block) * 0.05) nifi_ind = int(len(sort_f_block) * 0.95) range_f = sort_f_block[nifi_ind] - sort_f_block[fi_ind] var_arr[i] = range_f per_cent[i] = scipy.mean(t_block) var_arr_real = var_arr[scipy.isfinite(var_arr) == True] per_cent = per_cent[scipy.isfinite(var_arr) == True] var_med = scipy.median(var_arr_real) savefilen = '%s/PDCQ%s_output/dat_files/%s_amps.txt' % (dir, quarter_var, kid) savefile = open(savefilen, 'w') savefile.write('per_centre amp\n') for z in scipy.arange(len(per_cent)): savefile.write('%f %f\n' %(per_cent[z], var_arr_real[z])) savefile.close() else: var_med = -9999 per_cent = scipy.array([-9999]) var_arr_real = scipy.array([-9999]) return amp_flc, var_med, per_cent, var_arr_real