def square_and_sum(a,s): """ Writes np.sum(a**2,axis=0) into s """ cmin, cmax = thread_partition_array(a) map_noreturn(asqs, [(a,s,cmin[i],cmax[i]) for i in xrange(len(cmax))]) return a
def square_and_sum(a, s): """ Writes np.sum(a**2,axis=0) into s """ cmin, cmax = thread_partition_array(a) map_noreturn(asqs, [(a, s, cmin[i], cmax[i]) for i in xrange(len(cmax))]) return a
def vivax(sp_sub): cmin, cmax = thread_partition_array(sp_sub) out = sp_sub_b.copy('F') ttf = two_ten_factors[np.random.randint(len(two_ten_factors))] pm.map_noreturn(vivax_postproc, [(out, sp_sub_0, sp_sub_v, p1, ttf, cmin[i], cmax[i]) for i in xrange(len(cmax))]) return out
def vivax(sp_sub_b, sp_sub_0, sp_sub_v, p1): cmin, cmax = thread_partition_array(sp_sub_b) out = sp_sub_b.copy('F') # ttf = two_ten_factors[np.random.randint(len(two_ten_factors))] ttf = 1 pm.map_noreturn(vivax_postproc, [(out, sp_sub_0, sp_sub_v, p1, ttf, cmin[i], cmax[i]) for i in xrange(len(cmax))]) # pm.map_noreturn(vivax_postproc, [(out, sp_sub_0, sp_sub_v, p1, 1, cmin[i], cmax[i]) for i in xrange(len(cmax))]) return out
def invlogit(x): """A shape-preserving, in-place, threaded inverse logit function.""" if np.prod(np.shape(x))<10000: return pm.flib.invlogit(x) if not x.flags['F_CONTIGUOUS']: raise ValueError, 'x is not Fortran-contiguous' cmin, cmax = thread_partition_array(x) pm.map_noreturn(iinvlogit, [(x,cmin[i],cmax[i]) for i in xrange(len(cmax))]) return x
def mahalanobis_covariance(x, y, diff_degree, amp, val, vec, symm=None): """ Converts x and y to a matrix of covariances. x and y are assumed to have columns (long,lat,t). Parameters are: - t_gam_fun: A function returning a matrix of variogram values. Inputs will be the 't' columns of x and y, as well as kwds. - amp: The MS amplitude of realizations. - scale: Scales distance. - inc, ecc: Anisotropy parameters. - n_threads: Maximum number of threads available to function. - symm: Flag indicating whether matrix will be symmetric (optional). - kwds: Passed to t_gam_fun. Output value should never drop below -1. This happens when: -1 > -sf*c+k """ # Allocate nx = x.shape[0] ny = y.shape[0] ndim = x.shape[1] C = np.asmatrix(np.empty((nx, ny), order="F")) # Figure out symmetry and threading if symm is None: symm = x is y n_threads = min(pm.get_threadpool_size(), nx * ny / 2000) if n_threads > 1: if not symm: bounds = np.linspace(0, ny, n_threads + 1) else: bounds = np.array(np.sqrt(np.linspace(0, ny * ny, n_threads + 1)), dtype=int) # Target function for threads def targ(C, x, y, symm, diff_degree, amp, val, vec, cmin, cmax): mahal(C, x, y, symm, diff_degree, gamma(diff_degree), amp, val, vec, cmin, cmax) # Dispatch threads if n_threads <= 1: mahal(C, x, y, symm, diff_degree, gamma(diff_degree), amp, val, vec, cmin=0, cmax=C.shape[1]) else: thread_args = [(C, x, y, symm, diff_degree, amp, val, vec, bounds[i], bounds[i + 1]) for i in xrange(n_threads)] pm.map_noreturn(targ, thread_args) if symm: pm.gp.symmetrize(C) # eigs = np.linalg.eigh(C) # val,vec = eigs # if np.any(val<-1.e-3): # raise RuntimeError, 'Negative eigenvalues: %s'%val[np.where(val<0)] return C
def __call__(self,x,y,amp=1.,scale=1.,symm=None,*args,**kwargs): if amp<0. or scale<0.: raise ValueError, 'The amp and scale parameters must be positive.' if symm is None: symm = (x is y) # Figure out how to divide job up between threads. nx = x.shape[0] ny = y.shape[0] n_threads = min(get_threadpool_size(), nx*ny / 10000) if n_threads > 1: if not symm: bounds = np.linspace(0,ny,n_threads+1) else: bounds = np.array(np.sqrt(np.linspace(0,ny*ny,n_threads+1)),dtype=int) # Split off the distance arguments distance_arg_dict = {} if hasattr(self.distance_fun, 'extra_parameters'): for key in self.extra_distance_params.iterkeys(): if key in kwargs.keys(): distance_arg_dict[key] = kwargs.pop(key) # Allocate the matrix C = np.asmatrix(np.empty((nx,ny),dtype=float,order='F')) def targ(C,x,y, cmin, cmax,symm, d_kwargs=distance_arg_dict, c_args=args, c_kwargs=kwargs): # Compute distance for this bit self.distance_fun(C, x, y, cmin=cmin, cmax=cmax, symm=symm, **d_kwargs) imul(C, 1./scale, cmin=cmin, cmax=cmax, symm=symm) # Compute covariance for this bit self.cov_fun(C, cmin=cmin, cmax=cmax,symm=symm, *c_args, **c_kwargs) imul(C, amp*amp, cmin=cmin, cmax=cmax, symm=symm) # Possibly symmetrize this bit # FIXME: Intermittent errors apparently originating in symmetrize! # if symm: # symmetrize(C, cmin=cmin, cmax=cmax) if n_threads <= 1: targ(C,x,y,0,-1,symm) else: thread_args = [(C,x,y,bounds[i],bounds[i+1],symm) for i in xrange(n_threads)] map_noreturn(targ, thread_args) if symm: symmetrize(C) return C
def __call__(self,x,y,amp=1.,scale=1.,symm=None,*args,**kwargs): if amp<0. or scale<0.: raise ValueError('The amp and scale parameters must be positive.') if symm is None: symm = (x is y) # Figure out how to divide job up between threads. nx = x.shape[0] ny = y.shape[0] n_threads = min(get_threadpool_size(), nx*ny // 10000) if n_threads > 1: if not symm: bounds = np.linspace(0,ny,n_threads+1) else: bounds = np.array(np.sqrt(np.linspace(0,ny*ny,n_threads+1)),dtype=int) # Split off the distance arguments distance_arg_dict = {} if hasattr(self.distance_fun, 'extra_parameters'): for key in self.extra_distance_params: if key in kwargs.keys(): distance_arg_dict[key] = kwargs.pop(key) # Allocate the matrix C = np.asmatrix(np.empty((nx,ny),dtype=float,order='F')) def targ(C,x,y, cmin, cmax,symm, d_kwargs=distance_arg_dict, c_args=args, c_kwargs=kwargs): # Compute distance for this bit self.distance_fun(C, x, y, cmin=cmin, cmax=cmax, symm=symm, **d_kwargs) imul(C, 1./scale, cmin=cmin, cmax=cmax, symm=symm) # Compute covariance for this bit if self.with_x: self.cov_fun(C,x,y,cmin=cmin, cmax=cmax,symm=symm,*c_args,**c_kwargs) else: self.cov_fun(C, cmin=cmin, cmax=cmax,symm=symm, *c_args, **c_kwargs) imul(C, amp*amp, cmin=cmin, cmax=cmax, symm=symm) if n_threads <= 1: targ(C,x,y,0,-1,symm) else: thread_args = [(C,x,y,bounds[i],bounds[i+1],symm) for i in xrange(n_threads)] map_noreturn(targ, thread_args) if symm: symmetrize(C) return C
def gen0(eps_p_fb, eps_p_f0): cmin, cmax = thread_partition_array(eps_p_fb) out = eps_p_fb.copy('F') pm.map_noreturn(gen0_postproc, [(out, eps_p_f0, cmin[i], cmax[i]) for i in xrange(len(cmax))]) return out
def brownian(x, y, amp=1.0, scale=1.0, origin=None, h=0.5, symm=None): """ brownian(x,y,amp=1., scale=1.,h=.5,origin=None) Fractional n-dimensional brownian motion. h=.5 corresponds to standard Brownian motion. A covariance function. Remember, broadcasting for covariance functions works differently than for numpy universal functions. C(x,y) returns a matrix, and C(x) returns a vector. :Parameters: - `amp`: The pointwise standard deviation of f. - `scale`: The factor by which to scale the distance between points. Large value implies long-range correlation. - `h': The fractional parameter. - `x and y` are arrays of points in Euclidean coordinates formatted as follows: [[x_{0,0} ... x_{0,ndim}], [x_{1,0} ... x_{1,ndim}], ... [x_{N,0} ... x_{N,ndim}]] - `symm` indicates whether x and y are references to the same array. - `cmin' and `cmax' indicate which columns to compute. These are used for multithreaded evaluation. :Reference: http://en.wikipedia.org/wiki/Fractional_brownian_motion """ # Thanks to Anne Archibald for handythread.py, the model for the # multithreaded call. if h < 0 or h > 1: raise ValueError("Parameter h must be between 0 and 1.") if amp < 0.0 or scale < 0.0: raise ValueError("The amp and scale parameters must be positive.") if symm is None: symm = x is y # Figure out how to divide job up between threads. nx = x.shape[0] ny = y.shape[0] n_threads = min(get_threadpool_size(), nx * ny / 10000) if n_threads > 1: if not symm: bounds = np.linspace(0, ny, n_threads + 1) else: bounds = np.array(np.sqrt(np.linspace(0, ny * ny, n_threads + 1)), dtype=int) # Allocate the matrix C = np.asmatrix(np.empty((nx, ny), dtype=float, order="F")) if origin is not None: x = x - origin y = y - origin x = x / float(scale) y = y / float(scale) if n_threads <= 1: brownian_targ(C, x, y, h, amp, 0, -1, symm) else: thread_args = [(C, x, y, h, amp, bounds[i], bounds[i + 1], symm) for i in xrange(n_threads)] map_noreturn(brownian_targ, thread_args) return C
def hdf5_to_samps(hf, x, burn, thin, total, fns, f_label, f_has_nugget, x_label, pred_cv_dict=None, nugget_label=None, postproc=None, finalize=None, diag_safe=False, **non_cov_columns): """ Parameters: hf : PyTables file The trace file from which predictions should be made. x : array The lon, lat locations at which predictions are desired. burn : int Burnin iterations to discard. thin : int Number of iterations between ones that get used in the predictions. total : int Total number of iterations to use in thinning. fns : list of functions Each function should take four arguments: sofar, next, cols and i. Sofar may be None. The functions will be applied according to the reduce pattern. f_label : string The name of the hdf5 node containing f f_has_nugget : boolean Whether f is nuggeted. x_label : string The name of the hdf5 node containing the input mesh associated with f in the metadata. pred_cv_dict : dictionary {name : value on x} nugget_label : string (optional) The name of the hdf5 node giving the nugget variance postproc : function (optional) This function is applied to the realization before it is passed to the fns. finalize : function (optional) This function is applied to the products before returning. It should take a second argument which is the actual number of realizations produced. """ # Add constant mean if pred_cv_dict is None: pred_cv_dict = {} pred_cv_dict['m'] = np.ones(x.shape[0]) products = dict(zip(fns, [None]*len(fns))) iter = np.arange(burn,all_chain_len(hf),thin) if len(iter)==0: raise ValueError, 'You asked for %i burnin iterations with thinnnig %i but the chains are only %i iterations long.'%(burn, thin, all_chain_len(hf)) n_per = total/len(iter)+1 actual_total = n_per * len(iter) x_obs = getattr(hf.root.metadata,x_label)[:] # Avoid memory errors # max_chunksize = 1.e8 / x_obs.shape[0] # n_chunks = int(x.shape[0]/max_chunksize+1) # splits = np.array(np.linspace(0,x.shape[0],n_chunks+1)[1:-1],dtype='int') # x_chunks = np.split(x,splits) # i_chunks = np.split(np.arange(x.shape[0]), splits) # If postproc is not None, close on non-covariate columns. if postproc is not None: if len(non_cov_columns) > 0: postproc = postproc(**non_cov_columns) time_count = -np.inf time_start = time.time() for k in xrange(len(iter)): i = iter[k] if time.time() - time_count > 10: print ((k*100)/len(iter)), '% complete', time_count = time.time() if k > 0: print 'expect results '+time.ctime((time_count-time_start)*len(iter)/float(k)+time_start) else: print M_pred, S_pred = predictive_mean_and_std(hf, i, f_label, x_label, x, f_has_nugget, pred_cv_dict, nugget_label, diag_safe) if M_pred is None: actual_total -= n_per continue else: cmin, cmax = thread_partition_array(M_pred) # Postprocess if necessary: logit, etc. norms = np.random.normal(size=n_per) for j in xrange(n_per): # surf = M_pred # surf = M_pred + S_pred * norms[j] surf = M_pred.copy('F') pm.map_noreturn(iaaxpy, [(norms[j], S_pred, surf, cmin[l], cmax[l]) for l in xrange(len(cmax))]) if postproc is not None: surf = postproc(surf) # Reduction step for f in fns: products[f] = f(products[f], surf) if finalize is not None: return finalize(products, actual_total) else: return products
def fast_inplace_square(a): cmin, cmax = thread_partition_array(a) pm.map_noreturn(iasq, [(a,cmin[i],cmax[i]) for i in xrange(len(cmax))]) return a
def brownian(x,y,amp=1.,scale=1.,origin=None,h=.5,symm=None): """ brownian(x,y,amp=1., scale=1.,h=.5,origin=None) Fractional n-dimensional brownian motion. h=.5 corresponds to standard Brownian motion. A covariance function. Remember, broadcasting for covariance functions works differently than for numpy universal functions. C(x,y) returns a matrix, and C(x) returns a vector. :Parameters: - `amp`: The pointwise standard deviation of f. - `scale`: The factor by which to scale the distance between points. Large value implies long-range correlation. - `h': The fractional parameter. - `x and y` are arrays of points in Euclidean coordinates formatted as follows: [[x_{0,0} ... x_{0,ndim}], [x_{1,0} ... x_{1,ndim}], ... [x_{N,0} ... x_{N,ndim}]] - `symm` indicates whether x and y are references to the same array. - `cmin' and `cmax' indicate which columns to compute. These are used for multithreaded evaluation. :Reference: http://en.wikipedia.org/wiki/Fractional_brownian_motion """ # Thanks to Anne Archibald for handythread.py, the model for the # multithreaded call. if h<0 or h>1: raise ValueError, 'Parameter h must be between 0 and 1.' if amp<0. or scale<0.: raise ValueError, 'The amp and scale parameters must be positive.' if symm is None: symm = (x is y) # Figure out how to divide job up between threads. nx = x.shape[0] ny = y.shape[0] n_threads = min(get_threadpool_size(), nx*ny / 10000) if n_threads > 1: if not symm: bounds = np.linspace(0,ny,n_threads+1) else: bounds = np.array(np.sqrt(np.linspace(0,ny*ny,n_threads+1)),dtype=int) # Allocate the matrix C = np.asmatrix(np.empty((nx,ny),dtype=float,order='F')) if origin is not None: x = x-origin y = y-origin x = x / float(scale) y = y / float(scale) if n_threads <= 1: brownian_targ(C,x,y,h,amp,0,-1,symm) else: thread_args=[(C,x,y,h,amp,bounds[i],bounds[i+1],symm) for i in xrange(n_threads)] map_noreturn(brownian_targ, thread_args) return C
def nonstationary_spatiotemporal(x,y,amp,scale,diff_degree,t_gam_fun=default_t_gam_fun,h=default_h,symm=None,geometry='aniso_geo_rad',**kwds): """ Spatiotemporal covariance function. Converts x and y to a matrix of covariances. x and y are assumed to have columns (long,lat,t). Parameters are: - t_gam_fun: A function returning a matrix of variogram values. Inputs will be the 't' columns of x and y, as well as kwds. - amp: The MS amplitude of realizations. - scale: Scales distance. - diff_degree: A function that returns local degree of differentiability at x. - h: A function that returns local relative amplitude at x. - inc, ecc: Anisotropy parameters. Needed if geometry=='aniso_geo_rad'. - n_threads: Maximum number of threads available to function. - symm: Flag indicating whether matrix will be symmetric (optional). - geometry: Must be 'aniso_geo_rad' or 'euclidean'. - kwds: Passed to t_gam_fun. References: Stein, 2005. "Space-Time Covariance Functions". Journal of the American Statistical Association 100(469). Pintore and Holmes, 2010, "Spatially adaptive non-stationary covariance functions via spatially adaptive spectra". Journal of the American Statistical Association. Forthcoming. """ # Allocate nx = x.shape[0] ny = y.shape[0] if kwds.has_key('n_threads'): kwds.pop('n_threads') if geometry=='aniso_geo_rad': inc = kwds.pop('inc') ecc = kwds.pop('ecc') else: inc = None ecc = None if geometry not in ['aniso_geo_rad','euclidean']: raise ValueError, 'Geometry %s unknown, must be aniso_geo_rad or euclidean.'%geometry D = np.asmatrix(np.empty((nx,ny),order='F')) GT = np.asmatrix(np.empty((nx,ny),order='F')) # Figure out symmetry and threading if symm is None: symm = (x is y) n_threads = min(get_threadpool_size(), nx*ny / 10000) if n_threads > 1: if not symm: bounds = np.linspace(0,ny,n_threads+1) else: bounds = np.array(np.sqrt(np.linspace(0,ny*ny,n_threads+1)),dtype=int) # Target function for threads def targ(D,GT,x,y,cmin,cmax,symm,inc=inc,ecc=ecc,amp=amp,scale=scale,diff_degree=diff_degree,h=h,geometry=geometry,kwds=kwds): # Spatial distance if geometry=='aniso_geo_rad': aniso_geo_rad(D, x[:,:-1], y[:,:-1], inc, ecc,cmin=cmin,cmax=cmax,symm=symm) else: euclidean(D, x[:,:-1], y[:,:-1], cmin=cmin,cmax=cmax,symm=symm) imul(D,1./scale,cmin=cmin,cmax=cmax,symm=symm) # Temporal variogram ddx, ddy = diff_degree(x), diff_degree(y) origin_val = t_gam_fun(GT, x[:,-1], y[:,-1], ddx, ddy, cmin=cmin,cmax=cmax,symm=False,**kwds) if np.any(GT<0): raise pm.ZeroProbability, 'GT < 0.' # GT = np.add.outer(ddx*.5,ddy*.5) # Local properties hx, hy = h(x), h(y) # Covariance nsst(D,GT,origin_val,hx,hy,cmin=cmin,cmax=cmax,symm=symm) imul(D,amp*amp,cmin=cmin,cmax=cmax,symm=symm) # Serial version if n_threads <= 1: targ(D,GT,x,y,0,-1,symm) # Parallel version else: thread_args = [(D,GT,x,y,bounds[i],bounds[i+1],symm) for i in xrange(n_threads)] map_noreturn(targ, thread_args) if symm: symmetrize(D) return D
def my_st(x,y,amp,scale,inc,ecc,symm=None,**kwds): """ Spatiotemporal covariance function. Converts x and y to a matrix of covariances. x and y are assumed to have columns (long,lat,t). Parameters are: - t_gam_fun: A function returning a matrix of variogram values. Inputs will be the 't' columns of x and y, as well as kwds. - amp: The MS amplitude of realizations. - scale: Scales distance. - inc, ecc: Anisotropy parameters. - n_threads: Maximum number of threads available to function. - symm: Flag indicating whether matrix will be symmetric (optional). - kwds: Passed to t_gam_fun. Output value should never drop below -1. This happens when: -1 > -sf*c+k """ # Allocate nx = x.shape[0] ny = y.shape[0] k=kwds['tlc']/kwds['sd'] c=1./kwds['sd']-k sf=kwds['sf'] tlc=kwds['tlc'] sd=kwds['sd'] if kwds.has_key('n_threads'): kwds.pop('n_threads') # If parameter values are illegal, just return zeros. # This case will be caught by the Potential. if -sd >= 1./(-sf*(1-tlc)+tlc): return np.zeros((nx,ny)) D = np.asmatrix(np.empty((nx,ny),order='F')) GT = np.asmatrix(np.empty((nx,ny),order='F')) # Figure out symmetry and threading if symm is None: symm = (x is y) n_threads = min(get_threadpool_size(), nx*ny / 10000) if n_threads > 1: if not symm: bounds = np.linspace(0,ny,n_threads+1) else: bounds = np.array(np.sqrt(np.linspace(0,ny*ny,n_threads+1)),dtype=int) # Target function for threads def targ(D,GT,x,y,cmin,cmax,symm,inc=inc,ecc=ecc,amp=amp,scale=scale,kwds=kwds): # Spatial distance aniso_geo_rad(D, x[:,:-1], y[:,:-1], inc, ecc,cmin=cmin,cmax=cmax,symm=symm) imul(D,1./scale,cmin=cmin,cmax=cmax,symm=symm) # Temporal variogram origin_val = t_gam_fun(GT, x[:,-1], y[:,-1],cmin=cmin,cmax=cmax,symm=symm,**kwds) # Covariance stein_spatiotemporal(D,GT,origin_val,cmin=cmin,cmax=cmax,symm=symm) imul(D,amp*amp,cmin=cmin,cmax=cmax,symm=symm) # if symm: # symmetrize(D, cmin=cmin, cmax=cmax) # Serial version if n_threads <= 1: targ(D,GT,x,y,0,-1,symm) # Parallel version else: thread_args = [(D,GT,x,y,bounds[i],bounds[i+1],symm) for i in xrange(n_threads)] map_noreturn(targ, thread_args) if symm: symmetrize(D) return D
def phe0(sp_sub_b, sp_sub_0, sp_sub_v, p1): cmin, cmax = thread_partition_array(sp_sub_b) out = sp_sub_b.copy('F') pm.map_noreturn(phe0_postproc, [(out, sp_sub_0, p1, cmin[i], cmax[i]) for i in xrange(len(cmax))]) return out
def square_and_sum(a,s): cmin, cmax = thread_partition_array(a) pm.map_noreturn(asqs, [(a,s,cmin[i],cmax[i]) for i in xrange(len(cmax))]) return a
def crossmul_and_sum(c,x,d,y): cmin, cmax = thread_partition_array(y) pm.map_noreturn(icsum, [(c,x,d,y,cmin[i],cmax[i]) for i in xrange(len(cmax))]) return c