def square_and_sum(a,s):
"""
Writes np.sum(a**2,axis=0) into s
"""
cmin, cmax = thread_partition_array(a)
map_noreturn(asqs, [(a,s,cmin[i],cmax[i]) for i in xrange(len(cmax))])
return a
def square_and_sum(a, s):
"""
Writes np.sum(a**2,axis=0) into s
"""
cmin, cmax = thread_partition_array(a)
map_noreturn(asqs, [(a, s, cmin[i], cmax[i]) for i in xrange(len(cmax))])
return a
def vivax(sp_sub):
cmin, cmax = thread_partition_array(sp_sub)
out = sp_sub_b.copy('F')
ttf = two_ten_factors[np.random.randint(len(two_ten_factors))]
pm.map_noreturn(vivax_postproc, [(out, sp_sub_0, sp_sub_v, p1, ttf, cmin[i], cmax[i]) for i in xrange(len(cmax))])
return out
def vivax(sp_sub_b, sp_sub_0, sp_sub_v, p1):
cmin, cmax = thread_partition_array(sp_sub_b)
out = sp_sub_b.copy('F')
# ttf = two_ten_factors[np.random.randint(len(two_ten_factors))]
ttf = 1
pm.map_noreturn(vivax_postproc, [(out, sp_sub_0, sp_sub_v, p1, ttf, cmin[i], cmax[i]) for i in xrange(len(cmax))])
# pm.map_noreturn(vivax_postproc, [(out, sp_sub_0, sp_sub_v, p1, 1, cmin[i], cmax[i]) for i in xrange(len(cmax))])
return out
def invlogit(x):
"""A shape-preserving, in-place, threaded inverse logit function."""
if np.prod(np.shape(x))<10000:
return pm.flib.invlogit(x)
if not x.flags['F_CONTIGUOUS']:
raise ValueError, 'x is not Fortran-contiguous'
cmin, cmax = thread_partition_array(x)
pm.map_noreturn(iinvlogit, [(x,cmin[i],cmax[i]) for i in xrange(len(cmax))])
return x
def mahalanobis_covariance(x, y, diff_degree, amp, val, vec, symm=None):
"""
Converts x and y to a matrix of covariances. x and y are assumed to have
columns (long,lat,t). Parameters are:
- t_gam_fun: A function returning a matrix of variogram values.
Inputs will be the 't' columns of x and y, as well as kwds.
- amp: The MS amplitude of realizations.
- scale: Scales distance.
- inc, ecc: Anisotropy parameters.
- n_threads: Maximum number of threads available to function.
- symm: Flag indicating whether matrix will be symmetric (optional).
- kwds: Passed to t_gam_fun.
Output value should never drop below -1. This happens when:
-1 > -sf*c+k
"""
# Allocate
nx = x.shape[0]
ny = y.shape[0]
ndim = x.shape[1]
C = np.asmatrix(np.empty((nx, ny), order="F"))
# Figure out symmetry and threading
if symm is None:
symm = x is y
n_threads = min(pm.get_threadpool_size(), nx * ny / 2000)
if n_threads > 1:
if not symm:
bounds = np.linspace(0, ny, n_threads + 1)
else:
bounds = np.array(np.sqrt(np.linspace(0, ny * ny, n_threads + 1)), dtype=int)
# Target function for threads
def targ(C, x, y, symm, diff_degree, amp, val, vec, cmin, cmax):
mahal(C, x, y, symm, diff_degree, gamma(diff_degree), amp, val, vec, cmin, cmax)
# Dispatch threads
if n_threads <= 1:
mahal(C, x, y, symm, diff_degree, gamma(diff_degree), amp, val, vec, cmin=0, cmax=C.shape[1])
else:
thread_args = [(C, x, y, symm, diff_degree, amp, val, vec, bounds[i], bounds[i + 1]) for i in xrange(n_threads)]
pm.map_noreturn(targ, thread_args)
if symm:
pm.gp.symmetrize(C)
# eigs = np.linalg.eigh(C)
# val,vec = eigs
# if np.any(val<-1.e-3):
# raise RuntimeError, 'Negative eigenvalues: %s'%val[np.where(val<0)]
return C
def __call__(self,x,y,amp=1.,scale=1.,symm=None,*args,**kwargs):
if amp<0. or scale<0.:
raise ValueError, 'The amp and scale parameters must be positive.'
if symm is None:
symm = (x is y)
# Figure out how to divide job up between threads.
nx = x.shape[0]
ny = y.shape[0]
n_threads = min(get_threadpool_size(), nx*ny / 10000)
if n_threads > 1:
if not symm:
bounds = np.linspace(0,ny,n_threads+1)
else:
bounds = np.array(np.sqrt(np.linspace(0,ny*ny,n_threads+1)),dtype=int)
# Split off the distance arguments
distance_arg_dict = {}
if hasattr(self.distance_fun, 'extra_parameters'):
for key in self.extra_distance_params.iterkeys():
if key in kwargs.keys():
distance_arg_dict[key] = kwargs.pop(key)
# Allocate the matrix
C = np.asmatrix(np.empty((nx,ny),dtype=float,order='F'))
def targ(C,x,y, cmin, cmax,symm, d_kwargs=distance_arg_dict, c_args=args, c_kwargs=kwargs):
# Compute distance for this bit
self.distance_fun(C, x, y, cmin=cmin, cmax=cmax, symm=symm, **d_kwargs)
imul(C, 1./scale, cmin=cmin, cmax=cmax, symm=symm)
# Compute covariance for this bit
self.cov_fun(C, cmin=cmin, cmax=cmax,symm=symm, *c_args, **c_kwargs)
imul(C, amp*amp, cmin=cmin, cmax=cmax, symm=symm)
# Possibly symmetrize this bit
# FIXME: Intermittent errors apparently originating in symmetrize!
# if symm:
# symmetrize(C, cmin=cmin, cmax=cmax)
if n_threads <= 1:
targ(C,x,y,0,-1,symm)
else:
thread_args = [(C,x,y,bounds[i],bounds[i+1],symm) for i in xrange(n_threads)]
map_noreturn(targ, thread_args)
if symm:
symmetrize(C)
return C
def __call__(self,x,y,amp=1.,scale=1.,symm=None,*args,**kwargs):
if amp<0. or scale<0.:
raise ValueError('The amp and scale parameters must be positive.')
if symm is None:
symm = (x is y)
# Figure out how to divide job up between threads.
nx = x.shape[0]
ny = y.shape[0]
n_threads = min(get_threadpool_size(), nx*ny // 10000)
if n_threads > 1:
if not symm:
bounds = np.linspace(0,ny,n_threads+1)
else:
bounds = np.array(np.sqrt(np.linspace(0,ny*ny,n_threads+1)),dtype=int)
# Split off the distance arguments
distance_arg_dict = {}
if hasattr(self.distance_fun, 'extra_parameters'):
for key in self.extra_distance_params:
if key in kwargs.keys():
distance_arg_dict[key] = kwargs.pop(key)
# Allocate the matrix
C = np.asmatrix(np.empty((nx,ny),dtype=float,order='F'))
def targ(C,x,y, cmin, cmax,symm, d_kwargs=distance_arg_dict, c_args=args, c_kwargs=kwargs):
# Compute distance for this bit
self.distance_fun(C, x, y, cmin=cmin, cmax=cmax, symm=symm, **d_kwargs)
imul(C, 1./scale, cmin=cmin, cmax=cmax, symm=symm)
# Compute covariance for this bit
if self.with_x:
self.cov_fun(C,x,y,cmin=cmin, cmax=cmax,symm=symm,*c_args,**c_kwargs)
else:
self.cov_fun(C, cmin=cmin, cmax=cmax,symm=symm, *c_args, **c_kwargs)
imul(C, amp*amp, cmin=cmin, cmax=cmax, symm=symm)
if n_threads <= 1:
targ(C,x,y,0,-1,symm)
else:
thread_args = [(C,x,y,bounds[i],bounds[i+1],symm) for i in xrange(n_threads)]
map_noreturn(targ, thread_args)
if symm:
symmetrize(C)
return C
def gen0(eps_p_fb, eps_p_f0):
cmin, cmax = thread_partition_array(eps_p_fb)
out = eps_p_fb.copy('F')
pm.map_noreturn(gen0_postproc, [(out, eps_p_f0, cmin[i], cmax[i]) for i in xrange(len(cmax))])
return out
def brownian(x, y, amp=1.0, scale=1.0, origin=None, h=0.5, symm=None):
"""
brownian(x,y,amp=1., scale=1.,h=.5,origin=None)
Fractional n-dimensional brownian motion. h=.5 corresponds to standard
Brownian motion.
A covariance function. Remember, broadcasting for covariance functions works
differently than for numpy universal functions. C(x,y) returns a matrix, and
C(x) returns a vector.
:Parameters:
- `amp`: The pointwise standard deviation of f.
- `scale`: The factor by which to scale the distance between points.
Large value implies long-range correlation.
- `h': The fractional parameter.
- `x and y` are arrays of points in Euclidean coordinates
formatted as follows:
[[x_{0,0} ... x_{0,ndim}],
[x_{1,0} ... x_{1,ndim}],
...
[x_{N,0} ... x_{N,ndim}]]
- `symm` indicates whether x and y are references to
the same array.
- `cmin' and `cmax' indicate which columns to compute.
These are used for multithreaded evaluation.
:Reference: http://en.wikipedia.org/wiki/Fractional_brownian_motion
"""
# Thanks to Anne Archibald for handythread.py, the model for the
# multithreaded call.
if h < 0 or h > 1:
raise ValueError("Parameter h must be between 0 and 1.")
if amp < 0.0 or scale < 0.0:
raise ValueError("The amp and scale parameters must be positive.")
if symm is None:
symm = x is y
# Figure out how to divide job up between threads.
nx = x.shape[0]
ny = y.shape[0]
n_threads = min(get_threadpool_size(), nx * ny / 10000)
if n_threads > 1:
if not symm:
bounds = np.linspace(0, ny, n_threads + 1)
else:
bounds = np.array(np.sqrt(np.linspace(0, ny * ny, n_threads + 1)), dtype=int)
# Allocate the matrix
C = np.asmatrix(np.empty((nx, ny), dtype=float, order="F"))
if origin is not None:
x = x - origin
y = y - origin
x = x / float(scale)
y = y / float(scale)
if n_threads <= 1:
brownian_targ(C, x, y, h, amp, 0, -1, symm)
else:
thread_args = [(C, x, y, h, amp, bounds[i], bounds[i + 1], symm) for i in xrange(n_threads)]
map_noreturn(brownian_targ, thread_args)
return C
def hdf5_to_samps(hf, x, burn, thin, total, fns, f_label, f_has_nugget, x_label, pred_cv_dict=None, nugget_label=None, postproc=None, finalize=None, diag_safe=False, **non_cov_columns):
"""
Parameters:
hf : PyTables file
The trace file from which predictions should be made.
x : array
The lon, lat locations at which predictions are desired.
burn : int
Burnin iterations to discard.
thin : int
Number of iterations between ones that get used in the predictions.
total : int
Total number of iterations to use in thinning.
fns : list of functions
Each function should take four arguments: sofar, next, cols and i.
Sofar may be None.
The functions will be applied according to the reduce pattern.
f_label : string
The name of the hdf5 node containing f
f_has_nugget : boolean
Whether f is nuggeted.
x_label : string
The name of the hdf5 node containing the input mesh associated with f
in the metadata.
pred_cv_dict : dictionary
{name : value on x}
nugget_label : string (optional)
The name of the hdf5 node giving the nugget variance
postproc : function (optional)
This function is applied to the realization before it is passed to
the fns.
finalize : function (optional)
This function is applied to the products before returning. It should
take a second argument which is the actual number of realizations
produced.
"""
# Add constant mean
if pred_cv_dict is None:
pred_cv_dict = {}
pred_cv_dict['m'] = np.ones(x.shape[0])
products = dict(zip(fns, [None]*len(fns)))
iter = np.arange(burn,all_chain_len(hf),thin)
if len(iter)==0:
raise ValueError, 'You asked for %i burnin iterations with thinnnig %i but the chains are only %i iterations long.'%(burn, thin, all_chain_len(hf))
n_per = total/len(iter)+1
actual_total = n_per * len(iter)
x_obs = getattr(hf.root.metadata,x_label)[:]
# Avoid memory errors
# max_chunksize = 1.e8 / x_obs.shape[0]
# n_chunks = int(x.shape[0]/max_chunksize+1)
# splits = np.array(np.linspace(0,x.shape[0],n_chunks+1)[1:-1],dtype='int')
# x_chunks = np.split(x,splits)
# i_chunks = np.split(np.arange(x.shape[0]), splits)
# If postproc is not None, close on non-covariate columns.
if postproc is not None:
if len(non_cov_columns) > 0:
postproc = postproc(**non_cov_columns)
time_count = -np.inf
time_start = time.time()
for k in xrange(len(iter)):
i = iter[k]
if time.time() - time_count > 10:
print ((k*100)/len(iter)), '% complete',
time_count = time.time()
if k > 0:
print 'expect results '+time.ctime((time_count-time_start)*len(iter)/float(k)+time_start)
else:
print
M_pred, S_pred = predictive_mean_and_std(hf, i, f_label, x_label, x, f_has_nugget, pred_cv_dict, nugget_label, diag_safe)
if M_pred is None:
actual_total -= n_per
continue
else:
cmin, cmax = thread_partition_array(M_pred)
# Postprocess if necessary: logit, etc.
norms = np.random.normal(size=n_per)
for j in xrange(n_per):
# surf = M_pred
# surf = M_pred + S_pred * norms[j]
surf = M_pred.copy('F')
pm.map_noreturn(iaaxpy, [(norms[j], S_pred, surf, cmin[l], cmax[l]) for l in xrange(len(cmax))])
if postproc is not None:
surf = postproc(surf)
# Reduction step
for f in fns:
products[f] = f(products[f], surf)
if finalize is not None:
return finalize(products, actual_total)
else:
return products
def fast_inplace_square(a):
cmin, cmax = thread_partition_array(a)
pm.map_noreturn(iasq, [(a,cmin[i],cmax[i]) for i in xrange(len(cmax))])
return a
def brownian(x,y,amp=1.,scale=1.,origin=None,h=.5,symm=None):
"""
brownian(x,y,amp=1., scale=1.,h=.5,origin=None)
Fractional n-dimensional brownian motion. h=.5 corresponds to standard
Brownian motion.
A covariance function. Remember, broadcasting for covariance functions works
differently than for numpy universal functions. C(x,y) returns a matrix, and
C(x) returns a vector.
:Parameters:
- `amp`: The pointwise standard deviation of f.
- `scale`: The factor by which to scale the distance between points.
Large value implies long-range correlation.
- `h': The fractional parameter.
- `x and y` are arrays of points in Euclidean coordinates
formatted as follows:
[[x_{0,0} ... x_{0,ndim}],
[x_{1,0} ... x_{1,ndim}],
...
[x_{N,0} ... x_{N,ndim}]]
- `symm` indicates whether x and y are references to
the same array.
- `cmin' and `cmax' indicate which columns to compute.
These are used for multithreaded evaluation.
:Reference: http://en.wikipedia.org/wiki/Fractional_brownian_motion
"""
# Thanks to Anne Archibald for handythread.py, the model for the
# multithreaded call.
if h<0 or h>1:
raise ValueError, 'Parameter h must be between 0 and 1.'
if amp<0. or scale<0.:
raise ValueError, 'The amp and scale parameters must be positive.'
if symm is None:
symm = (x is y)
# Figure out how to divide job up between threads.
nx = x.shape[0]
ny = y.shape[0]
n_threads = min(get_threadpool_size(), nx*ny / 10000)
if n_threads > 1:
if not symm:
bounds = np.linspace(0,ny,n_threads+1)
else:
bounds = np.array(np.sqrt(np.linspace(0,ny*ny,n_threads+1)),dtype=int)
# Allocate the matrix
C = np.asmatrix(np.empty((nx,ny),dtype=float,order='F'))
if origin is not None:
x = x-origin
y = y-origin
x = x / float(scale)
y = y / float(scale)
if n_threads <= 1:
brownian_targ(C,x,y,h,amp,0,-1,symm)
else:
thread_args=[(C,x,y,h,amp,bounds[i],bounds[i+1],symm) for i in xrange(n_threads)]
map_noreturn(brownian_targ, thread_args)
return C
def nonstationary_spatiotemporal(x,y,amp,scale,diff_degree,t_gam_fun=default_t_gam_fun,h=default_h,symm=None,geometry='aniso_geo_rad',**kwds):
"""
Spatiotemporal covariance function. Converts x and y
to a matrix of covariances. x and y are assumed to have
columns (long,lat,t). Parameters are:
- t_gam_fun: A function returning a matrix of variogram values.
Inputs will be the 't' columns of x and y, as well as kwds.
- amp: The MS amplitude of realizations.
- scale: Scales distance.
- diff_degree: A function that returns local degree of differentiability at x.
- h: A function that returns local relative amplitude at x.
- inc, ecc: Anisotropy parameters. Needed if geometry=='aniso_geo_rad'.
- n_threads: Maximum number of threads available to function.
- symm: Flag indicating whether matrix will be symmetric (optional).
- geometry: Must be 'aniso_geo_rad' or 'euclidean'.
- kwds: Passed to t_gam_fun.
References:
Stein, 2005. "Space-Time Covariance Functions". Journal of the American Statistical
Association 100(469).
Pintore and Holmes, 2010, "Spatially adaptive non-stationary covariance functions
via spatially adaptive spectra". Journal of the American Statistical Association.
Forthcoming.
"""
# Allocate
nx = x.shape[0]
ny = y.shape[0]
if kwds.has_key('n_threads'):
kwds.pop('n_threads')
if geometry=='aniso_geo_rad':
inc = kwds.pop('inc')
ecc = kwds.pop('ecc')
else:
inc = None
ecc = None
if geometry not in ['aniso_geo_rad','euclidean']:
raise ValueError, 'Geometry %s unknown, must be aniso_geo_rad or euclidean.'%geometry
D = np.asmatrix(np.empty((nx,ny),order='F'))
GT = np.asmatrix(np.empty((nx,ny),order='F'))
# Figure out symmetry and threading
if symm is None:
symm = (x is y)
n_threads = min(get_threadpool_size(), nx*ny / 10000)
if n_threads > 1:
if not symm:
bounds = np.linspace(0,ny,n_threads+1)
else:
bounds = np.array(np.sqrt(np.linspace(0,ny*ny,n_threads+1)),dtype=int)
# Target function for threads
def targ(D,GT,x,y,cmin,cmax,symm,inc=inc,ecc=ecc,amp=amp,scale=scale,diff_degree=diff_degree,h=h,geometry=geometry,kwds=kwds):
# Spatial distance
if geometry=='aniso_geo_rad':
aniso_geo_rad(D, x[:,:-1], y[:,:-1], inc, ecc,cmin=cmin,cmax=cmax,symm=symm)
else:
euclidean(D, x[:,:-1], y[:,:-1], cmin=cmin,cmax=cmax,symm=symm)
imul(D,1./scale,cmin=cmin,cmax=cmax,symm=symm)
# Temporal variogram
ddx, ddy = diff_degree(x), diff_degree(y)
origin_val = t_gam_fun(GT, x[:,-1], y[:,-1], ddx, ddy, cmin=cmin,cmax=cmax,symm=False,**kwds)
if np.any(GT<0):
raise pm.ZeroProbability, 'GT < 0.'
# GT = np.add.outer(ddx*.5,ddy*.5)
# Local properties
hx, hy = h(x), h(y)
# Covariance
nsst(D,GT,origin_val,hx,hy,cmin=cmin,cmax=cmax,symm=symm)
imul(D,amp*amp,cmin=cmin,cmax=cmax,symm=symm)
# Serial version
if n_threads <= 1:
targ(D,GT,x,y,0,-1,symm)
# Parallel version
else:
thread_args = [(D,GT,x,y,bounds[i],bounds[i+1],symm) for i in xrange(n_threads)]
map_noreturn(targ, thread_args)
if symm:
symmetrize(D)
return D
def my_st(x,y,amp,scale,inc,ecc,symm=None,**kwds):
"""
Spatiotemporal covariance function. Converts x and y
to a matrix of covariances. x and y are assumed to have
columns (long,lat,t). Parameters are:
- t_gam_fun: A function returning a matrix of variogram values.
Inputs will be the 't' columns of x and y, as well as kwds.
- amp: The MS amplitude of realizations.
- scale: Scales distance.
- inc, ecc: Anisotropy parameters.
- n_threads: Maximum number of threads available to function.
- symm: Flag indicating whether matrix will be symmetric (optional).
- kwds: Passed to t_gam_fun.
Output value should never drop below -1. This happens when:
-1 > -sf*c+k
"""
# Allocate
nx = x.shape[0]
ny = y.shape[0]
k=kwds['tlc']/kwds['sd']
c=1./kwds['sd']-k
sf=kwds['sf']
tlc=kwds['tlc']
sd=kwds['sd']
if kwds.has_key('n_threads'):
kwds.pop('n_threads')
# If parameter values are illegal, just return zeros.
# This case will be caught by the Potential.
if -sd >= 1./(-sf*(1-tlc)+tlc):
return np.zeros((nx,ny))
D = np.asmatrix(np.empty((nx,ny),order='F'))
GT = np.asmatrix(np.empty((nx,ny),order='F'))
# Figure out symmetry and threading
if symm is None:
symm = (x is y)
n_threads = min(get_threadpool_size(), nx*ny / 10000)
if n_threads > 1:
if not symm:
bounds = np.linspace(0,ny,n_threads+1)
else:
bounds = np.array(np.sqrt(np.linspace(0,ny*ny,n_threads+1)),dtype=int)
# Target function for threads
def targ(D,GT,x,y,cmin,cmax,symm,inc=inc,ecc=ecc,amp=amp,scale=scale,kwds=kwds):
# Spatial distance
aniso_geo_rad(D, x[:,:-1], y[:,:-1], inc, ecc,cmin=cmin,cmax=cmax,symm=symm)
imul(D,1./scale,cmin=cmin,cmax=cmax,symm=symm)
# Temporal variogram
origin_val = t_gam_fun(GT, x[:,-1], y[:,-1],cmin=cmin,cmax=cmax,symm=symm,**kwds)
# Covariance
stein_spatiotemporal(D,GT,origin_val,cmin=cmin,cmax=cmax,symm=symm)
imul(D,amp*amp,cmin=cmin,cmax=cmax,symm=symm)
# if symm:
# symmetrize(D, cmin=cmin, cmax=cmax)
# Serial version
if n_threads <= 1:
targ(D,GT,x,y,0,-1,symm)
# Parallel version
else:
thread_args = [(D,GT,x,y,bounds[i],bounds[i+1],symm) for i in xrange(n_threads)]
map_noreturn(targ, thread_args)
if symm:
symmetrize(D)
return D
def phe0(sp_sub_b, sp_sub_0, sp_sub_v, p1):
cmin, cmax = thread_partition_array(sp_sub_b)
out = sp_sub_b.copy('F')
pm.map_noreturn(phe0_postproc, [(out, sp_sub_0, p1, cmin[i], cmax[i]) for i in xrange(len(cmax))])
return out
def square_and_sum(a,s):
cmin, cmax = thread_partition_array(a)
pm.map_noreturn(asqs, [(a,s,cmin[i],cmax[i]) for i in xrange(len(cmax))])
return a
def crossmul_and_sum(c,x,d,y):
cmin, cmax = thread_partition_array(y)
pm.map_noreturn(icsum, [(c,x,d,y,cmin[i],cmax[i]) for i in xrange(len(cmax))])
return c
