def subset_and_writeout(hf_in, fname, thin, maskval, binfn=lambda x:x):
print 'Subsetting for %s'%fname
res=5
hf_out = tb.openFile(os.path.join('5k-covariates',fname.replace('-','_').replace('.','_')+'.hdf5'),'w')
hf_out.createArray('/','lon',lon[lon_min_i:lon_max_i:res])
hf_out.createArray('/','lat',lat[lat_min_i:lat_max_i:res])
d = hf_in.root.data[(hf_in.root.data.shape[0]-lat_max_i*thin):\
(hf_in.root.data.shape[0]-lat_min_i*thin):\
thin,
lon_min_i*thin:\
lon_max_i*thin:\
thin]
d = map_utils.grid_convert(map_utils.grid_convert(d,'y-x+','x+y+')[::res,::res], 'x+y+','y-x+')
hf_out.createCArray('/','data',atom=tb.FloatAtom(),shape=d.shape,filters=tb.Filters(complevel=1,complib='zlib'))
hf_out.createCArray('/','mask',atom=tb.BoolAtom(),shape=d.shape,filters=tb.Filters(complevel=1,complib='zlib'))
hf_out.root.data.attrs.view = 'y-x+'
hf_out.root.data[:]=binfn(d)
hf_out.root.mask[:] = (d==maskval)+clipped_pete_mask
hf_out.close()
def test_reconciliation(self):
"Tests that reconciling multiple rasters works correctly for multiple views."
lims = {}
ns = {}
hfs = {}
views = []
for order in ['xy','yx']:
for first in ['+','-']:
for second in ['+','-']:
view = order[0]+first+order[1]+second
views.append(view)
lim_lo = np.random.uniform(-1,-.5,size=2)
lim_hi = np.random.uniform(.5,1,size=2)
lim = [lim_lo[0], lim_hi[0], lim_lo[1], lim_hi[1]]
n = np.random.randint(200,500,size=2)
lims[view]=np.array(lim)*180./np.pi
ns[view]=n
x = np.linspace(lim[0],lim[1],n[0])
y = np.linspace(lim[2],lim[3],n[1])
print view
fname, hf = write_datafile(x,y,view=view)
hfs[view] = hf
lon, lat, layers = reconcile_multiple_rasters([hfs[v].root for v in views], thin=2)
# Check that the layers all have the right shape
assert(np.all([(l.shape == (len(lon),len(lat))) for l in layers]))
# Check that the limits are inside the joint intersection
assert(np.all([(lon.min() >= l[0]) * (lon.max() <= l[1]) * (lat.min() >= l[2]) * (lat.max() <= l[3]) for l in lims.itervalues()]))
# Check that the coarseness is maximal
assert([(lims[v][1]-lims[v][0])/float(ns[v][0]-1) <= lon[1]-lon[0] for v in views])
assert([(lims[v][2]-lims[v][3])/float(ns[v][1]-1) <= lat[1]-lat[0] for v in views])
lonmin = np.min([l[0] for l in lims.itervalues()])
lonmax = np.max([l[1] for l in lims.itervalues()])
latmin = np.min([l[2] for l in lims.itervalues()])
latmax = np.max([l[3] for l in lims.itervalues()])
import pylab as pl
for i in xrange(len(views)):
pl.figure(figsize=(10,6))
pl.subplot(1,2,2)
pl.imshow(grid_convert(layers[i],'x+y+','y+x+'),extent=[lon.min(),lon.max(),lat.min(),lat.max()])
pl.title(views[i] + ': ' + str(hfs[views[i]].root.data.shape))
pl.axis([lonmin, lonmax, latmin, latmax])
pl.subplot(1,2,1)
pl.imshow(grid_convert(hfs[views[i]].root.data[:], views[i], 'y+x+'), extent = lims[views[i]])
pl.plot([lon.min(), lon.max(), lon.max(), lon.min()],[lat.min(), lat.min(), lat.max(), lat.max()],'r-')
pl.axis([lonmin, lonmax, latmin, latmax])
def write_datafile(x,y,t=0,sc=1,view='x+y+'):
hf = tb.openFile(os.path.join(data_dirname, 'test_extraction_%s.hdf5'%view),'w')
hf.createArray('/','lon',x*180./np.pi)
hf.createArray('/','lat',y*180./np.pi)
d = grid_convert(f(x,y,t,sc), 'x+y+', view)
hf.createCArray('/','data',shape=d.shape,chunkshape=(10,10),atom=tb.FloatAtom())
hf.root.data[:] = d
hf.root.data.attrs.view = view
hf.close()
return 'test_extraction_%s'%view, tb.openFile(os.path.join(data_dirname, 'test_extraction_%s.hdf5'%view))
def targ(inner, continent, fname, missing_val, t_start, t_end, t_chunk,
chunk_str):
grid_lims = getattr(master_grid, continent + '_lims')
xllc_here = (xllc + cellsize * (grid_lims['leftCol'] - 1))
xurc_here = (xllc + cellsize * (grid_lims['rightCol'] - 1))
yllc_here = (yllc + cellsize * (nrows - grid_lims['bottomRow']))
yurc_here = (yllc + cellsize * (nrows - grid_lims['topRow']))
outer = 0
hf = openFile(fname)
r = hf.root.realizations
V = hf.root.PyMCsamples.cols.V[:]
sl = np.empty((r.shape[1], r.shape[2], t_chunk))
sh = sl.shape
t = hf.root.t_axis[inner] + 2009
mo = int(round(rem(t * 12, 12)))
yr = int(t)
time_str = moname[mo] + ' %i' % yr
for k in xrange(t_chunk):
sl[:, :, k] = r[outer, :, :, inner + k]
sl[sl == missing_val] = NaN
out = 0
out = sl + np.random.normal(size=sh) * np.sqrt(V[outer])
out = invlogit(out.ravel()).reshape(sh)
out = np.mean(out, axis=2)
out = ma.masked_array(out, isnan(out))
b = basemap.Basemap(llcrnrlon=xllc_here,
llcrnrlat=yllc_here,
urcrnrlon=xurc_here,
urcrnrlat=yurc_here)
b.imshow(grid_convert(out, 'y-x+', 'x+y+'), cmap=matplotlib.cm.hot)
# b.drawcoastlines(color='.8',linewidth='2')
b.drawcountries(color='.8', linewidth='1')
colorbar()
# axis('off')
# title('%s starting %s'%(chunk_str, time_str))
title(str(yr))
savefig('anim-scratch/%i.png' % (inner))
close('all')
del sl, out
gc.collect()
hf.close()
def targ(inner,continent,fname,missing_val,t_start,t_end,t_chunk,chunk_str):
grid_lims = getattr(master_grid, continent+'_lims')
xllc_here = (xllc + cellsize * (grid_lims['leftCol']-1))
xurc_here = (xllc + cellsize * (grid_lims['rightCol']-1))
yllc_here = (yllc + cellsize * (nrows-grid_lims['bottomRow']))
yurc_here = (yllc + cellsize * (nrows-grid_lims['topRow']))
outer=0
hf = openFile(fname)
r = hf.root.realizations
V = hf.root.PyMCsamples.cols.V[:]
sl = np.empty((r.shape[1], r.shape[2], t_chunk))
sh = sl.shape
t = hf.root.t_axis[inner] + 2009
mo = int(round(rem(t*12,12)))
yr = int(t)
time_str = moname[mo] + ' %i'%yr
for k in xrange(t_chunk):
sl[:,:,k] = r[outer,:,:,inner + k]
sl[sl==missing_val]=NaN
out = 0
out = sl + np.random.normal(size=sh)*np.sqrt(V[outer])
out = invlogit(out.ravel()).reshape(sh)
out = np.mean(out, axis=2)
out = ma.masked_array(out, isnan(out))
b = basemap.Basemap(llcrnrlon=xllc_here, llcrnrlat=yllc_here,
urcrnrlon=xurc_here, urcrnrlat=yurc_here)
b.imshow(grid_convert(out,'y-x+','x+y+'), cmap=matplotlib.cm.hot)
# b.drawcoastlines(color='.8',linewidth='2')
b.drawcountries(color='.8',linewidth='1')
colorbar()
# axis('off')
# title('%s starting %s'%(chunk_str, time_str))
title(str(yr))
savefig('anim-scratch/%i.png'%(inner))
close('all')
del sl, out
gc.collect()
hf.close()
def current_state_map(M, session, species, mask, x, img_extent, thin=1, f2p=None, **kwds):
"Maps the current state of the model."
out = pm.utils.value(M.p)(x, f2p=f2p)
b = basemap.Basemap(*img_extent)
arr = np.ma.masked_array(out, mask=True-mask)
b.imshow(grid_convert(arr,'x+y+','y+x+'), interpolation='nearest')
# b.plot(pm.value(M.x_fr)[:,0]*180./np.pi, pm.value(M.x_fr)[:,1]*180./np.pi, 'g.', markersize=2)
# b.drawcoastlines(color=(.9,.4,.5))
# pl.colorbar()
return out, arr
def clip_raster(geom, lon, lat, view="y+x+"):
"""
Returns an array in the desired view indicating which pixels in the
meshgrid generated from lon and lat are inside geom.
"""
# Internal view is x+y+.
isin = np.zeros((len(lon), len(lat)), dtype="bool")
if isinstance(geom, shapely.geometry.multipolygon.MultiPolygon):
geoms = geom.geoms
else:
geoms = [geom]
for i, g in enumerate(geoms):
clip_raster_to_polygon(g, lon, lat, isin)
return map_utils.grid_convert(isin, "x+y+", view)
def get_covariate_submesh(name, grid_lims):
"""
Matches the specified spatial mesh to the 'master' covariate mesh.
"""
try:
order = getattr(mbgw.auxiliary_data, name)._v_attrs.order
except:
order = 'y-x+'
raw_shape = getattr(mbgw.auxiliary_data, name).data.shape
raw = getattr(mbgw.auxiliary_data, name).data[grid_lims['topRow']-1:grid_lims['bottomRow'], grid_lims['leftCol']-1:grid_lims['rightCol']]
out = grid_convert(raw, order, 'x+y+').copy()
targ_shape = (grid_lims['rightCol']-grid_lims['leftCol']+1, grid_lims['bottomRow']-grid_lims['topRow']+1)
if out.shape != targ_shape:
raise ValueError, "Grid %s's shape is %s with order %s. Cannot be sliced to shape %s. \n\tGrid limits: %s\n\tnrows: %i\n\tncols: %i"\
%(name, raw_shape, order, targ_shape, grid_lims, nrows, ncols)
return out
def test_extraction(self):
"Tests that extracting environmental layers works correctly for multiple views."
for order in ['xy','yx']:
for first in ['+','-']:
for second in ['+','-']:
view = order[0]+first+order[1]+second
fname, hf = write_datafile(x,y,view=view)
x_ind = np.random.randint(len(x)-2,size=n)+1
y_ind = np.random.randint(len(y)-2,size=n)+1
x_extract = x[x_ind]*180./np.pi + np.random.normal()*.001
y_extract = y[y_ind]*180./np.pi + np.random.normal()*.001
e2 = grid_convert(hf.root.data[:], view, 'x+y+')[(x_ind,y_ind)]
hf.close()
e1 = extract_environment(fname, np.vstack((x_extract,y_extract)).T, cache=False)
assert_almost_equal(e1,e2,decimal=3)
def presence_map(M, session, species, burn=0, thin=1, trace_thin=1, **kwds):
"Converts the trace to a map of presence probability."
from mpl_toolkits import basemap
import pylab as pl
import time
chain_len = len(M.db._h5file.root.chain0.PyMCsamples)
mask, x, img_extent = make_covering_raster(thin, M.env_variables, **kwds)
out = np.zeros(mask.shape)
time_count = -np.inf
time_start = time.time()
ptrace = M.trace('p')[:]
for i in xrange(burn, len(M.trace('p_find')[:]), trace_thin):
if time.time() - time_count > 10:
print (((i-burn)*100)/(chain_len)), '% complete',
if i>burn:
time_count = time.time()
print 'expect results '+time.ctime((time_count-time_start)*(chain_len-burn)/float(i-burn)+time_start)
else:
print
p = ptrace[i]
pe = p(x)
out += pe/float(chain_len-burn)
b = basemap.Basemap(*img_extent)
arr = np.ma.masked_array(out, mask=True-mask)
b.imshow(grid_convert(arr,'x+y+','y+x+'), interpolation='nearest')
# pl.colorbar()
plot_species(session, species[0], species[1], b, negs=True, **kwds)
return out, arr
def get_pseudoabsences(eo, buffer_width, n_pseudoabsences, layer_names, glob_name, eo_expand=0):
if eo_expand:
eo = lomem_buffer(eo, eo_expand)
fname = hashlib.sha1(geojson.dumps(eo)+'_'+str(buffer_width)+'_'+str(n_pseudoabsences)).hexdigest()+'.npy'
if fname in os.listdir('anopheles-caches'):
pseudoabsences = np.load(os.path.join('anopheles-caches', fname))
else:
if buffer_width >= 0:
buff = lomem_buffer(eo, buffer_width)
diff_buffer = buff.difference(eo)
elif buffer_width == -1:
diff_buffer = eo
else:
raise ValueError, 'Buffer width is negative, but not -1.'
if len(layer_names)>0:
template = layer_names[0]
else:
template = glob_name
lon, lat, test_raster, rtype = map_utils.import_raster(*os.path.split(template)[::-1])
# Right the raster
test_raster = map_utils.grid_convert(test_raster,'y-x+','x+y+')
# Convert lower-left coords to centroids
lon += (lon[1]-lon[0])/2.
lat += (lat[1]-lat[0])/2.
def testfn(lon_test,lat_test,r=test_raster,lon=lon,lat=lat):
lon_ind = np.argmin(np.abs(lon-lon_test))
lat_ind = np.argmin(np.abs(lat-lat_test))
return True-test_raster.mask[lon_ind,lat_ind]
pseudoabsences = np.vstack(map_utils.shapefile_utils.multipoly_sample(n_pseudoabsences, diff_buffer, test=testfn)).T
if not np.all([testfn(l1,l2) for l1,l2 in pseudoabsences]):
raise ValueError, 'Test failed for some pseudoabsences.'
np.save(os.path.join('anopheles-caches',fname), pseudoabsences)
return pseudoabsences, eo
def get_covariate_submesh(name, grid_lims):
"""
Matches the specified spatial mesh to the 'master' covariate mesh.
"""
try:
order = getattr(mbgw.auxiliary_data, name)._v_attrs.order
except:
order = 'y-x+'
raw_shape = getattr(mbgw.auxiliary_data, name).data.shape
raw = getattr(mbgw.auxiliary_data,
name).data[grid_lims['topRow'] - 1:grid_lims['bottomRow'],
grid_lims['leftCol'] - 1:grid_lims['rightCol']]
out = grid_convert(raw, order, 'x+y+').copy()
targ_shape = (grid_lims['rightCol'] - grid_lims['leftCol'] + 1,
grid_lims['bottomRow'] - grid_lims['topRow'] + 1)
if out.shape != targ_shape:
raise ValueError, "Grid %s's shape is %s with order %s. Cannot be sliced to shape %s. \n\tGrid limits: %s\n\tnrows: %i\n\tncols: %i"\
%(name, raw_shape, order, targ_shape, grid_lims, nrows, ncols)
return out
def subset_raster(r, llclati, llcloni, urclati, urcloni):
r_ = map_utils.grid_convert(r,'y-x+','x+y+')
return map_utils.grid_convert(r_[llcloni:urcloni,llclati:urclati],'x+y+','y-x+').astype('float32')
def create_realization(outfile_root,real_index, C,C_straightfromtrace, mean_ondata, M, covariate_mesh, tdata, data_locs, grids, axes, data_mesh_indices, where_in, where_out, n_blocks_x, n_blocks_y, relp, mask, thinning,indices,paramfileINDEX,NinThinnedBlock,TESTRANGE,TESTSQUARE):
print '\nON REALIZATION '+str(real_index)+'\n'
# define only realizations chunk of hdf5 realization file
out_arr = outfile_root.realizations
"""
Creates a single realization from the predictive distribution over specified space-time mesh.
"""
grid_shape = tuple([grid[2] for grid in grids])
#r.X11(width=8,height=4)
#r.par(mfrow=(1,2))
#r.plot(data_mesh_indices[:,0],data_mesh_indices[:,1],xlab="",ylab="",main="",cex=0.5)
#r.plot(data_locs[:,0],data_locs[:,1],xlab="",ylab="",main="",cex=0.5)
#from IPython.Debugger import Pdb
#Pdb(color_scheme='Linux').set_trace()
thin_grids = tuple([grid[:2]+(grid[2]/thinning,) for grid in grids])
thin_grid_shape = tuple([thin_grid[2] for thin_grid in thin_grids])
thin_axes = tuple([np.linspace(*thin_grid) for thin_grid in thin_grids])
mapgrid = np.array(mgrid[0:grid_shape[0],0:grid_shape[1]], dtype=float)
for i in xrange(2): normalize_for_mapcoords(mapgrid[i], thin_grid_shape[i]-1)
thin_mapgrid = np.array(mgrid[0:thin_grid_shape[0], 0:thin_grid_shape[1]], dtype=float)
for i in xrange(2): normalize_for_mapcoords(thin_mapgrid[i], grid_shape[i]-1)
def thin_to_full(thin_row):
return ndimage.map_coordinates(thin_row, mapgrid)
def full_to_thin(row):
return ndimage.map_coordinates(row, thin_mapgrid)
thin_mask = np.array(np.round(full_to_thin(mask)), dtype='bool')
# Container for x
thin_x = np.empty(thin_grid_shape[:2] + (3,))
mlon,mlat = np.meshgrid(*thin_axes[:2])
thin_x[:,:,0] = mlon.T
thin_x[:,:,1] = mlat.T
thin_x[:,:,2] = 0
x = np.empty(grid_shape[:2] + (3,))
mlon,mlat = np.meshgrid(*axes[:2])
x[:,:,0] = mlon.T
x[:,:,1] = mlat.T
x[:,:,2] = 0
del mlon, mlat
Cp = C.params
# In special case (america region) run only a test square using direct simulation, if it does not fail then just return
if TESTSQUARE:
getUnconditionedBlock(out_arr,real_index,grids,C_straightfromtrace,NinThinnedBlock=None,relp=relp,FULLRANK=False)
return()
# Prepare input dictionaries
covParamObj = {'Scale': Cp['scale'][0],
'amp': Cp['amp'][0],
'inc': Cp['inc'][0],
'ecc': Cp['ecc'][0],
't.lim.corr': Cp['tlc'][0],
'scale.t': Cp['st'][0],
'sin.frac': Cp['sf'][0]}
gridParamObj = {'YLLCORNER': grids[1][0]*rad_to_deg,
'CELLSIZE': (grids[1][1]-grids[1][0])/(grids[1][2]-1.)*rad_to_deg,
'NROWS':grid_shape[1],
'NCOLS':grid_shape[0]}
monthParamObj = {'Nmonths':grid_shape[2],'StartMonth':grids[2][0]}
# Call R preprocessing function and check to make sure no screwy re-casting has taken place.
t1 = time.time()
os.chdir(r_path)
preLoopObj = r.CONDSIMpreloop(covParamObj,gridParamObj,monthParamObj,indices.min(), indices.max(),paramfileINDEX)
tree_reader = reader(file('listSummary_preLoopObj_original_%i_%i.txt'%(indices.min(), indices.max())),delimiter=' ')
preLoopClassTree, junk = parse_tree(tree_reader)
preLoopObj = compare_tree(preLoopObj, preLoopClassTree)
OutMATlist = preLoopObj['OutMATlist']
tree_reader = reader(file('listSummary_OutMATlist_original_%i_%i.txt'%(indices.min(), indices.max())),delimiter=' ')
OutMATClassTree, junk = parse_tree(tree_reader)
OutMATlist = compare_tree(OutMATlist, OutMATClassTree)
os.chdir(curpath)
preLoop_time = time.time()-t1
print "preLoop_time :"+str(preLoop_time)
#from IPython.Debugger import Pdb
#Pdb(color_scheme='Linux').set_trace()
## Create and store unconditional realizations
print '\tGenerating unconditional realizations.'
t1 = time.time()
for i in xrange(grid_shape[2]):
print 'On month :'+str(i)
#print 'OutMATlist:'
#print OutMATlist
os.chdir(r_path)
monthObject = r.CONDSIMmonthloop(i+1,preLoopObj,OutMATlist, indices.min(), indices.max(),paramfileINDEX)
#monthObject = r.CONDSIMmonthloop(i+1,preLoopObj,OutMATlist,paramfileINDEX)
os.chdir(curpath)
OutMATlist= monthObject['OutMATlist']
MonthGrid = monthObject['MonthGrid']
out_arr[real_index,:,:,i] = MonthGrid[:grid_shape[1],:grid_shape[0]]
t2 = time.time()
print '\t\tDone in %f'%(t2-t1)
print "monthloop_time :"+str(t2-t1)+" for "+str(grid_shape[2])+" months"
# delete unneeded R products
del OutMATlist, preLoopObj, MonthGrid, monthObject
#############################~TEMP
# ################################~TEMP DIRECTLY JOIN SIMULATE UNCODITIONED BLOCK FOR TESTING
# getUnconditionedBlock(out_arr,real_index,grids,C_straightfromtrace,NinThinnedBlock=None,relp=relp,FULLRANK=False)
# #print 'variance of unconditioned block = '+str(round(np.var(out_arr),10))
# #print 'variance of unconditioned block month 6 = '+str(round(np.var(out_arr[:,:,:,6]),10))
# #examineRealization(outfile_root,real_index,6,15,None,None,conditioned=False,flipVertical="FALSE",SPACE=True,TIME=True)
# ################################~TEMP
# Figure out pdata
pdata = np.empty(tdata.shape)
for i in xrange(len(where_in)):
index = where_in[i]
pdata[index] = out_arr[real_index, grid_shape[1]-1-data_mesh_indices[index,1], data_mesh_indices[index,0], data_mesh_indices[index,2]]
# jointly simulate at data points conditional on block
## first get XYZT list of locations of a regular thinned sample from block
#array3d = out_arr[real_index,:,:,:]
ThinnedBlockXYTZlists = getThinnedBlockXYTZlists (out_arr,real_index,grids,NinThinnedBlock)
xyt_in = ThinnedBlockXYTZlists['xyt_in']
z_in = ThinnedBlockXYTZlists['z_in']
# get locations of data outside block
xyt_out = data_locs[where_out]
# now we have locations and values of thinned sample from the block, and locations we want to predict at outside the block,go ahead and
# get simulated values of the latter, conditonal on the former
print '\tsimulating over '+str(len(xyt_out[:,0]))+' locations outside block using thinned block sample of '+str(len(z_in))+' points'
t1=time.time()
z_out = predictPointsFromBlock(xyt_in,z_in, xyt_out,C_straightfromtrace,relp)
print '\ttime for simulation: '+str(time.time()-t1)
#########################################CHECK COVARIANCE STRUCTURE
#r.X11(width=12,height=12)
#r.par(mfrow=(3,2))
# test marginal space and time covariance structures of points outside block
#cfdict_out = getEmpiricalCovarianceFunction_STmarginals(xyt_out,z_out,mu=0,margTol_S=0.05,margTol_T=0.9/12,nbins=20, cutoff = 0.8)
#plotEmpiricalCovarianceFunction(cfdict_out['space'],CovModelObj=C_straightfromtrace,spaceORtime="space", cutoff = 0.8, title="Points outside (S) "+str(paramfileINDEX))
#plotEmpiricalCovarianceFunction(cfdict_out['time'],CovModelObj=C_straightfromtrace,spaceORtime="time", cutoff = 0.8, title="Points outside (T)"+str(paramfileINDEX))
# test marginal space and time covariance structures of points inside block
#cfdict_in = getEmpiricalCovarianceFunction_STmarginals(xyt_in,z_in,mu=0,margTol_S=0.05,margTol_T=0.9/12,nbins=20, cutoff = 0.8)
#plotEmpiricalCovarianceFunction(cfdict_in['space'],CovModelObj=C_straightfromtrace,spaceORtime="space", cutoff = 0.8, title="Points inside (S)"+str(paramfileINDEX))
#plotEmpiricalCovarianceFunction(cfdict_in['time'],CovModelObj=C_straightfromtrace,spaceORtime="time", cutoff = 0.8, title="Points inside (T)"+str(paramfileINDEX))
# test marginal space and time covariance structures of points inside and outside block
#cfdict_inout = getEmpiricalCovarianceFunction_STmarginals(np.vstack((xyt_in,xyt_out)),np.hstack((z_in,z_out)),mu=0,margTol_S=0.05,margTol_T=0.9/12,nbins=20, cutoff = 0.8)
#plotEmpiricalCovarianceFunction(cfdict_inout['space'],CovModelObj=C_straightfromtrace,spaceORtime="space", cutoff = 0.8, title="Points inside (S)"+str(paramfileINDEX))
#plotEmpiricalCovarianceFunction(cfdict_inout['time'],CovModelObj=C_straightfromtrace,spaceORtime="time", cutoff = 0.8, title="Points inside (T)"+str(paramfileINDEX))
#########################################CHECK COVARIANCE STRUCTURE
# assign these values to pdata
pdata[where_out] = z_out
###############################~~TEMP
#return()
#####################################
# Bring in data.
print '\tKriging to bring in data.'
print '\tPreprocessing.'
t1 = time.time()
dev_posdef, xbi, ybi, dl_posdef = preprocess(C, data_locs, thin_grids, thin_x, n_blocks_x, n_blocks_y, tdata, pdata, relp, mean_ondata)
t2 = time.time()
print '\t\tDone in %f'%(t2-t1)
#from IPython.Debugger import Pdb
#Pdb(color_scheme='Linux').set_trace()
thin_row = np.empty(thin_grid_shape[:2], dtype=np.float32)
print '\tKriging.'
t1 = time.time()
for i in xrange(grid_shape[2]-1,-1,-1):
thin_row.fill(0.)
thin_x[:,:,2] = axes[2][i]
x[:,:,2] = axes[2][i]
krige_month(C, i, dl_posdef, thin_grid_shape, n_blocks_x, n_blocks_y, xbi, ybi, thin_x, dev_posdef, thin_row, thin_mask)
row = ndimage.map_coordinates(thin_row, mapgrid)
row += covariate_mesh
row += M(x)
row += grid_convert(out_arr[real_index,:,:,i], 'y-x+', 'x+y+')
# NaN the oceans to save storage
row[np.where(1-mask)] = missing_val
# if we are checking for plausible max and min values (Vs f at data), implement tet on conditioned values for this month
monthMin = np.min(row[np.where(mask)])
monthMax = np.max(row[np.where(mask)])
pointsMin = np.min(tdata)
pointsMax = np.max(tdata)
pointsRange = pointsMax-pointsMin
threshMin = pointsMin-(pointsRange*TESTRANGE)
threshMax = pointsMax+(pointsRange*TESTRANGE)
print('On month '+str(i)+' : f range=('+str(pointsMin)+','+str(pointsMax)+') ; month range=('+str(monthMin)+','+str(monthMax)+')')
if TESTRANGE!=False:
if(((monthMin<threshMin) | (monthMax>threshMax))):
raise ValueError ('Killing realization on month '+str(i)+' : f range=('+str(pointsMin)+','+str(pointsMax)+') ; month range=('+str(monthMin)+','+str(monthMax)+')')
out_arr[real_index,:,:,i] = grid_convert(row, 'x+y+','y-x+')
####################################TEMP
#print 'variance of conditioned block month 6 = '+str(round(np.var(out_arr[:,:,:,6]),10))
#print 'variance of conditioned block = '+str(round(np.var(out_arr),10))
#examineRealization(outfile_root,real_index,6,15,None,None,conditioned=True,flipVertical="FALSE",SPACE=True,TIME=True)
########################################
print '\t\tDone in %f'%(time.time()-t1)
def create_realization(outfile_root, real_index, C, C_straightfromtrace,
mean_ondata, M, covariate_mesh, tdata, data_locs, grids,
axes, data_mesh_indices, where_in, where_out,
n_blocks_x, n_blocks_y, relp, mask, thinning, indices,
paramfileINDEX, NinThinnedBlock):
# define only realizations chunk of hdf5 realization file
out_arr = outfile_root.realizations
"""
Creates a single realization from the predictive distribution over specified space-time mesh.
"""
grid_shape = tuple([grid[2] for grid in grids])
#r.X11(width=8,height=4)
#r.par(mfrow=(1,2))
#r.plot(data_mesh_indices[:,0],data_mesh_indices[:,1],xlab="",ylab="",main="",cex=0.5)
#r.plot(data_locs[:,0],data_locs[:,1],xlab="",ylab="",main="",cex=0.5)
#from IPython.Debugger import Pdb
#Pdb(color_scheme='Linux').set_trace()
thin_grids = tuple([grid[:2] + (grid[2] / thinning, ) for grid in grids])
thin_grid_shape = tuple([thin_grid[2] for thin_grid in thin_grids])
thin_axes = tuple([np.linspace(*thin_grid) for thin_grid in thin_grids])
mapgrid = np.array(mgrid[0:grid_shape[0], 0:grid_shape[1]], dtype=float)
for i in xrange(2):
normalize_for_mapcoords(mapgrid[i], thin_grid_shape[i] - 1)
thin_mapgrid = np.array(mgrid[0:thin_grid_shape[0], 0:thin_grid_shape[1]],
dtype=float)
for i in xrange(2):
normalize_for_mapcoords(thin_mapgrid[i], grid_shape[i] - 1)
def thin_to_full(thin_row):
return ndimage.map_coordinates(thin_row, mapgrid)
def full_to_thin(row):
return ndimage.map_coordinates(row, thin_mapgrid)
thin_mask = np.array(np.round(full_to_thin(mask)), dtype='bool')
# Container for x
thin_x = np.empty(thin_grid_shape[:2] + (3, ))
mlon, mlat = np.meshgrid(*thin_axes[:2])
thin_x[:, :, 0] = mlon.T
thin_x[:, :, 1] = mlat.T
thin_x[:, :, 2] = 0
x = np.empty(grid_shape[:2] + (3, ))
mlon, mlat = np.meshgrid(*axes[:2])
x[:, :, 0] = mlon.T
x[:, :, 1] = mlat.T
x[:, :, 2] = 0
del mlon, mlat
Cp = C.params
# Prepare input dictionaries
covParamObj = {
'Scale': Cp['scale'][0],
'amp': Cp['amp'][0],
'inc': Cp['inc'][0],
'ecc': Cp['ecc'][0],
't.lim.corr': Cp['tlc'][0],
'scale.t': Cp['st'][0],
'sin.frac': Cp['sf'][0]
}
gridParamObj = {
'YLLCORNER': grids[1][0] * rad_to_deg,
'CELLSIZE':
(grids[1][1] - grids[1][0]) / (grids[1][2] - 1.) * rad_to_deg,
'NROWS': grid_shape[1],
'NCOLS': grid_shape[0]
}
monthParamObj = {'Nmonths': grid_shape[2], 'StartMonth': grids[2][0]}
################################~TEMP
# Call R preprocessing function and check to make sure no screwy re-casting has taken place.
#t1 = time.time()
#os.chdir(r_path)
#preLoopObj = r.CONDSIMpreloop(covParamObj,gridParamObj,monthParamObj,indices.min(), indices.max(),paramfileINDEX)
#tree_reader = reader(file('listSummary_preLoopObj_original_%i_%i.txt'%(indices.min(), indices.max())),delimiter=' ')
#preLoopClassTree, junk = parse_tree(tree_reader)
#preLoopObj = compare_tree(preLoopObj, preLoopClassTree)
#OutMATlist = preLoopObj['OutMATlist']
#tree_reader = reader(file('listSummary_OutMATlist_original_%i_%i.txt'%(indices.min(), indices.max())),delimiter=' ')
#OutMATClassTree, junk = parse_tree(tree_reader)
#OutMATlist = compare_tree(OutMATlist, OutMATClassTree)
#os.chdir(curpath)
#preLoop_time = time.time()-t1
#print "preLoop_time :"+str(preLoop_time)
## Create and store unconditional realizations
#print '\tGenerating unconditional realizations.'
#t1 = time.time()
#for i in xrange(grid_shape[2]):
# print 'On month :'+str(i)
# #print 'OutMATlist:'
# #print OutMATlist
# os.chdir(r_path)
# monthObject = r.CONDSIMmonthloop(i+1,preLoopObj,OutMATlist, indices.min(), indices.max(),paramfileINDEX)
# #monthObject = r.CONDSIMmonthloop(i+1,preLoopObj,OutMATlist,paramfileINDEX)
# os.chdir(curpath)
# OutMATlist= monthObject['OutMATlist']
# MonthGrid = monthObject['MonthGrid']
# out_arr[real_index,:,:,i] = MonthGrid[:grid_shape[1],:grid_shape[0]]
#t2 = time.time()
#print '\t\tDone in %f'%(t2-t1)
#print "monthloop_time :"+str(t2-t1)+" for "+str(grid_shape[2])+" months"
# delete unneeded R products
#del OutMATlist, preLoopObj, MonthGrid, monthObject
################################~TEMP
################################~TEMP DIRECTLY JOIN SIMULATE UNCODITIONED BLOCK FOR TESTING
getUnconditionedBlock(out_arr,
real_index,
grids,
C_straightfromtrace,
NinThinnedBlock=None,
relp=None,
FULLRANK=False)
print 'variance of unconditioned block = ' + str(round(
np.var(out_arr), 10))
print 'variance of unconditioned block month 6 = ' + str(
round(np.var(out_arr[:, :, :, 6]), 10))
examineRealization(outfile_root,
0,
6,
15,
None,
None,
conditioned=False,
flipVertical="FALSE",
SPACE=True,
TIME=True)
################################~TEMP
# Figure out pdata
pdata = np.empty(tdata.shape)
for i in xrange(len(where_in)):
pdata[where_in[i]] = out_arr[real_index, grid_shape[1] - 1 -
data_mesh_indices[i, 1],
data_mesh_indices[i, 0],
data_mesh_indices[i, 2]]
# jointly simulate at data points conditional on block
## first get XYZT list of locations of a regular thinned sample from block
#array3d = out_arr[real_index,:,:,:]
ThinnedBlockXYTZlists = getThinnedBlockXYTZlists(out_arr, real_index,
grids, NinThinnedBlock)
xyt_in = ThinnedBlockXYTZlists['xyt_in']
z_in = ThinnedBlockXYTZlists['z_in']
## get locations of data outside block (referenced by grid location)
#XYT_out_gridlocs = data_mesh_indices[where_out]
#
### convert these grid lcoations into actual position in radians and months
#coordsDict = gridParams_2_XYTmarginallists(grids)
#xcoords = coordsDict['xcoords']
#ycoords = coordsDict['ycoords']
#tcoords = coordsDict['tcoords']
#
#x_out = xcoords[XYT_out_gridlocs[:,0]]
#y_out = ycoords[XYT_out_gridlocs[:,1]]
#t_out = tcoords[XYT_out_gridlocs[:,2]]
#
#xyt_out = np.vstack((x_out,y_out,t_out)).T
# get locations of data outside block
xyt_out = data_locs[where_out]
# now we have locations and values of thinned sample from the block, and locations we want to predict at outside the block,go ahead and
# get simulated values of the latter, conditonal on the former
print '\tsimulating over ' + str(
len(xyt_out[:, 0])
) + ' locations outside block using thinned block sample of ' + str(
len(z_in)) + ' points'
t1 = time.time()
z_out = predictPointsFromBlock(xyt_in, z_in, xyt_out, C_straightfromtrace,
relp)
print '\ttime for simulation: ' + str(time.time() - t1)
#########################################CHECK COVARIANCE STRUCTURE
#r.X11(width=12,height=12)
#r.par(mfrow=(3,2))
# test marginal space and time covariance structures of points outside block
#cfdict_out = getEmpiricalCovarianceFunction_STmarginals(xyt_out,z_out,mu=0,margTol_S=0.05,margTol_T=0.9/12,nbins=20, cutoff = 0.8)
#plotEmpiricalCovarianceFunction(cfdict_out['space'],CovModelObj=C_straightfromtrace,spaceORtime="space", cutoff = 0.8, title="Points outside (S) "+str(paramfileINDEX))
#plotEmpiricalCovarianceFunction(cfdict_out['time'],CovModelObj=C_straightfromtrace,spaceORtime="time", cutoff = 0.8, title="Points outside (T)"+str(paramfileINDEX))
# test marginal space and time covariance structures of points inside block
#cfdict_in = getEmpiricalCovarianceFunction_STmarginals(xyt_in,z_in,mu=0,margTol_S=0.05,margTol_T=0.9/12,nbins=20, cutoff = 0.8)
#plotEmpiricalCovarianceFunction(cfdict_in['space'],CovModelObj=C_straightfromtrace,spaceORtime="space", cutoff = 0.8, title="Points inside (S)"+str(paramfileINDEX))
#plotEmpiricalCovarianceFunction(cfdict_in['time'],CovModelObj=C_straightfromtrace,spaceORtime="time", cutoff = 0.8, title="Points inside (T)"+str(paramfileINDEX))
# test marginal space and time covariance structures of points inside and outside block
#cfdict_inout = getEmpiricalCovarianceFunction_STmarginals(np.vstack((xyt_in,xyt_out)),np.hstack((z_in,z_out)),mu=0,margTol_S=0.05,margTol_T=0.9/12,nbins=20, cutoff = 0.8)
#plotEmpiricalCovarianceFunction(cfdict_inout['space'],CovModelObj=C_straightfromtrace,spaceORtime="space", cutoff = 0.8, title="Points inside (S)"+str(paramfileINDEX))
#plotEmpiricalCovarianceFunction(cfdict_inout['time'],CovModelObj=C_straightfromtrace,spaceORtime="time", cutoff = 0.8, title="Points inside (T)"+str(paramfileINDEX))
#########################################CHECK COVARIANCE STRUCTURE
# assign these values to pdata
pdata[where_out] = z_out
###############################~~TEMP
#return()
#####################################
# Bring in data.
print '\tKriging to bring in data.'
print '\tPreprocessing.'
t1 = time.time()
dev_posdef, xbi, ybi, dl_posdef = preprocess(C, data_locs, thin_grids,
thin_x, n_blocks_x,
n_blocks_y, tdata, pdata,
relp, mean_ondata)
t2 = time.time()
print '\t\tDone in %f' % (t2 - t1)
###############################~~TEMP
#from IPython.Debugger import Pdb
#Pdb(color_scheme='Linux').set_trace()
#####################################
thin_row = np.empty(thin_grid_shape[:2], dtype=np.float32)
print '\tKriging.'
t1 = time.time()
C1 = pm.gp.NearlyFullRankCovariance(C_straightfromtrace.eval_fun,
**C_straightfromtrace.params)
C2 = pm.gp.NearlyFullRankCovariance(C_straightfromtrace.eval_fun,
**C_straightfromtrace.params)
M1 = pm.gp.Mean(lambda x: np.zeros(x.shape[:-1]))
M2 = pm.gp.Mean(lambda x: np.zeros(x.shape[:-1]))
cholfac = C1.cholesky(data_locs, apply_pivot=False)
U = cholfac['U']
piv = cholfac['pivots']
m = U.shape[0]
w = np.linalg.solve(
U[:m, :m], np.linalg.solve(U[:m, :m].T,
(tdata - mean_ondata)[piv[:m]]))
pm.gp.observe(M2, C2, data_locs, pdata)
for i in xrange(grid_shape[2] - 1, -1, -1):
thin_row.fill(0.)
thin_x[:, :, 2] = axes[2][i]
x[:, :, 2] = axes[2][i]
simkrige = M2(x)
dkrige = covariate_mesh + M(x) + np.dot(
C1(x, data_locs[piv[:m]]).view(ndarray), w).reshape(x.shape[:-1])
simsurf = grid_convert(out_arr[real_index, :, :, i], 'y-x+', 'x+y+')
import pylab as pl
pl.imshow(simsurf)
pl.colorbar()
pl.title('Simulated')
pl.figure()
pl.imshow(simkrige)
pl.colorbar()
pl.title('Sim krige')
pl.figure()
pl.imshow(simsurf - simkrige)
pl.colorbar()
pl.title('Adjusted sim')
pl.figure()
pl.imshow(dkrige)
pl.colorbar()
pl.title('Data krige')
pl.figure()
pl.imshow(simsurf - simkrige + dkrige)
pl.colorbar()
pl.title('Kriged sim')
krige_month(C, i, dl_posdef, thin_grid_shape, n_blocks_x, n_blocks_y,
xbi, ybi, thin_x, dev_posdef, thin_row, thin_mask)
row = ndimage.map_coordinates(thin_row, mapgrid)
row += covariate_mesh
row += M(x)
row += grid_convert(out_arr[real_index, :, :, i], 'y-x+', 'x+y+')
pl.figure()
pl.imshow(row)
pl.colorbar()
pl.title('Original method')
from IPython.Debugger import Pdb
Pdb(color_scheme='Linux').set_trace()
# NaN the oceans to save storage
row[np.where(1 - mask)] = missing_val
out_arr[real_index, :, :, i] = grid_convert(row, 'x+y+', 'y-x+')
####################################TEMP
print 'variance of conditioned block month 6 = ' + str(
round(np.var(out_arr[:, :, :, 6]), 10))
print 'variance of conditioned block = ' + str(round(np.var(out_arr), 10))
examineRealization(outfile_root,
0,
6,
15,
None,
None,
conditioned=True,
flipVertical="FALSE",
SPACE=True,
TIME=True)
########################################
print '\t\tDone in %f' % (time.time() - t1)
def create_realization(outfile_root, real_index, C, C_straightfromtrace,
mean_ondata, M, covariate_mesh, tdata, data_locs, grids,
axes, data_mesh_indices, where_in, where_out,
n_blocks_x, n_blocks_y, relp, mask, thinning, indices,
paramfileINDEX, NinThinnedBlock, TESTRANGE, TESTSQUARE):
print '\nON REALIZATION ' + str(real_index) + '\n'
# define only realizations chunk of hdf5 realization file
out_arr = outfile_root.realizations
"""
Creates a single realization from the predictive distribution over specified space-time mesh.
"""
grid_shape = tuple([grid[2] for grid in grids])
#r.X11(width=8,height=4)
#r.par(mfrow=(1,2))
#r.plot(data_mesh_indices[:,0],data_mesh_indices[:,1],xlab="",ylab="",main="",cex=0.5)
#r.plot(data_locs[:,0],data_locs[:,1],xlab="",ylab="",main="",cex=0.5)
#from IPython.Debugger import Pdb
#Pdb(color_scheme='Linux').set_trace()
thin_grids = tuple([grid[:2] + (grid[2] / thinning, ) for grid in grids])
thin_grid_shape = tuple([thin_grid[2] for thin_grid in thin_grids])
thin_axes = tuple([np.linspace(*thin_grid) for thin_grid in thin_grids])
mapgrid = np.array(mgrid[0:grid_shape[0], 0:grid_shape[1]], dtype=float)
for i in xrange(2):
normalize_for_mapcoords(mapgrid[i], thin_grid_shape[i] - 1)
thin_mapgrid = np.array(mgrid[0:thin_grid_shape[0], 0:thin_grid_shape[1]],
dtype=float)
for i in xrange(2):
normalize_for_mapcoords(thin_mapgrid[i], grid_shape[i] - 1)
def thin_to_full(thin_row):
return ndimage.map_coordinates(thin_row, mapgrid)
def full_to_thin(row):
return ndimage.map_coordinates(row, thin_mapgrid)
thin_mask = np.array(np.round(full_to_thin(mask)), dtype='bool')
# Container for x
thin_x = np.empty(thin_grid_shape[:2] + (3, ))
mlon, mlat = np.meshgrid(*thin_axes[:2])
thin_x[:, :, 0] = mlon.T
thin_x[:, :, 1] = mlat.T
thin_x[:, :, 2] = 0
x = np.empty(grid_shape[:2] + (3, ))
mlon, mlat = np.meshgrid(*axes[:2])
x[:, :, 0] = mlon.T
x[:, :, 1] = mlat.T
x[:, :, 2] = 0
del mlon, mlat
Cp = C.params
# In special case (america region) run only a test square using direct simulation, if it does not fail then just return
if TESTSQUARE:
getUnconditionedBlock(out_arr,
real_index,
grids,
C_straightfromtrace,
NinThinnedBlock=None,
relp=relp,
FULLRANK=False)
return ()
# Prepare input dictionaries
covParamObj = {
'Scale': Cp['scale'][0],
'amp': Cp['amp'][0],
'inc': Cp['inc'][0],
'ecc': Cp['ecc'][0],
't.lim.corr': Cp['tlc'][0],
'scale.t': Cp['st'][0],
'sin.frac': Cp['sf'][0]
}
gridParamObj = {
'YLLCORNER': grids[1][0] * rad_to_deg,
'CELLSIZE':
(grids[1][1] - grids[1][0]) / (grids[1][2] - 1.) * rad_to_deg,
'NROWS': grid_shape[1],
'NCOLS': grid_shape[0]
}
monthParamObj = {'Nmonths': grid_shape[2], 'StartMonth': grids[2][0]}
# Call R preprocessing function and check to make sure no screwy re-casting has taken place.
t1 = time.time()
os.chdir(r_path)
preLoopObj = r.CONDSIMpreloop(covParamObj, gridParamObj, monthParamObj,
indices.min(), indices.max(), paramfileINDEX)
tree_reader = reader(file('listSummary_preLoopObj_original_%i_%i.txt' %
(indices.min(), indices.max())),
delimiter=' ')
preLoopClassTree, junk = parse_tree(tree_reader)
preLoopObj = compare_tree(preLoopObj, preLoopClassTree)
OutMATlist = preLoopObj['OutMATlist']
tree_reader = reader(file('listSummary_OutMATlist_original_%i_%i.txt' %
(indices.min(), indices.max())),
delimiter=' ')
OutMATClassTree, junk = parse_tree(tree_reader)
OutMATlist = compare_tree(OutMATlist, OutMATClassTree)
os.chdir(curpath)
preLoop_time = time.time() - t1
print "preLoop_time :" + str(preLoop_time)
#from IPython.Debugger import Pdb
#Pdb(color_scheme='Linux').set_trace()
## Create and store unconditional realizations
print '\tGenerating unconditional realizations.'
t1 = time.time()
for i in xrange(grid_shape[2]):
print 'On month :' + str(i)
#print 'OutMATlist:'
#print OutMATlist
os.chdir(r_path)
monthObject = r.CONDSIMmonthloop(i + 1, preLoopObj, OutMATlist,
indices.min(), indices.max(),
paramfileINDEX)
#monthObject = r.CONDSIMmonthloop(i+1,preLoopObj,OutMATlist,paramfileINDEX)
os.chdir(curpath)
OutMATlist = monthObject['OutMATlist']
MonthGrid = monthObject['MonthGrid']
out_arr[real_index, :, :,
i] = MonthGrid[:grid_shape[1], :grid_shape[0]]
t2 = time.time()
print '\t\tDone in %f' % (t2 - t1)
print "monthloop_time :" + str(t2 - t1) + " for " + str(
grid_shape[2]) + " months"
# delete unneeded R products
del OutMATlist, preLoopObj, MonthGrid, monthObject
#############################~TEMP
# ################################~TEMP DIRECTLY JOIN SIMULATE UNCODITIONED BLOCK FOR TESTING
# getUnconditionedBlock(out_arr,real_index,grids,C_straightfromtrace,NinThinnedBlock=None,relp=relp,FULLRANK=False)
# #print 'variance of unconditioned block = '+str(round(np.var(out_arr),10))
# #print 'variance of unconditioned block month 6 = '+str(round(np.var(out_arr[:,:,:,6]),10))
# #examineRealization(outfile_root,real_index,6,15,None,None,conditioned=False,flipVertical="FALSE",SPACE=True,TIME=True)
# ################################~TEMP
# Figure out pdata
pdata = np.empty(tdata.shape)
for i in xrange(len(where_in)):
index = where_in[i]
pdata[index] = out_arr[real_index,
grid_shape[1] - 1 - data_mesh_indices[index, 1],
data_mesh_indices[index,
0], data_mesh_indices[index,
2]]
# jointly simulate at data points conditional on block
## first get XYZT list of locations of a regular thinned sample from block
#array3d = out_arr[real_index,:,:,:]
ThinnedBlockXYTZlists = getThinnedBlockXYTZlists(out_arr, real_index,
grids, NinThinnedBlock)
xyt_in = ThinnedBlockXYTZlists['xyt_in']
z_in = ThinnedBlockXYTZlists['z_in']
# get locations of data outside block
xyt_out = data_locs[where_out]
# now we have locations and values of thinned sample from the block, and locations we want to predict at outside the block,go ahead and
# get simulated values of the latter, conditonal on the former
print '\tsimulating over ' + str(
len(xyt_out[:, 0])
) + ' locations outside block using thinned block sample of ' + str(
len(z_in)) + ' points'
t1 = time.time()
z_out = predictPointsFromBlock(xyt_in, z_in, xyt_out, C_straightfromtrace,
relp)
print '\ttime for simulation: ' + str(time.time() - t1)
#########################################CHECK COVARIANCE STRUCTURE
#r.X11(width=12,height=12)
#r.par(mfrow=(3,2))
# test marginal space and time covariance structures of points outside block
#cfdict_out = getEmpiricalCovarianceFunction_STmarginals(xyt_out,z_out,mu=0,margTol_S=0.05,margTol_T=0.9/12,nbins=20, cutoff = 0.8)
#plotEmpiricalCovarianceFunction(cfdict_out['space'],CovModelObj=C_straightfromtrace,spaceORtime="space", cutoff = 0.8, title="Points outside (S) "+str(paramfileINDEX))
#plotEmpiricalCovarianceFunction(cfdict_out['time'],CovModelObj=C_straightfromtrace,spaceORtime="time", cutoff = 0.8, title="Points outside (T)"+str(paramfileINDEX))
# test marginal space and time covariance structures of points inside block
#cfdict_in = getEmpiricalCovarianceFunction_STmarginals(xyt_in,z_in,mu=0,margTol_S=0.05,margTol_T=0.9/12,nbins=20, cutoff = 0.8)
#plotEmpiricalCovarianceFunction(cfdict_in['space'],CovModelObj=C_straightfromtrace,spaceORtime="space", cutoff = 0.8, title="Points inside (S)"+str(paramfileINDEX))
#plotEmpiricalCovarianceFunction(cfdict_in['time'],CovModelObj=C_straightfromtrace,spaceORtime="time", cutoff = 0.8, title="Points inside (T)"+str(paramfileINDEX))
# test marginal space and time covariance structures of points inside and outside block
#cfdict_inout = getEmpiricalCovarianceFunction_STmarginals(np.vstack((xyt_in,xyt_out)),np.hstack((z_in,z_out)),mu=0,margTol_S=0.05,margTol_T=0.9/12,nbins=20, cutoff = 0.8)
#plotEmpiricalCovarianceFunction(cfdict_inout['space'],CovModelObj=C_straightfromtrace,spaceORtime="space", cutoff = 0.8, title="Points inside (S)"+str(paramfileINDEX))
#plotEmpiricalCovarianceFunction(cfdict_inout['time'],CovModelObj=C_straightfromtrace,spaceORtime="time", cutoff = 0.8, title="Points inside (T)"+str(paramfileINDEX))
#########################################CHECK COVARIANCE STRUCTURE
# assign these values to pdata
pdata[where_out] = z_out
###############################~~TEMP
#return()
#####################################
# Bring in data.
print '\tKriging to bring in data.'
print '\tPreprocessing.'
t1 = time.time()
dev_posdef, xbi, ybi, dl_posdef = preprocess(C, data_locs, thin_grids,
thin_x, n_blocks_x,
n_blocks_y, tdata, pdata,
relp, mean_ondata)
t2 = time.time()
print '\t\tDone in %f' % (t2 - t1)
#from IPython.Debugger import Pdb
#Pdb(color_scheme='Linux').set_trace()
thin_row = np.empty(thin_grid_shape[:2], dtype=np.float32)
print '\tKriging.'
t1 = time.time()
for i in xrange(grid_shape[2] - 1, -1, -1):
thin_row.fill(0.)
thin_x[:, :, 2] = axes[2][i]
x[:, :, 2] = axes[2][i]
krige_month(C, i, dl_posdef, thin_grid_shape, n_blocks_x, n_blocks_y,
xbi, ybi, thin_x, dev_posdef, thin_row, thin_mask)
row = ndimage.map_coordinates(thin_row, mapgrid)
row += covariate_mesh
row += M(x)
row += grid_convert(out_arr[real_index, :, :, i], 'y-x+', 'x+y+')
# NaN the oceans to save storage
row[np.where(1 - mask)] = missing_val
# if we are checking for plausible max and min values (Vs f at data), implement tet on conditioned values for this month
monthMin = np.min(row[np.where(mask)])
monthMax = np.max(row[np.where(mask)])
pointsMin = np.min(tdata)
pointsMax = np.max(tdata)
pointsRange = pointsMax - pointsMin
threshMin = pointsMin - (pointsRange * TESTRANGE)
threshMax = pointsMax + (pointsRange * TESTRANGE)
print('On month ' + str(i) + ' : f range=(' + str(pointsMin) + ',' +
str(pointsMax) + ') ; month range=(' + str(monthMin) + ',' +
str(monthMax) + ')')
if TESTRANGE != False:
if (((monthMin < threshMin) | (monthMax > threshMax))):
raise ValueError('Killing realization on month ' + str(i) +
' : f range=(' + str(pointsMin) + ',' +
str(pointsMax) + ') ; month range=(' +
str(monthMin) + ',' + str(monthMax) + ')')
out_arr[real_index, :, :, i] = grid_convert(row, 'x+y+', 'y-x+')
####################################TEMP
#print 'variance of conditioned block month 6 = '+str(round(np.var(out_arr[:,:,:,6]),10))
#print 'variance of conditioned block = '+str(round(np.var(out_arr),10))
#examineRealization(outfile_root,real_index,6,15,None,None,conditioned=True,flipVertical="FALSE",SPACE=True,TIME=True)
########################################
print '\t\tDone in %f' % (time.time() - t1)
def create_realization(outfile_root,real_index, C,C_straightfromtrace, mean_ondata, M, covariate_mesh, tdata, data_locs, grids, axes, data_mesh_indices, where_in, where_out, n_blocks_x, n_blocks_y, relp, mask, thinning,indices,paramfileINDEX,NinThinnedBlock):
# define only realizations chunk of hdf5 realization file
out_arr = outfile_root.realizations
"""
Creates a single realization from the predictive distribution over specified space-time mesh.
"""
grid_shape = tuple([grid[2] for grid in grids])
#r.X11(width=8,height=4)
#r.par(mfrow=(1,2))
#r.plot(data_mesh_indices[:,0],data_mesh_indices[:,1],xlab="",ylab="",main="",cex=0.5)
#r.plot(data_locs[:,0],data_locs[:,1],xlab="",ylab="",main="",cex=0.5)
#from IPython.Debugger import Pdb
#Pdb(color_scheme='Linux').set_trace()
thin_grids = tuple([grid[:2]+(grid[2]/thinning,) for grid in grids])
thin_grid_shape = tuple([thin_grid[2] for thin_grid in thin_grids])
thin_axes = tuple([np.linspace(*thin_grid) for thin_grid in thin_grids])
mapgrid = np.array(mgrid[0:grid_shape[0],0:grid_shape[1]], dtype=float)
for i in xrange(2): normalize_for_mapcoords(mapgrid[i], thin_grid_shape[i]-1)
thin_mapgrid = np.array(mgrid[0:thin_grid_shape[0], 0:thin_grid_shape[1]], dtype=float)
for i in xrange(2): normalize_for_mapcoords(thin_mapgrid[i], grid_shape[i]-1)
def thin_to_full(thin_row):
return ndimage.map_coordinates(thin_row, mapgrid)
def full_to_thin(row):
return ndimage.map_coordinates(row, thin_mapgrid)
thin_mask = np.array(np.round(full_to_thin(mask)), dtype='bool')
# Container for x
thin_x = np.empty(thin_grid_shape[:2] + (3,))
mlon,mlat = np.meshgrid(*thin_axes[:2])
thin_x[:,:,0] = mlon.T
thin_x[:,:,1] = mlat.T
thin_x[:,:,2] = 0
x = np.empty(grid_shape[:2] + (3,))
mlon,mlat = np.meshgrid(*axes[:2])
x[:,:,0] = mlon.T
x[:,:,1] = mlat.T
x[:,:,2] = 0
del mlon, mlat
Cp = C.params
# Prepare input dictionaries
covParamObj = {'Scale': Cp['scale'][0],
'amp': Cp['amp'][0],
'inc': Cp['inc'][0],
'ecc': Cp['ecc'][0],
't.lim.corr': Cp['tlc'][0],
'scale.t': Cp['st'][0],
'sin.frac': Cp['sf'][0]}
gridParamObj = {'YLLCORNER': grids[1][0]*rad_to_deg,
'CELLSIZE': (grids[1][1]-grids[1][0])/(grids[1][2]-1.)*rad_to_deg,
'NROWS':grid_shape[1],
'NCOLS':grid_shape[0]}
monthParamObj = {'Nmonths':grid_shape[2],'StartMonth':grids[2][0]}
################################~TEMP
# Call R preprocessing function and check to make sure no screwy re-casting has taken place.
#t1 = time.time()
#os.chdir(r_path)
#preLoopObj = r.CONDSIMpreloop(covParamObj,gridParamObj,monthParamObj,indices.min(), indices.max(),paramfileINDEX)
#tree_reader = reader(file('listSummary_preLoopObj_original_%i_%i.txt'%(indices.min(), indices.max())),delimiter=' ')
#preLoopClassTree, junk = parse_tree(tree_reader)
#preLoopObj = compare_tree(preLoopObj, preLoopClassTree)
#OutMATlist = preLoopObj['OutMATlist']
#tree_reader = reader(file('listSummary_OutMATlist_original_%i_%i.txt'%(indices.min(), indices.max())),delimiter=' ')
#OutMATClassTree, junk = parse_tree(tree_reader)
#OutMATlist = compare_tree(OutMATlist, OutMATClassTree)
#os.chdir(curpath)
#preLoop_time = time.time()-t1
#print "preLoop_time :"+str(preLoop_time)
## Create and store unconditional realizations
#print '\tGenerating unconditional realizations.'
#t1 = time.time()
#for i in xrange(grid_shape[2]):
# print 'On month :'+str(i)
# #print 'OutMATlist:'
# #print OutMATlist
# os.chdir(r_path)
# monthObject = r.CONDSIMmonthloop(i+1,preLoopObj,OutMATlist, indices.min(), indices.max(),paramfileINDEX)
# #monthObject = r.CONDSIMmonthloop(i+1,preLoopObj,OutMATlist,paramfileINDEX)
# os.chdir(curpath)
# OutMATlist= monthObject['OutMATlist']
# MonthGrid = monthObject['MonthGrid']
# out_arr[real_index,:,:,i] = MonthGrid[:grid_shape[1],:grid_shape[0]]
#t2 = time.time()
#print '\t\tDone in %f'%(t2-t1)
#print "monthloop_time :"+str(t2-t1)+" for "+str(grid_shape[2])+" months"
# delete unneeded R products
#del OutMATlist, preLoopObj, MonthGrid, monthObject
################################~TEMP
################################~TEMP DIRECTLY JOIN SIMULATE UNCODITIONED BLOCK FOR TESTING
getUnconditionedBlock(out_arr,real_index,grids,C_straightfromtrace,NinThinnedBlock=None,relp=None,FULLRANK=False)
print 'variance of unconditioned block = '+str(round(np.var(out_arr),10))
print 'variance of unconditioned block month 6 = '+str(round(np.var(out_arr[:,:,:,6]),10))
examineRealization(outfile_root,0,6,15,None,None,conditioned=False,flipVertical="FALSE",SPACE=True,TIME=True)
################################~TEMP
# Figure out pdata
pdata = np.empty(tdata.shape)
for i in xrange(len(where_in)):
pdata[where_in[i]] = out_arr[real_index, grid_shape[1]-1-data_mesh_indices[i,1], data_mesh_indices[i,0], data_mesh_indices[i,2]]
# jointly simulate at data points conditional on block
## first get XYZT list of locations of a regular thinned sample from block
#array3d = out_arr[real_index,:,:,:]
ThinnedBlockXYTZlists = getThinnedBlockXYTZlists (out_arr,real_index,grids,NinThinnedBlock)
xyt_in = ThinnedBlockXYTZlists['xyt_in']
z_in = ThinnedBlockXYTZlists['z_in']
## get locations of data outside block (referenced by grid location)
#XYT_out_gridlocs = data_mesh_indices[where_out]
#
### convert these grid lcoations into actual position in radians and months
#coordsDict = gridParams_2_XYTmarginallists(grids)
#xcoords = coordsDict['xcoords']
#ycoords = coordsDict['ycoords']
#tcoords = coordsDict['tcoords']
#
#x_out = xcoords[XYT_out_gridlocs[:,0]]
#y_out = ycoords[XYT_out_gridlocs[:,1]]
#t_out = tcoords[XYT_out_gridlocs[:,2]]
#
#xyt_out = np.vstack((x_out,y_out,t_out)).T
# get locations of data outside block
xyt_out = data_locs[where_out]
# now we have locations and values of thinned sample from the block, and locations we want to predict at outside the block,go ahead and
# get simulated values of the latter, conditonal on the former
print '\tsimulating over '+str(len(xyt_out[:,0]))+' locations outside block using thinned block sample of '+str(len(z_in))+' points'
t1=time.time()
z_out = predictPointsFromBlock(xyt_in,z_in, xyt_out,C_straightfromtrace,relp)
print '\ttime for simulation: '+str(time.time()-t1)
#########################################CHECK COVARIANCE STRUCTURE
#r.X11(width=12,height=12)
#r.par(mfrow=(3,2))
# test marginal space and time covariance structures of points outside block
#cfdict_out = getEmpiricalCovarianceFunction_STmarginals(xyt_out,z_out,mu=0,margTol_S=0.05,margTol_T=0.9/12,nbins=20, cutoff = 0.8)
#plotEmpiricalCovarianceFunction(cfdict_out['space'],CovModelObj=C_straightfromtrace,spaceORtime="space", cutoff = 0.8, title="Points outside (S) "+str(paramfileINDEX))
#plotEmpiricalCovarianceFunction(cfdict_out['time'],CovModelObj=C_straightfromtrace,spaceORtime="time", cutoff = 0.8, title="Points outside (T)"+str(paramfileINDEX))
# test marginal space and time covariance structures of points inside block
#cfdict_in = getEmpiricalCovarianceFunction_STmarginals(xyt_in,z_in,mu=0,margTol_S=0.05,margTol_T=0.9/12,nbins=20, cutoff = 0.8)
#plotEmpiricalCovarianceFunction(cfdict_in['space'],CovModelObj=C_straightfromtrace,spaceORtime="space", cutoff = 0.8, title="Points inside (S)"+str(paramfileINDEX))
#plotEmpiricalCovarianceFunction(cfdict_in['time'],CovModelObj=C_straightfromtrace,spaceORtime="time", cutoff = 0.8, title="Points inside (T)"+str(paramfileINDEX))
# test marginal space and time covariance structures of points inside and outside block
#cfdict_inout = getEmpiricalCovarianceFunction_STmarginals(np.vstack((xyt_in,xyt_out)),np.hstack((z_in,z_out)),mu=0,margTol_S=0.05,margTol_T=0.9/12,nbins=20, cutoff = 0.8)
#plotEmpiricalCovarianceFunction(cfdict_inout['space'],CovModelObj=C_straightfromtrace,spaceORtime="space", cutoff = 0.8, title="Points inside (S)"+str(paramfileINDEX))
#plotEmpiricalCovarianceFunction(cfdict_inout['time'],CovModelObj=C_straightfromtrace,spaceORtime="time", cutoff = 0.8, title="Points inside (T)"+str(paramfileINDEX))
#########################################CHECK COVARIANCE STRUCTURE
# assign these values to pdata
pdata[where_out] = z_out
###############################~~TEMP
#return()
#####################################
# Bring in data.
print '\tKriging to bring in data.'
print '\tPreprocessing.'
t1 = time.time()
dev_posdef, xbi, ybi, dl_posdef = preprocess(C, data_locs, thin_grids, thin_x, n_blocks_x, n_blocks_y, tdata, pdata, relp, mean_ondata)
t2 = time.time()
print '\t\tDone in %f'%(t2-t1)
###############################~~TEMP
#from IPython.Debugger import Pdb
#Pdb(color_scheme='Linux').set_trace()
#####################################
thin_row = np.empty(thin_grid_shape[:2], dtype=np.float32)
print '\tKriging.'
t1 = time.time()
C1 = pm.gp.NearlyFullRankCovariance(C_straightfromtrace.eval_fun, **C_straightfromtrace.params)
C2 = pm.gp.NearlyFullRankCovariance(C_straightfromtrace.eval_fun, **C_straightfromtrace.params)
M1 = pm.gp.Mean(lambda x: np.zeros(x.shape[:-1]))
M2 = pm.gp.Mean(lambda x: np.zeros(x.shape[:-1]))
cholfac = C1.cholesky(data_locs, apply_pivot=False)
U = cholfac['U']
piv = cholfac['pivots']
m = U.shape[0]
w = np.linalg.solve(U[:m,:m],np.linalg.solve(U[:m,:m].T,(tdata-mean_ondata)[piv[:m]]))
pm.gp.observe(M2,C2,data_locs,pdata)
for i in xrange(grid_shape[2]-1,-1,-1):
thin_row.fill(0.)
thin_x[:,:,2] = axes[2][i]
x[:,:,2] = axes[2][i]
simkrige = M2(x)
dkrige = covariate_mesh + M(x) + np.dot(C1(x,data_locs[piv[:m]]).view(ndarray), w).reshape(x.shape[:-1])
simsurf = grid_convert(out_arr[real_index,:,:,i], 'y-x+', 'x+y+')
import pylab as pl
pl.imshow(simsurf)
pl.colorbar()
pl.title('Simulated')
pl.figure()
pl.imshow(simkrige)
pl.colorbar()
pl.title('Sim krige')
pl.figure()
pl.imshow(simsurf-simkrige)
pl.colorbar()
pl.title('Adjusted sim')
pl.figure()
pl.imshow(dkrige)
pl.colorbar()
pl.title('Data krige')
pl.figure()
pl.imshow(simsurf-simkrige+dkrige)
pl.colorbar()
pl.title('Kriged sim')
krige_month(C, i, dl_posdef, thin_grid_shape, n_blocks_x, n_blocks_y, xbi, ybi, thin_x, dev_posdef, thin_row, thin_mask)
row = ndimage.map_coordinates(thin_row, mapgrid)
row += covariate_mesh
row += M(x)
row += grid_convert(out_arr[real_index,:,:,i], 'y-x+', 'x+y+')
pl.figure()
pl.imshow(row)
pl.colorbar()
pl.title('Original method')
from IPython.Debugger import Pdb
Pdb(color_scheme='Linux').set_trace()
# NaN the oceans to save storage
row[np.where(1-mask)] = missing_val
out_arr[real_index,:,:,i] = grid_convert(row, 'x+y+','y-x+')
####################################TEMP
print 'variance of conditioned block month 6 = '+str(round(np.var(out_arr[:,:,:,6]),10))
print 'variance of conditioned block = '+str(round(np.var(out_arr),10))
examineRealization(outfile_root,0,6,15,None,None,conditioned=True,flipVertical="FALSE",SPACE=True,TIME=True)
########################################
print '\t\tDone in %f'%(time.time()-t1)
modis_res = (21600, 43200)
lon = np.linspace(-180,180,modis_res[1])
lat = np.linspace(-90,90,modis_res[0])
af_lon, af_lat, af_data, af_type = map_utils.import_raster('africa','.')
lon_min, lon_max, lat_min, lat_max = af_lon.min(), af_lon.max(), af_lat.min(), af_lat.max()
pete_lon, pete_lat, pete_mask, junk = map_utils.import_raster('ls','.')
# Subset the rasters
lon_min_i, lon_max_i = ((np.array([lon_min, lon_max])+180.)/360.*modis_res[1]).astype('int')
lat_min_i, lat_max_i = ((np.array([lat_min, lat_max])+90.)/180.*modis_res[0]).astype('int')
pete_lat_min_i = np.argmin(np.abs(pete_lat - af_lat.min()))
pete_lat_max_i = np.argmin(np.abs(pete_lat - af_lat.max()))
clipped_pete_mask = grid_convert(grid_convert(pete_mask,'y-x+','x+y+')[lon_min_i/5.:lon_max_i/5.,pete_lat_min_i:pete_lat_max_i],'x+y+','y-x+')
af_lon = af_lon[:-1]
af_lat = af_lat[:-1]
af_data = af_data[:-1,:-1]
<<<<<<< Updated upstream
af_data.mask = af_data.mask + clipped_pete_mask.mask
map_utils.export_raster(af_lon,af_lat,af_data,'africa','5k-covariates','flt')
=======
af_data.mask = af_data.mask + clipped_pete_mask
map_utils.export_raster(af_lon,af_lat,af_data,'africa','5k-covariates','hdf5')
>>>>>>> Stashed changes
def subset_and_writeout(hf_in, fname, thin, maskval, binfn=lambda x:x):
print 'Subsetting for %s'%fname
