def subset_and_writeout(hf_in, fname, thin, maskval, binfn=lambda x:x): print 'Subsetting for %s'%fname res=5 hf_out = tb.openFile(os.path.join('5k-covariates',fname.replace('-','_').replace('.','_')+'.hdf5'),'w') hf_out.createArray('/','lon',lon[lon_min_i:lon_max_i:res]) hf_out.createArray('/','lat',lat[lat_min_i:lat_max_i:res]) d = hf_in.root.data[(hf_in.root.data.shape[0]-lat_max_i*thin):\ (hf_in.root.data.shape[0]-lat_min_i*thin):\ thin, lon_min_i*thin:\ lon_max_i*thin:\ thin] d = map_utils.grid_convert(map_utils.grid_convert(d,'y-x+','x+y+')[::res,::res], 'x+y+','y-x+') hf_out.createCArray('/','data',atom=tb.FloatAtom(),shape=d.shape,filters=tb.Filters(complevel=1,complib='zlib')) hf_out.createCArray('/','mask',atom=tb.BoolAtom(),shape=d.shape,filters=tb.Filters(complevel=1,complib='zlib')) hf_out.root.data.attrs.view = 'y-x+' hf_out.root.data[:]=binfn(d) hf_out.root.mask[:] = (d==maskval)+clipped_pete_mask hf_out.close()
def test_reconciliation(self): "Tests that reconciling multiple rasters works correctly for multiple views." lims = {} ns = {} hfs = {} views = [] for order in ['xy','yx']: for first in ['+','-']: for second in ['+','-']: view = order[0]+first+order[1]+second views.append(view) lim_lo = np.random.uniform(-1,-.5,size=2) lim_hi = np.random.uniform(.5,1,size=2) lim = [lim_lo[0], lim_hi[0], lim_lo[1], lim_hi[1]] n = np.random.randint(200,500,size=2) lims[view]=np.array(lim)*180./np.pi ns[view]=n x = np.linspace(lim[0],lim[1],n[0]) y = np.linspace(lim[2],lim[3],n[1]) print view fname, hf = write_datafile(x,y,view=view) hfs[view] = hf lon, lat, layers = reconcile_multiple_rasters([hfs[v].root for v in views], thin=2) # Check that the layers all have the right shape assert(np.all([(l.shape == (len(lon),len(lat))) for l in layers])) # Check that the limits are inside the joint intersection assert(np.all([(lon.min() >= l[0]) * (lon.max() <= l[1]) * (lat.min() >= l[2]) * (lat.max() <= l[3]) for l in lims.itervalues()])) # Check that the coarseness is maximal assert([(lims[v][1]-lims[v][0])/float(ns[v][0]-1) <= lon[1]-lon[0] for v in views]) assert([(lims[v][2]-lims[v][3])/float(ns[v][1]-1) <= lat[1]-lat[0] for v in views]) lonmin = np.min([l[0] for l in lims.itervalues()]) lonmax = np.max([l[1] for l in lims.itervalues()]) latmin = np.min([l[2] for l in lims.itervalues()]) latmax = np.max([l[3] for l in lims.itervalues()]) import pylab as pl for i in xrange(len(views)): pl.figure(figsize=(10,6)) pl.subplot(1,2,2) pl.imshow(grid_convert(layers[i],'x+y+','y+x+'),extent=[lon.min(),lon.max(),lat.min(),lat.max()]) pl.title(views[i] + ': ' + str(hfs[views[i]].root.data.shape)) pl.axis([lonmin, lonmax, latmin, latmax]) pl.subplot(1,2,1) pl.imshow(grid_convert(hfs[views[i]].root.data[:], views[i], 'y+x+'), extent = lims[views[i]]) pl.plot([lon.min(), lon.max(), lon.max(), lon.min()],[lat.min(), lat.min(), lat.max(), lat.max()],'r-') pl.axis([lonmin, lonmax, latmin, latmax])
def write_datafile(x,y,t=0,sc=1,view='x+y+'): hf = tb.openFile(os.path.join(data_dirname, 'test_extraction_%s.hdf5'%view),'w') hf.createArray('/','lon',x*180./np.pi) hf.createArray('/','lat',y*180./np.pi) d = grid_convert(f(x,y,t,sc), 'x+y+', view) hf.createCArray('/','data',shape=d.shape,chunkshape=(10,10),atom=tb.FloatAtom()) hf.root.data[:] = d hf.root.data.attrs.view = view hf.close() return 'test_extraction_%s'%view, tb.openFile(os.path.join(data_dirname, 'test_extraction_%s.hdf5'%view))
def targ(inner, continent, fname, missing_val, t_start, t_end, t_chunk, chunk_str): grid_lims = getattr(master_grid, continent + '_lims') xllc_here = (xllc + cellsize * (grid_lims['leftCol'] - 1)) xurc_here = (xllc + cellsize * (grid_lims['rightCol'] - 1)) yllc_here = (yllc + cellsize * (nrows - grid_lims['bottomRow'])) yurc_here = (yllc + cellsize * (nrows - grid_lims['topRow'])) outer = 0 hf = openFile(fname) r = hf.root.realizations V = hf.root.PyMCsamples.cols.V[:] sl = np.empty((r.shape[1], r.shape[2], t_chunk)) sh = sl.shape t = hf.root.t_axis[inner] + 2009 mo = int(round(rem(t * 12, 12))) yr = int(t) time_str = moname[mo] + ' %i' % yr for k in xrange(t_chunk): sl[:, :, k] = r[outer, :, :, inner + k] sl[sl == missing_val] = NaN out = 0 out = sl + np.random.normal(size=sh) * np.sqrt(V[outer]) out = invlogit(out.ravel()).reshape(sh) out = np.mean(out, axis=2) out = ma.masked_array(out, isnan(out)) b = basemap.Basemap(llcrnrlon=xllc_here, llcrnrlat=yllc_here, urcrnrlon=xurc_here, urcrnrlat=yurc_here) b.imshow(grid_convert(out, 'y-x+', 'x+y+'), cmap=matplotlib.cm.hot) # b.drawcoastlines(color='.8',linewidth='2') b.drawcountries(color='.8', linewidth='1') colorbar() # axis('off') # title('%s starting %s'%(chunk_str, time_str)) title(str(yr)) savefig('anim-scratch/%i.png' % (inner)) close('all') del sl, out gc.collect() hf.close()
def targ(inner,continent,fname,missing_val,t_start,t_end,t_chunk,chunk_str): grid_lims = getattr(master_grid, continent+'_lims') xllc_here = (xllc + cellsize * (grid_lims['leftCol']-1)) xurc_here = (xllc + cellsize * (grid_lims['rightCol']-1)) yllc_here = (yllc + cellsize * (nrows-grid_lims['bottomRow'])) yurc_here = (yllc + cellsize * (nrows-grid_lims['topRow'])) outer=0 hf = openFile(fname) r = hf.root.realizations V = hf.root.PyMCsamples.cols.V[:] sl = np.empty((r.shape[1], r.shape[2], t_chunk)) sh = sl.shape t = hf.root.t_axis[inner] + 2009 mo = int(round(rem(t*12,12))) yr = int(t) time_str = moname[mo] + ' %i'%yr for k in xrange(t_chunk): sl[:,:,k] = r[outer,:,:,inner + k] sl[sl==missing_val]=NaN out = 0 out = sl + np.random.normal(size=sh)*np.sqrt(V[outer]) out = invlogit(out.ravel()).reshape(sh) out = np.mean(out, axis=2) out = ma.masked_array(out, isnan(out)) b = basemap.Basemap(llcrnrlon=xllc_here, llcrnrlat=yllc_here, urcrnrlon=xurc_here, urcrnrlat=yurc_here) b.imshow(grid_convert(out,'y-x+','x+y+'), cmap=matplotlib.cm.hot) # b.drawcoastlines(color='.8',linewidth='2') b.drawcountries(color='.8',linewidth='1') colorbar() # axis('off') # title('%s starting %s'%(chunk_str, time_str)) title(str(yr)) savefig('anim-scratch/%i.png'%(inner)) close('all') del sl, out gc.collect() hf.close()
def current_state_map(M, session, species, mask, x, img_extent, thin=1, f2p=None, **kwds): "Maps the current state of the model." out = pm.utils.value(M.p)(x, f2p=f2p) b = basemap.Basemap(*img_extent) arr = np.ma.masked_array(out, mask=True-mask) b.imshow(grid_convert(arr,'x+y+','y+x+'), interpolation='nearest') # b.plot(pm.value(M.x_fr)[:,0]*180./np.pi, pm.value(M.x_fr)[:,1]*180./np.pi, 'g.', markersize=2) # b.drawcoastlines(color=(.9,.4,.5)) # pl.colorbar() return out, arr
def clip_raster(geom, lon, lat, view="y+x+"): """ Returns an array in the desired view indicating which pixels in the meshgrid generated from lon and lat are inside geom. """ # Internal view is x+y+. isin = np.zeros((len(lon), len(lat)), dtype="bool") if isinstance(geom, shapely.geometry.multipolygon.MultiPolygon): geoms = geom.geoms else: geoms = [geom] for i, g in enumerate(geoms): clip_raster_to_polygon(g, lon, lat, isin) return map_utils.grid_convert(isin, "x+y+", view)
def get_covariate_submesh(name, grid_lims): """ Matches the specified spatial mesh to the 'master' covariate mesh. """ try: order = getattr(mbgw.auxiliary_data, name)._v_attrs.order except: order = 'y-x+' raw_shape = getattr(mbgw.auxiliary_data, name).data.shape raw = getattr(mbgw.auxiliary_data, name).data[grid_lims['topRow']-1:grid_lims['bottomRow'], grid_lims['leftCol']-1:grid_lims['rightCol']] out = grid_convert(raw, order, 'x+y+').copy() targ_shape = (grid_lims['rightCol']-grid_lims['leftCol']+1, grid_lims['bottomRow']-grid_lims['topRow']+1) if out.shape != targ_shape: raise ValueError, "Grid %s's shape is %s with order %s. Cannot be sliced to shape %s. \n\tGrid limits: %s\n\tnrows: %i\n\tncols: %i"\ %(name, raw_shape, order, targ_shape, grid_lims, nrows, ncols) return out
def test_extraction(self): "Tests that extracting environmental layers works correctly for multiple views." for order in ['xy','yx']: for first in ['+','-']: for second in ['+','-']: view = order[0]+first+order[1]+second fname, hf = write_datafile(x,y,view=view) x_ind = np.random.randint(len(x)-2,size=n)+1 y_ind = np.random.randint(len(y)-2,size=n)+1 x_extract = x[x_ind]*180./np.pi + np.random.normal()*.001 y_extract = y[y_ind]*180./np.pi + np.random.normal()*.001 e2 = grid_convert(hf.root.data[:], view, 'x+y+')[(x_ind,y_ind)] hf.close() e1 = extract_environment(fname, np.vstack((x_extract,y_extract)).T, cache=False) assert_almost_equal(e1,e2,decimal=3)
def presence_map(M, session, species, burn=0, thin=1, trace_thin=1, **kwds): "Converts the trace to a map of presence probability." from mpl_toolkits import basemap import pylab as pl import time chain_len = len(M.db._h5file.root.chain0.PyMCsamples) mask, x, img_extent = make_covering_raster(thin, M.env_variables, **kwds) out = np.zeros(mask.shape) time_count = -np.inf time_start = time.time() ptrace = M.trace('p')[:] for i in xrange(burn, len(M.trace('p_find')[:]), trace_thin): if time.time() - time_count > 10: print (((i-burn)*100)/(chain_len)), '% complete', if i>burn: time_count = time.time() print 'expect results '+time.ctime((time_count-time_start)*(chain_len-burn)/float(i-burn)+time_start) else: print p = ptrace[i] pe = p(x) out += pe/float(chain_len-burn) b = basemap.Basemap(*img_extent) arr = np.ma.masked_array(out, mask=True-mask) b.imshow(grid_convert(arr,'x+y+','y+x+'), interpolation='nearest') # pl.colorbar() plot_species(session, species[0], species[1], b, negs=True, **kwds) return out, arr
def get_pseudoabsences(eo, buffer_width, n_pseudoabsences, layer_names, glob_name, eo_expand=0): if eo_expand: eo = lomem_buffer(eo, eo_expand) fname = hashlib.sha1(geojson.dumps(eo)+'_'+str(buffer_width)+'_'+str(n_pseudoabsences)).hexdigest()+'.npy' if fname in os.listdir('anopheles-caches'): pseudoabsences = np.load(os.path.join('anopheles-caches', fname)) else: if buffer_width >= 0: buff = lomem_buffer(eo, buffer_width) diff_buffer = buff.difference(eo) elif buffer_width == -1: diff_buffer = eo else: raise ValueError, 'Buffer width is negative, but not -1.' if len(layer_names)>0: template = layer_names[0] else: template = glob_name lon, lat, test_raster, rtype = map_utils.import_raster(*os.path.split(template)[::-1]) # Right the raster test_raster = map_utils.grid_convert(test_raster,'y-x+','x+y+') # Convert lower-left coords to centroids lon += (lon[1]-lon[0])/2. lat += (lat[1]-lat[0])/2. def testfn(lon_test,lat_test,r=test_raster,lon=lon,lat=lat): lon_ind = np.argmin(np.abs(lon-lon_test)) lat_ind = np.argmin(np.abs(lat-lat_test)) return True-test_raster.mask[lon_ind,lat_ind] pseudoabsences = np.vstack(map_utils.shapefile_utils.multipoly_sample(n_pseudoabsences, diff_buffer, test=testfn)).T if not np.all([testfn(l1,l2) for l1,l2 in pseudoabsences]): raise ValueError, 'Test failed for some pseudoabsences.' np.save(os.path.join('anopheles-caches',fname), pseudoabsences) return pseudoabsences, eo
def get_covariate_submesh(name, grid_lims): """ Matches the specified spatial mesh to the 'master' covariate mesh. """ try: order = getattr(mbgw.auxiliary_data, name)._v_attrs.order except: order = 'y-x+' raw_shape = getattr(mbgw.auxiliary_data, name).data.shape raw = getattr(mbgw.auxiliary_data, name).data[grid_lims['topRow'] - 1:grid_lims['bottomRow'], grid_lims['leftCol'] - 1:grid_lims['rightCol']] out = grid_convert(raw, order, 'x+y+').copy() targ_shape = (grid_lims['rightCol'] - grid_lims['leftCol'] + 1, grid_lims['bottomRow'] - grid_lims['topRow'] + 1) if out.shape != targ_shape: raise ValueError, "Grid %s's shape is %s with order %s. Cannot be sliced to shape %s. \n\tGrid limits: %s\n\tnrows: %i\n\tncols: %i"\ %(name, raw_shape, order, targ_shape, grid_lims, nrows, ncols) return out
def subset_raster(r, llclati, llcloni, urclati, urcloni): r_ = map_utils.grid_convert(r,'y-x+','x+y+') return map_utils.grid_convert(r_[llcloni:urcloni,llclati:urclati],'x+y+','y-x+').astype('float32')
def create_realization(outfile_root,real_index, C,C_straightfromtrace, mean_ondata, M, covariate_mesh, tdata, data_locs, grids, axes, data_mesh_indices, where_in, where_out, n_blocks_x, n_blocks_y, relp, mask, thinning,indices,paramfileINDEX,NinThinnedBlock,TESTRANGE,TESTSQUARE): print '\nON REALIZATION '+str(real_index)+'\n' # define only realizations chunk of hdf5 realization file out_arr = outfile_root.realizations """ Creates a single realization from the predictive distribution over specified space-time mesh. """ grid_shape = tuple([grid[2] for grid in grids]) #r.X11(width=8,height=4) #r.par(mfrow=(1,2)) #r.plot(data_mesh_indices[:,0],data_mesh_indices[:,1],xlab="",ylab="",main="",cex=0.5) #r.plot(data_locs[:,0],data_locs[:,1],xlab="",ylab="",main="",cex=0.5) #from IPython.Debugger import Pdb #Pdb(color_scheme='Linux').set_trace() thin_grids = tuple([grid[:2]+(grid[2]/thinning,) for grid in grids]) thin_grid_shape = tuple([thin_grid[2] for thin_grid in thin_grids]) thin_axes = tuple([np.linspace(*thin_grid) for thin_grid in thin_grids]) mapgrid = np.array(mgrid[0:grid_shape[0],0:grid_shape[1]], dtype=float) for i in xrange(2): normalize_for_mapcoords(mapgrid[i], thin_grid_shape[i]-1) thin_mapgrid = np.array(mgrid[0:thin_grid_shape[0], 0:thin_grid_shape[1]], dtype=float) for i in xrange(2): normalize_for_mapcoords(thin_mapgrid[i], grid_shape[i]-1) def thin_to_full(thin_row): return ndimage.map_coordinates(thin_row, mapgrid) def full_to_thin(row): return ndimage.map_coordinates(row, thin_mapgrid) thin_mask = np.array(np.round(full_to_thin(mask)), dtype='bool') # Container for x thin_x = np.empty(thin_grid_shape[:2] + (3,)) mlon,mlat = np.meshgrid(*thin_axes[:2]) thin_x[:,:,0] = mlon.T thin_x[:,:,1] = mlat.T thin_x[:,:,2] = 0 x = np.empty(grid_shape[:2] + (3,)) mlon,mlat = np.meshgrid(*axes[:2]) x[:,:,0] = mlon.T x[:,:,1] = mlat.T x[:,:,2] = 0 del mlon, mlat Cp = C.params # In special case (america region) run only a test square using direct simulation, if it does not fail then just return if TESTSQUARE: getUnconditionedBlock(out_arr,real_index,grids,C_straightfromtrace,NinThinnedBlock=None,relp=relp,FULLRANK=False) return() # Prepare input dictionaries covParamObj = {'Scale': Cp['scale'][0], 'amp': Cp['amp'][0], 'inc': Cp['inc'][0], 'ecc': Cp['ecc'][0], 't.lim.corr': Cp['tlc'][0], 'scale.t': Cp['st'][0], 'sin.frac': Cp['sf'][0]} gridParamObj = {'YLLCORNER': grids[1][0]*rad_to_deg, 'CELLSIZE': (grids[1][1]-grids[1][0])/(grids[1][2]-1.)*rad_to_deg, 'NROWS':grid_shape[1], 'NCOLS':grid_shape[0]} monthParamObj = {'Nmonths':grid_shape[2],'StartMonth':grids[2][0]} # Call R preprocessing function and check to make sure no screwy re-casting has taken place. t1 = time.time() os.chdir(r_path) preLoopObj = r.CONDSIMpreloop(covParamObj,gridParamObj,monthParamObj,indices.min(), indices.max(),paramfileINDEX) tree_reader = reader(file('listSummary_preLoopObj_original_%i_%i.txt'%(indices.min(), indices.max())),delimiter=' ') preLoopClassTree, junk = parse_tree(tree_reader) preLoopObj = compare_tree(preLoopObj, preLoopClassTree) OutMATlist = preLoopObj['OutMATlist'] tree_reader = reader(file('listSummary_OutMATlist_original_%i_%i.txt'%(indices.min(), indices.max())),delimiter=' ') OutMATClassTree, junk = parse_tree(tree_reader) OutMATlist = compare_tree(OutMATlist, OutMATClassTree) os.chdir(curpath) preLoop_time = time.time()-t1 print "preLoop_time :"+str(preLoop_time) #from IPython.Debugger import Pdb #Pdb(color_scheme='Linux').set_trace() ## Create and store unconditional realizations print '\tGenerating unconditional realizations.' t1 = time.time() for i in xrange(grid_shape[2]): print 'On month :'+str(i) #print 'OutMATlist:' #print OutMATlist os.chdir(r_path) monthObject = r.CONDSIMmonthloop(i+1,preLoopObj,OutMATlist, indices.min(), indices.max(),paramfileINDEX) #monthObject = r.CONDSIMmonthloop(i+1,preLoopObj,OutMATlist,paramfileINDEX) os.chdir(curpath) OutMATlist= monthObject['OutMATlist'] MonthGrid = monthObject['MonthGrid'] out_arr[real_index,:,:,i] = MonthGrid[:grid_shape[1],:grid_shape[0]] t2 = time.time() print '\t\tDone in %f'%(t2-t1) print "monthloop_time :"+str(t2-t1)+" for "+str(grid_shape[2])+" months" # delete unneeded R products del OutMATlist, preLoopObj, MonthGrid, monthObject #############################~TEMP # ################################~TEMP DIRECTLY JOIN SIMULATE UNCODITIONED BLOCK FOR TESTING # getUnconditionedBlock(out_arr,real_index,grids,C_straightfromtrace,NinThinnedBlock=None,relp=relp,FULLRANK=False) # #print 'variance of unconditioned block = '+str(round(np.var(out_arr),10)) # #print 'variance of unconditioned block month 6 = '+str(round(np.var(out_arr[:,:,:,6]),10)) # #examineRealization(outfile_root,real_index,6,15,None,None,conditioned=False,flipVertical="FALSE",SPACE=True,TIME=True) # ################################~TEMP # Figure out pdata pdata = np.empty(tdata.shape) for i in xrange(len(where_in)): index = where_in[i] pdata[index] = out_arr[real_index, grid_shape[1]-1-data_mesh_indices[index,1], data_mesh_indices[index,0], data_mesh_indices[index,2]] # jointly simulate at data points conditional on block ## first get XYZT list of locations of a regular thinned sample from block #array3d = out_arr[real_index,:,:,:] ThinnedBlockXYTZlists = getThinnedBlockXYTZlists (out_arr,real_index,grids,NinThinnedBlock) xyt_in = ThinnedBlockXYTZlists['xyt_in'] z_in = ThinnedBlockXYTZlists['z_in'] # get locations of data outside block xyt_out = data_locs[where_out] # now we have locations and values of thinned sample from the block, and locations we want to predict at outside the block,go ahead and # get simulated values of the latter, conditonal on the former print '\tsimulating over '+str(len(xyt_out[:,0]))+' locations outside block using thinned block sample of '+str(len(z_in))+' points' t1=time.time() z_out = predictPointsFromBlock(xyt_in,z_in, xyt_out,C_straightfromtrace,relp) print '\ttime for simulation: '+str(time.time()-t1) #########################################CHECK COVARIANCE STRUCTURE #r.X11(width=12,height=12) #r.par(mfrow=(3,2)) # test marginal space and time covariance structures of points outside block #cfdict_out = getEmpiricalCovarianceFunction_STmarginals(xyt_out,z_out,mu=0,margTol_S=0.05,margTol_T=0.9/12,nbins=20, cutoff = 0.8) #plotEmpiricalCovarianceFunction(cfdict_out['space'],CovModelObj=C_straightfromtrace,spaceORtime="space", cutoff = 0.8, title="Points outside (S) "+str(paramfileINDEX)) #plotEmpiricalCovarianceFunction(cfdict_out['time'],CovModelObj=C_straightfromtrace,spaceORtime="time", cutoff = 0.8, title="Points outside (T)"+str(paramfileINDEX)) # test marginal space and time covariance structures of points inside block #cfdict_in = getEmpiricalCovarianceFunction_STmarginals(xyt_in,z_in,mu=0,margTol_S=0.05,margTol_T=0.9/12,nbins=20, cutoff = 0.8) #plotEmpiricalCovarianceFunction(cfdict_in['space'],CovModelObj=C_straightfromtrace,spaceORtime="space", cutoff = 0.8, title="Points inside (S)"+str(paramfileINDEX)) #plotEmpiricalCovarianceFunction(cfdict_in['time'],CovModelObj=C_straightfromtrace,spaceORtime="time", cutoff = 0.8, title="Points inside (T)"+str(paramfileINDEX)) # test marginal space and time covariance structures of points inside and outside block #cfdict_inout = getEmpiricalCovarianceFunction_STmarginals(np.vstack((xyt_in,xyt_out)),np.hstack((z_in,z_out)),mu=0,margTol_S=0.05,margTol_T=0.9/12,nbins=20, cutoff = 0.8) #plotEmpiricalCovarianceFunction(cfdict_inout['space'],CovModelObj=C_straightfromtrace,spaceORtime="space", cutoff = 0.8, title="Points inside (S)"+str(paramfileINDEX)) #plotEmpiricalCovarianceFunction(cfdict_inout['time'],CovModelObj=C_straightfromtrace,spaceORtime="time", cutoff = 0.8, title="Points inside (T)"+str(paramfileINDEX)) #########################################CHECK COVARIANCE STRUCTURE # assign these values to pdata pdata[where_out] = z_out ###############################~~TEMP #return() ##################################### # Bring in data. print '\tKriging to bring in data.' print '\tPreprocessing.' t1 = time.time() dev_posdef, xbi, ybi, dl_posdef = preprocess(C, data_locs, thin_grids, thin_x, n_blocks_x, n_blocks_y, tdata, pdata, relp, mean_ondata) t2 = time.time() print '\t\tDone in %f'%(t2-t1) #from IPython.Debugger import Pdb #Pdb(color_scheme='Linux').set_trace() thin_row = np.empty(thin_grid_shape[:2], dtype=np.float32) print '\tKriging.' t1 = time.time() for i in xrange(grid_shape[2]-1,-1,-1): thin_row.fill(0.) thin_x[:,:,2] = axes[2][i] x[:,:,2] = axes[2][i] krige_month(C, i, dl_posdef, thin_grid_shape, n_blocks_x, n_blocks_y, xbi, ybi, thin_x, dev_posdef, thin_row, thin_mask) row = ndimage.map_coordinates(thin_row, mapgrid) row += covariate_mesh row += M(x) row += grid_convert(out_arr[real_index,:,:,i], 'y-x+', 'x+y+') # NaN the oceans to save storage row[np.where(1-mask)] = missing_val # if we are checking for plausible max and min values (Vs f at data), implement tet on conditioned values for this month monthMin = np.min(row[np.where(mask)]) monthMax = np.max(row[np.where(mask)]) pointsMin = np.min(tdata) pointsMax = np.max(tdata) pointsRange = pointsMax-pointsMin threshMin = pointsMin-(pointsRange*TESTRANGE) threshMax = pointsMax+(pointsRange*TESTRANGE) print('On month '+str(i)+' : f range=('+str(pointsMin)+','+str(pointsMax)+') ; month range=('+str(monthMin)+','+str(monthMax)+')') if TESTRANGE!=False: if(((monthMin<threshMin) | (monthMax>threshMax))): raise ValueError ('Killing realization on month '+str(i)+' : f range=('+str(pointsMin)+','+str(pointsMax)+') ; month range=('+str(monthMin)+','+str(monthMax)+')') out_arr[real_index,:,:,i] = grid_convert(row, 'x+y+','y-x+') ####################################TEMP #print 'variance of conditioned block month 6 = '+str(round(np.var(out_arr[:,:,:,6]),10)) #print 'variance of conditioned block = '+str(round(np.var(out_arr),10)) #examineRealization(outfile_root,real_index,6,15,None,None,conditioned=True,flipVertical="FALSE",SPACE=True,TIME=True) ######################################## print '\t\tDone in %f'%(time.time()-t1)
def create_realization(outfile_root, real_index, C, C_straightfromtrace, mean_ondata, M, covariate_mesh, tdata, data_locs, grids, axes, data_mesh_indices, where_in, where_out, n_blocks_x, n_blocks_y, relp, mask, thinning, indices, paramfileINDEX, NinThinnedBlock): # define only realizations chunk of hdf5 realization file out_arr = outfile_root.realizations """ Creates a single realization from the predictive distribution over specified space-time mesh. """ grid_shape = tuple([grid[2] for grid in grids]) #r.X11(width=8,height=4) #r.par(mfrow=(1,2)) #r.plot(data_mesh_indices[:,0],data_mesh_indices[:,1],xlab="",ylab="",main="",cex=0.5) #r.plot(data_locs[:,0],data_locs[:,1],xlab="",ylab="",main="",cex=0.5) #from IPython.Debugger import Pdb #Pdb(color_scheme='Linux').set_trace() thin_grids = tuple([grid[:2] + (grid[2] / thinning, ) for grid in grids]) thin_grid_shape = tuple([thin_grid[2] for thin_grid in thin_grids]) thin_axes = tuple([np.linspace(*thin_grid) for thin_grid in thin_grids]) mapgrid = np.array(mgrid[0:grid_shape[0], 0:grid_shape[1]], dtype=float) for i in xrange(2): normalize_for_mapcoords(mapgrid[i], thin_grid_shape[i] - 1) thin_mapgrid = np.array(mgrid[0:thin_grid_shape[0], 0:thin_grid_shape[1]], dtype=float) for i in xrange(2): normalize_for_mapcoords(thin_mapgrid[i], grid_shape[i] - 1) def thin_to_full(thin_row): return ndimage.map_coordinates(thin_row, mapgrid) def full_to_thin(row): return ndimage.map_coordinates(row, thin_mapgrid) thin_mask = np.array(np.round(full_to_thin(mask)), dtype='bool') # Container for x thin_x = np.empty(thin_grid_shape[:2] + (3, )) mlon, mlat = np.meshgrid(*thin_axes[:2]) thin_x[:, :, 0] = mlon.T thin_x[:, :, 1] = mlat.T thin_x[:, :, 2] = 0 x = np.empty(grid_shape[:2] + (3, )) mlon, mlat = np.meshgrid(*axes[:2]) x[:, :, 0] = mlon.T x[:, :, 1] = mlat.T x[:, :, 2] = 0 del mlon, mlat Cp = C.params # Prepare input dictionaries covParamObj = { 'Scale': Cp['scale'][0], 'amp': Cp['amp'][0], 'inc': Cp['inc'][0], 'ecc': Cp['ecc'][0], 't.lim.corr': Cp['tlc'][0], 'scale.t': Cp['st'][0], 'sin.frac': Cp['sf'][0] } gridParamObj = { 'YLLCORNER': grids[1][0] * rad_to_deg, 'CELLSIZE': (grids[1][1] - grids[1][0]) / (grids[1][2] - 1.) * rad_to_deg, 'NROWS': grid_shape[1], 'NCOLS': grid_shape[0] } monthParamObj = {'Nmonths': grid_shape[2], 'StartMonth': grids[2][0]} ################################~TEMP # Call R preprocessing function and check to make sure no screwy re-casting has taken place. #t1 = time.time() #os.chdir(r_path) #preLoopObj = r.CONDSIMpreloop(covParamObj,gridParamObj,monthParamObj,indices.min(), indices.max(),paramfileINDEX) #tree_reader = reader(file('listSummary_preLoopObj_original_%i_%i.txt'%(indices.min(), indices.max())),delimiter=' ') #preLoopClassTree, junk = parse_tree(tree_reader) #preLoopObj = compare_tree(preLoopObj, preLoopClassTree) #OutMATlist = preLoopObj['OutMATlist'] #tree_reader = reader(file('listSummary_OutMATlist_original_%i_%i.txt'%(indices.min(), indices.max())),delimiter=' ') #OutMATClassTree, junk = parse_tree(tree_reader) #OutMATlist = compare_tree(OutMATlist, OutMATClassTree) #os.chdir(curpath) #preLoop_time = time.time()-t1 #print "preLoop_time :"+str(preLoop_time) ## Create and store unconditional realizations #print '\tGenerating unconditional realizations.' #t1 = time.time() #for i in xrange(grid_shape[2]): # print 'On month :'+str(i) # #print 'OutMATlist:' # #print OutMATlist # os.chdir(r_path) # monthObject = r.CONDSIMmonthloop(i+1,preLoopObj,OutMATlist, indices.min(), indices.max(),paramfileINDEX) # #monthObject = r.CONDSIMmonthloop(i+1,preLoopObj,OutMATlist,paramfileINDEX) # os.chdir(curpath) # OutMATlist= monthObject['OutMATlist'] # MonthGrid = monthObject['MonthGrid'] # out_arr[real_index,:,:,i] = MonthGrid[:grid_shape[1],:grid_shape[0]] #t2 = time.time() #print '\t\tDone in %f'%(t2-t1) #print "monthloop_time :"+str(t2-t1)+" for "+str(grid_shape[2])+" months" # delete unneeded R products #del OutMATlist, preLoopObj, MonthGrid, monthObject ################################~TEMP ################################~TEMP DIRECTLY JOIN SIMULATE UNCODITIONED BLOCK FOR TESTING getUnconditionedBlock(out_arr, real_index, grids, C_straightfromtrace, NinThinnedBlock=None, relp=None, FULLRANK=False) print 'variance of unconditioned block = ' + str(round( np.var(out_arr), 10)) print 'variance of unconditioned block month 6 = ' + str( round(np.var(out_arr[:, :, :, 6]), 10)) examineRealization(outfile_root, 0, 6, 15, None, None, conditioned=False, flipVertical="FALSE", SPACE=True, TIME=True) ################################~TEMP # Figure out pdata pdata = np.empty(tdata.shape) for i in xrange(len(where_in)): pdata[where_in[i]] = out_arr[real_index, grid_shape[1] - 1 - data_mesh_indices[i, 1], data_mesh_indices[i, 0], data_mesh_indices[i, 2]] # jointly simulate at data points conditional on block ## first get XYZT list of locations of a regular thinned sample from block #array3d = out_arr[real_index,:,:,:] ThinnedBlockXYTZlists = getThinnedBlockXYTZlists(out_arr, real_index, grids, NinThinnedBlock) xyt_in = ThinnedBlockXYTZlists['xyt_in'] z_in = ThinnedBlockXYTZlists['z_in'] ## get locations of data outside block (referenced by grid location) #XYT_out_gridlocs = data_mesh_indices[where_out] # ### convert these grid lcoations into actual position in radians and months #coordsDict = gridParams_2_XYTmarginallists(grids) #xcoords = coordsDict['xcoords'] #ycoords = coordsDict['ycoords'] #tcoords = coordsDict['tcoords'] # #x_out = xcoords[XYT_out_gridlocs[:,0]] #y_out = ycoords[XYT_out_gridlocs[:,1]] #t_out = tcoords[XYT_out_gridlocs[:,2]] # #xyt_out = np.vstack((x_out,y_out,t_out)).T # get locations of data outside block xyt_out = data_locs[where_out] # now we have locations and values of thinned sample from the block, and locations we want to predict at outside the block,go ahead and # get simulated values of the latter, conditonal on the former print '\tsimulating over ' + str( len(xyt_out[:, 0]) ) + ' locations outside block using thinned block sample of ' + str( len(z_in)) + ' points' t1 = time.time() z_out = predictPointsFromBlock(xyt_in, z_in, xyt_out, C_straightfromtrace, relp) print '\ttime for simulation: ' + str(time.time() - t1) #########################################CHECK COVARIANCE STRUCTURE #r.X11(width=12,height=12) #r.par(mfrow=(3,2)) # test marginal space and time covariance structures of points outside block #cfdict_out = getEmpiricalCovarianceFunction_STmarginals(xyt_out,z_out,mu=0,margTol_S=0.05,margTol_T=0.9/12,nbins=20, cutoff = 0.8) #plotEmpiricalCovarianceFunction(cfdict_out['space'],CovModelObj=C_straightfromtrace,spaceORtime="space", cutoff = 0.8, title="Points outside (S) "+str(paramfileINDEX)) #plotEmpiricalCovarianceFunction(cfdict_out['time'],CovModelObj=C_straightfromtrace,spaceORtime="time", cutoff = 0.8, title="Points outside (T)"+str(paramfileINDEX)) # test marginal space and time covariance structures of points inside block #cfdict_in = getEmpiricalCovarianceFunction_STmarginals(xyt_in,z_in,mu=0,margTol_S=0.05,margTol_T=0.9/12,nbins=20, cutoff = 0.8) #plotEmpiricalCovarianceFunction(cfdict_in['space'],CovModelObj=C_straightfromtrace,spaceORtime="space", cutoff = 0.8, title="Points inside (S)"+str(paramfileINDEX)) #plotEmpiricalCovarianceFunction(cfdict_in['time'],CovModelObj=C_straightfromtrace,spaceORtime="time", cutoff = 0.8, title="Points inside (T)"+str(paramfileINDEX)) # test marginal space and time covariance structures of points inside and outside block #cfdict_inout = getEmpiricalCovarianceFunction_STmarginals(np.vstack((xyt_in,xyt_out)),np.hstack((z_in,z_out)),mu=0,margTol_S=0.05,margTol_T=0.9/12,nbins=20, cutoff = 0.8) #plotEmpiricalCovarianceFunction(cfdict_inout['space'],CovModelObj=C_straightfromtrace,spaceORtime="space", cutoff = 0.8, title="Points inside (S)"+str(paramfileINDEX)) #plotEmpiricalCovarianceFunction(cfdict_inout['time'],CovModelObj=C_straightfromtrace,spaceORtime="time", cutoff = 0.8, title="Points inside (T)"+str(paramfileINDEX)) #########################################CHECK COVARIANCE STRUCTURE # assign these values to pdata pdata[where_out] = z_out ###############################~~TEMP #return() ##################################### # Bring in data. print '\tKriging to bring in data.' print '\tPreprocessing.' t1 = time.time() dev_posdef, xbi, ybi, dl_posdef = preprocess(C, data_locs, thin_grids, thin_x, n_blocks_x, n_blocks_y, tdata, pdata, relp, mean_ondata) t2 = time.time() print '\t\tDone in %f' % (t2 - t1) ###############################~~TEMP #from IPython.Debugger import Pdb #Pdb(color_scheme='Linux').set_trace() ##################################### thin_row = np.empty(thin_grid_shape[:2], dtype=np.float32) print '\tKriging.' t1 = time.time() C1 = pm.gp.NearlyFullRankCovariance(C_straightfromtrace.eval_fun, **C_straightfromtrace.params) C2 = pm.gp.NearlyFullRankCovariance(C_straightfromtrace.eval_fun, **C_straightfromtrace.params) M1 = pm.gp.Mean(lambda x: np.zeros(x.shape[:-1])) M2 = pm.gp.Mean(lambda x: np.zeros(x.shape[:-1])) cholfac = C1.cholesky(data_locs, apply_pivot=False) U = cholfac['U'] piv = cholfac['pivots'] m = U.shape[0] w = np.linalg.solve( U[:m, :m], np.linalg.solve(U[:m, :m].T, (tdata - mean_ondata)[piv[:m]])) pm.gp.observe(M2, C2, data_locs, pdata) for i in xrange(grid_shape[2] - 1, -1, -1): thin_row.fill(0.) thin_x[:, :, 2] = axes[2][i] x[:, :, 2] = axes[2][i] simkrige = M2(x) dkrige = covariate_mesh + M(x) + np.dot( C1(x, data_locs[piv[:m]]).view(ndarray), w).reshape(x.shape[:-1]) simsurf = grid_convert(out_arr[real_index, :, :, i], 'y-x+', 'x+y+') import pylab as pl pl.imshow(simsurf) pl.colorbar() pl.title('Simulated') pl.figure() pl.imshow(simkrige) pl.colorbar() pl.title('Sim krige') pl.figure() pl.imshow(simsurf - simkrige) pl.colorbar() pl.title('Adjusted sim') pl.figure() pl.imshow(dkrige) pl.colorbar() pl.title('Data krige') pl.figure() pl.imshow(simsurf - simkrige + dkrige) pl.colorbar() pl.title('Kriged sim') krige_month(C, i, dl_posdef, thin_grid_shape, n_blocks_x, n_blocks_y, xbi, ybi, thin_x, dev_posdef, thin_row, thin_mask) row = ndimage.map_coordinates(thin_row, mapgrid) row += covariate_mesh row += M(x) row += grid_convert(out_arr[real_index, :, :, i], 'y-x+', 'x+y+') pl.figure() pl.imshow(row) pl.colorbar() pl.title('Original method') from IPython.Debugger import Pdb Pdb(color_scheme='Linux').set_trace() # NaN the oceans to save storage row[np.where(1 - mask)] = missing_val out_arr[real_index, :, :, i] = grid_convert(row, 'x+y+', 'y-x+') ####################################TEMP print 'variance of conditioned block month 6 = ' + str( round(np.var(out_arr[:, :, :, 6]), 10)) print 'variance of conditioned block = ' + str(round(np.var(out_arr), 10)) examineRealization(outfile_root, 0, 6, 15, None, None, conditioned=True, flipVertical="FALSE", SPACE=True, TIME=True) ######################################## print '\t\tDone in %f' % (time.time() - t1)
def create_realization(outfile_root, real_index, C, C_straightfromtrace, mean_ondata, M, covariate_mesh, tdata, data_locs, grids, axes, data_mesh_indices, where_in, where_out, n_blocks_x, n_blocks_y, relp, mask, thinning, indices, paramfileINDEX, NinThinnedBlock, TESTRANGE, TESTSQUARE): print '\nON REALIZATION ' + str(real_index) + '\n' # define only realizations chunk of hdf5 realization file out_arr = outfile_root.realizations """ Creates a single realization from the predictive distribution over specified space-time mesh. """ grid_shape = tuple([grid[2] for grid in grids]) #r.X11(width=8,height=4) #r.par(mfrow=(1,2)) #r.plot(data_mesh_indices[:,0],data_mesh_indices[:,1],xlab="",ylab="",main="",cex=0.5) #r.plot(data_locs[:,0],data_locs[:,1],xlab="",ylab="",main="",cex=0.5) #from IPython.Debugger import Pdb #Pdb(color_scheme='Linux').set_trace() thin_grids = tuple([grid[:2] + (grid[2] / thinning, ) for grid in grids]) thin_grid_shape = tuple([thin_grid[2] for thin_grid in thin_grids]) thin_axes = tuple([np.linspace(*thin_grid) for thin_grid in thin_grids]) mapgrid = np.array(mgrid[0:grid_shape[0], 0:grid_shape[1]], dtype=float) for i in xrange(2): normalize_for_mapcoords(mapgrid[i], thin_grid_shape[i] - 1) thin_mapgrid = np.array(mgrid[0:thin_grid_shape[0], 0:thin_grid_shape[1]], dtype=float) for i in xrange(2): normalize_for_mapcoords(thin_mapgrid[i], grid_shape[i] - 1) def thin_to_full(thin_row): return ndimage.map_coordinates(thin_row, mapgrid) def full_to_thin(row): return ndimage.map_coordinates(row, thin_mapgrid) thin_mask = np.array(np.round(full_to_thin(mask)), dtype='bool') # Container for x thin_x = np.empty(thin_grid_shape[:2] + (3, )) mlon, mlat = np.meshgrid(*thin_axes[:2]) thin_x[:, :, 0] = mlon.T thin_x[:, :, 1] = mlat.T thin_x[:, :, 2] = 0 x = np.empty(grid_shape[:2] + (3, )) mlon, mlat = np.meshgrid(*axes[:2]) x[:, :, 0] = mlon.T x[:, :, 1] = mlat.T x[:, :, 2] = 0 del mlon, mlat Cp = C.params # In special case (america region) run only a test square using direct simulation, if it does not fail then just return if TESTSQUARE: getUnconditionedBlock(out_arr, real_index, grids, C_straightfromtrace, NinThinnedBlock=None, relp=relp, FULLRANK=False) return () # Prepare input dictionaries covParamObj = { 'Scale': Cp['scale'][0], 'amp': Cp['amp'][0], 'inc': Cp['inc'][0], 'ecc': Cp['ecc'][0], 't.lim.corr': Cp['tlc'][0], 'scale.t': Cp['st'][0], 'sin.frac': Cp['sf'][0] } gridParamObj = { 'YLLCORNER': grids[1][0] * rad_to_deg, 'CELLSIZE': (grids[1][1] - grids[1][0]) / (grids[1][2] - 1.) * rad_to_deg, 'NROWS': grid_shape[1], 'NCOLS': grid_shape[0] } monthParamObj = {'Nmonths': grid_shape[2], 'StartMonth': grids[2][0]} # Call R preprocessing function and check to make sure no screwy re-casting has taken place. t1 = time.time() os.chdir(r_path) preLoopObj = r.CONDSIMpreloop(covParamObj, gridParamObj, monthParamObj, indices.min(), indices.max(), paramfileINDEX) tree_reader = reader(file('listSummary_preLoopObj_original_%i_%i.txt' % (indices.min(), indices.max())), delimiter=' ') preLoopClassTree, junk = parse_tree(tree_reader) preLoopObj = compare_tree(preLoopObj, preLoopClassTree) OutMATlist = preLoopObj['OutMATlist'] tree_reader = reader(file('listSummary_OutMATlist_original_%i_%i.txt' % (indices.min(), indices.max())), delimiter=' ') OutMATClassTree, junk = parse_tree(tree_reader) OutMATlist = compare_tree(OutMATlist, OutMATClassTree) os.chdir(curpath) preLoop_time = time.time() - t1 print "preLoop_time :" + str(preLoop_time) #from IPython.Debugger import Pdb #Pdb(color_scheme='Linux').set_trace() ## Create and store unconditional realizations print '\tGenerating unconditional realizations.' t1 = time.time() for i in xrange(grid_shape[2]): print 'On month :' + str(i) #print 'OutMATlist:' #print OutMATlist os.chdir(r_path) monthObject = r.CONDSIMmonthloop(i + 1, preLoopObj, OutMATlist, indices.min(), indices.max(), paramfileINDEX) #monthObject = r.CONDSIMmonthloop(i+1,preLoopObj,OutMATlist,paramfileINDEX) os.chdir(curpath) OutMATlist = monthObject['OutMATlist'] MonthGrid = monthObject['MonthGrid'] out_arr[real_index, :, :, i] = MonthGrid[:grid_shape[1], :grid_shape[0]] t2 = time.time() print '\t\tDone in %f' % (t2 - t1) print "monthloop_time :" + str(t2 - t1) + " for " + str( grid_shape[2]) + " months" # delete unneeded R products del OutMATlist, preLoopObj, MonthGrid, monthObject #############################~TEMP # ################################~TEMP DIRECTLY JOIN SIMULATE UNCODITIONED BLOCK FOR TESTING # getUnconditionedBlock(out_arr,real_index,grids,C_straightfromtrace,NinThinnedBlock=None,relp=relp,FULLRANK=False) # #print 'variance of unconditioned block = '+str(round(np.var(out_arr),10)) # #print 'variance of unconditioned block month 6 = '+str(round(np.var(out_arr[:,:,:,6]),10)) # #examineRealization(outfile_root,real_index,6,15,None,None,conditioned=False,flipVertical="FALSE",SPACE=True,TIME=True) # ################################~TEMP # Figure out pdata pdata = np.empty(tdata.shape) for i in xrange(len(where_in)): index = where_in[i] pdata[index] = out_arr[real_index, grid_shape[1] - 1 - data_mesh_indices[index, 1], data_mesh_indices[index, 0], data_mesh_indices[index, 2]] # jointly simulate at data points conditional on block ## first get XYZT list of locations of a regular thinned sample from block #array3d = out_arr[real_index,:,:,:] ThinnedBlockXYTZlists = getThinnedBlockXYTZlists(out_arr, real_index, grids, NinThinnedBlock) xyt_in = ThinnedBlockXYTZlists['xyt_in'] z_in = ThinnedBlockXYTZlists['z_in'] # get locations of data outside block xyt_out = data_locs[where_out] # now we have locations and values of thinned sample from the block, and locations we want to predict at outside the block,go ahead and # get simulated values of the latter, conditonal on the former print '\tsimulating over ' + str( len(xyt_out[:, 0]) ) + ' locations outside block using thinned block sample of ' + str( len(z_in)) + ' points' t1 = time.time() z_out = predictPointsFromBlock(xyt_in, z_in, xyt_out, C_straightfromtrace, relp) print '\ttime for simulation: ' + str(time.time() - t1) #########################################CHECK COVARIANCE STRUCTURE #r.X11(width=12,height=12) #r.par(mfrow=(3,2)) # test marginal space and time covariance structures of points outside block #cfdict_out = getEmpiricalCovarianceFunction_STmarginals(xyt_out,z_out,mu=0,margTol_S=0.05,margTol_T=0.9/12,nbins=20, cutoff = 0.8) #plotEmpiricalCovarianceFunction(cfdict_out['space'],CovModelObj=C_straightfromtrace,spaceORtime="space", cutoff = 0.8, title="Points outside (S) "+str(paramfileINDEX)) #plotEmpiricalCovarianceFunction(cfdict_out['time'],CovModelObj=C_straightfromtrace,spaceORtime="time", cutoff = 0.8, title="Points outside (T)"+str(paramfileINDEX)) # test marginal space and time covariance structures of points inside block #cfdict_in = getEmpiricalCovarianceFunction_STmarginals(xyt_in,z_in,mu=0,margTol_S=0.05,margTol_T=0.9/12,nbins=20, cutoff = 0.8) #plotEmpiricalCovarianceFunction(cfdict_in['space'],CovModelObj=C_straightfromtrace,spaceORtime="space", cutoff = 0.8, title="Points inside (S)"+str(paramfileINDEX)) #plotEmpiricalCovarianceFunction(cfdict_in['time'],CovModelObj=C_straightfromtrace,spaceORtime="time", cutoff = 0.8, title="Points inside (T)"+str(paramfileINDEX)) # test marginal space and time covariance structures of points inside and outside block #cfdict_inout = getEmpiricalCovarianceFunction_STmarginals(np.vstack((xyt_in,xyt_out)),np.hstack((z_in,z_out)),mu=0,margTol_S=0.05,margTol_T=0.9/12,nbins=20, cutoff = 0.8) #plotEmpiricalCovarianceFunction(cfdict_inout['space'],CovModelObj=C_straightfromtrace,spaceORtime="space", cutoff = 0.8, title="Points inside (S)"+str(paramfileINDEX)) #plotEmpiricalCovarianceFunction(cfdict_inout['time'],CovModelObj=C_straightfromtrace,spaceORtime="time", cutoff = 0.8, title="Points inside (T)"+str(paramfileINDEX)) #########################################CHECK COVARIANCE STRUCTURE # assign these values to pdata pdata[where_out] = z_out ###############################~~TEMP #return() ##################################### # Bring in data. print '\tKriging to bring in data.' print '\tPreprocessing.' t1 = time.time() dev_posdef, xbi, ybi, dl_posdef = preprocess(C, data_locs, thin_grids, thin_x, n_blocks_x, n_blocks_y, tdata, pdata, relp, mean_ondata) t2 = time.time() print '\t\tDone in %f' % (t2 - t1) #from IPython.Debugger import Pdb #Pdb(color_scheme='Linux').set_trace() thin_row = np.empty(thin_grid_shape[:2], dtype=np.float32) print '\tKriging.' t1 = time.time() for i in xrange(grid_shape[2] - 1, -1, -1): thin_row.fill(0.) thin_x[:, :, 2] = axes[2][i] x[:, :, 2] = axes[2][i] krige_month(C, i, dl_posdef, thin_grid_shape, n_blocks_x, n_blocks_y, xbi, ybi, thin_x, dev_posdef, thin_row, thin_mask) row = ndimage.map_coordinates(thin_row, mapgrid) row += covariate_mesh row += M(x) row += grid_convert(out_arr[real_index, :, :, i], 'y-x+', 'x+y+') # NaN the oceans to save storage row[np.where(1 - mask)] = missing_val # if we are checking for plausible max and min values (Vs f at data), implement tet on conditioned values for this month monthMin = np.min(row[np.where(mask)]) monthMax = np.max(row[np.where(mask)]) pointsMin = np.min(tdata) pointsMax = np.max(tdata) pointsRange = pointsMax - pointsMin threshMin = pointsMin - (pointsRange * TESTRANGE) threshMax = pointsMax + (pointsRange * TESTRANGE) print('On month ' + str(i) + ' : f range=(' + str(pointsMin) + ',' + str(pointsMax) + ') ; month range=(' + str(monthMin) + ',' + str(monthMax) + ')') if TESTRANGE != False: if (((monthMin < threshMin) | (monthMax > threshMax))): raise ValueError('Killing realization on month ' + str(i) + ' : f range=(' + str(pointsMin) + ',' + str(pointsMax) + ') ; month range=(' + str(monthMin) + ',' + str(monthMax) + ')') out_arr[real_index, :, :, i] = grid_convert(row, 'x+y+', 'y-x+') ####################################TEMP #print 'variance of conditioned block month 6 = '+str(round(np.var(out_arr[:,:,:,6]),10)) #print 'variance of conditioned block = '+str(round(np.var(out_arr),10)) #examineRealization(outfile_root,real_index,6,15,None,None,conditioned=True,flipVertical="FALSE",SPACE=True,TIME=True) ######################################## print '\t\tDone in %f' % (time.time() - t1)
def create_realization(outfile_root,real_index, C,C_straightfromtrace, mean_ondata, M, covariate_mesh, tdata, data_locs, grids, axes, data_mesh_indices, where_in, where_out, n_blocks_x, n_blocks_y, relp, mask, thinning,indices,paramfileINDEX,NinThinnedBlock): # define only realizations chunk of hdf5 realization file out_arr = outfile_root.realizations """ Creates a single realization from the predictive distribution over specified space-time mesh. """ grid_shape = tuple([grid[2] for grid in grids]) #r.X11(width=8,height=4) #r.par(mfrow=(1,2)) #r.plot(data_mesh_indices[:,0],data_mesh_indices[:,1],xlab="",ylab="",main="",cex=0.5) #r.plot(data_locs[:,0],data_locs[:,1],xlab="",ylab="",main="",cex=0.5) #from IPython.Debugger import Pdb #Pdb(color_scheme='Linux').set_trace() thin_grids = tuple([grid[:2]+(grid[2]/thinning,) for grid in grids]) thin_grid_shape = tuple([thin_grid[2] for thin_grid in thin_grids]) thin_axes = tuple([np.linspace(*thin_grid) for thin_grid in thin_grids]) mapgrid = np.array(mgrid[0:grid_shape[0],0:grid_shape[1]], dtype=float) for i in xrange(2): normalize_for_mapcoords(mapgrid[i], thin_grid_shape[i]-1) thin_mapgrid = np.array(mgrid[0:thin_grid_shape[0], 0:thin_grid_shape[1]], dtype=float) for i in xrange(2): normalize_for_mapcoords(thin_mapgrid[i], grid_shape[i]-1) def thin_to_full(thin_row): return ndimage.map_coordinates(thin_row, mapgrid) def full_to_thin(row): return ndimage.map_coordinates(row, thin_mapgrid) thin_mask = np.array(np.round(full_to_thin(mask)), dtype='bool') # Container for x thin_x = np.empty(thin_grid_shape[:2] + (3,)) mlon,mlat = np.meshgrid(*thin_axes[:2]) thin_x[:,:,0] = mlon.T thin_x[:,:,1] = mlat.T thin_x[:,:,2] = 0 x = np.empty(grid_shape[:2] + (3,)) mlon,mlat = np.meshgrid(*axes[:2]) x[:,:,0] = mlon.T x[:,:,1] = mlat.T x[:,:,2] = 0 del mlon, mlat Cp = C.params # Prepare input dictionaries covParamObj = {'Scale': Cp['scale'][0], 'amp': Cp['amp'][0], 'inc': Cp['inc'][0], 'ecc': Cp['ecc'][0], 't.lim.corr': Cp['tlc'][0], 'scale.t': Cp['st'][0], 'sin.frac': Cp['sf'][0]} gridParamObj = {'YLLCORNER': grids[1][0]*rad_to_deg, 'CELLSIZE': (grids[1][1]-grids[1][0])/(grids[1][2]-1.)*rad_to_deg, 'NROWS':grid_shape[1], 'NCOLS':grid_shape[0]} monthParamObj = {'Nmonths':grid_shape[2],'StartMonth':grids[2][0]} ################################~TEMP # Call R preprocessing function and check to make sure no screwy re-casting has taken place. #t1 = time.time() #os.chdir(r_path) #preLoopObj = r.CONDSIMpreloop(covParamObj,gridParamObj,monthParamObj,indices.min(), indices.max(),paramfileINDEX) #tree_reader = reader(file('listSummary_preLoopObj_original_%i_%i.txt'%(indices.min(), indices.max())),delimiter=' ') #preLoopClassTree, junk = parse_tree(tree_reader) #preLoopObj = compare_tree(preLoopObj, preLoopClassTree) #OutMATlist = preLoopObj['OutMATlist'] #tree_reader = reader(file('listSummary_OutMATlist_original_%i_%i.txt'%(indices.min(), indices.max())),delimiter=' ') #OutMATClassTree, junk = parse_tree(tree_reader) #OutMATlist = compare_tree(OutMATlist, OutMATClassTree) #os.chdir(curpath) #preLoop_time = time.time()-t1 #print "preLoop_time :"+str(preLoop_time) ## Create and store unconditional realizations #print '\tGenerating unconditional realizations.' #t1 = time.time() #for i in xrange(grid_shape[2]): # print 'On month :'+str(i) # #print 'OutMATlist:' # #print OutMATlist # os.chdir(r_path) # monthObject = r.CONDSIMmonthloop(i+1,preLoopObj,OutMATlist, indices.min(), indices.max(),paramfileINDEX) # #monthObject = r.CONDSIMmonthloop(i+1,preLoopObj,OutMATlist,paramfileINDEX) # os.chdir(curpath) # OutMATlist= monthObject['OutMATlist'] # MonthGrid = monthObject['MonthGrid'] # out_arr[real_index,:,:,i] = MonthGrid[:grid_shape[1],:grid_shape[0]] #t2 = time.time() #print '\t\tDone in %f'%(t2-t1) #print "monthloop_time :"+str(t2-t1)+" for "+str(grid_shape[2])+" months" # delete unneeded R products #del OutMATlist, preLoopObj, MonthGrid, monthObject ################################~TEMP ################################~TEMP DIRECTLY JOIN SIMULATE UNCODITIONED BLOCK FOR TESTING getUnconditionedBlock(out_arr,real_index,grids,C_straightfromtrace,NinThinnedBlock=None,relp=None,FULLRANK=False) print 'variance of unconditioned block = '+str(round(np.var(out_arr),10)) print 'variance of unconditioned block month 6 = '+str(round(np.var(out_arr[:,:,:,6]),10)) examineRealization(outfile_root,0,6,15,None,None,conditioned=False,flipVertical="FALSE",SPACE=True,TIME=True) ################################~TEMP # Figure out pdata pdata = np.empty(tdata.shape) for i in xrange(len(where_in)): pdata[where_in[i]] = out_arr[real_index, grid_shape[1]-1-data_mesh_indices[i,1], data_mesh_indices[i,0], data_mesh_indices[i,2]] # jointly simulate at data points conditional on block ## first get XYZT list of locations of a regular thinned sample from block #array3d = out_arr[real_index,:,:,:] ThinnedBlockXYTZlists = getThinnedBlockXYTZlists (out_arr,real_index,grids,NinThinnedBlock) xyt_in = ThinnedBlockXYTZlists['xyt_in'] z_in = ThinnedBlockXYTZlists['z_in'] ## get locations of data outside block (referenced by grid location) #XYT_out_gridlocs = data_mesh_indices[where_out] # ### convert these grid lcoations into actual position in radians and months #coordsDict = gridParams_2_XYTmarginallists(grids) #xcoords = coordsDict['xcoords'] #ycoords = coordsDict['ycoords'] #tcoords = coordsDict['tcoords'] # #x_out = xcoords[XYT_out_gridlocs[:,0]] #y_out = ycoords[XYT_out_gridlocs[:,1]] #t_out = tcoords[XYT_out_gridlocs[:,2]] # #xyt_out = np.vstack((x_out,y_out,t_out)).T # get locations of data outside block xyt_out = data_locs[where_out] # now we have locations and values of thinned sample from the block, and locations we want to predict at outside the block,go ahead and # get simulated values of the latter, conditonal on the former print '\tsimulating over '+str(len(xyt_out[:,0]))+' locations outside block using thinned block sample of '+str(len(z_in))+' points' t1=time.time() z_out = predictPointsFromBlock(xyt_in,z_in, xyt_out,C_straightfromtrace,relp) print '\ttime for simulation: '+str(time.time()-t1) #########################################CHECK COVARIANCE STRUCTURE #r.X11(width=12,height=12) #r.par(mfrow=(3,2)) # test marginal space and time covariance structures of points outside block #cfdict_out = getEmpiricalCovarianceFunction_STmarginals(xyt_out,z_out,mu=0,margTol_S=0.05,margTol_T=0.9/12,nbins=20, cutoff = 0.8) #plotEmpiricalCovarianceFunction(cfdict_out['space'],CovModelObj=C_straightfromtrace,spaceORtime="space", cutoff = 0.8, title="Points outside (S) "+str(paramfileINDEX)) #plotEmpiricalCovarianceFunction(cfdict_out['time'],CovModelObj=C_straightfromtrace,spaceORtime="time", cutoff = 0.8, title="Points outside (T)"+str(paramfileINDEX)) # test marginal space and time covariance structures of points inside block #cfdict_in = getEmpiricalCovarianceFunction_STmarginals(xyt_in,z_in,mu=0,margTol_S=0.05,margTol_T=0.9/12,nbins=20, cutoff = 0.8) #plotEmpiricalCovarianceFunction(cfdict_in['space'],CovModelObj=C_straightfromtrace,spaceORtime="space", cutoff = 0.8, title="Points inside (S)"+str(paramfileINDEX)) #plotEmpiricalCovarianceFunction(cfdict_in['time'],CovModelObj=C_straightfromtrace,spaceORtime="time", cutoff = 0.8, title="Points inside (T)"+str(paramfileINDEX)) # test marginal space and time covariance structures of points inside and outside block #cfdict_inout = getEmpiricalCovarianceFunction_STmarginals(np.vstack((xyt_in,xyt_out)),np.hstack((z_in,z_out)),mu=0,margTol_S=0.05,margTol_T=0.9/12,nbins=20, cutoff = 0.8) #plotEmpiricalCovarianceFunction(cfdict_inout['space'],CovModelObj=C_straightfromtrace,spaceORtime="space", cutoff = 0.8, title="Points inside (S)"+str(paramfileINDEX)) #plotEmpiricalCovarianceFunction(cfdict_inout['time'],CovModelObj=C_straightfromtrace,spaceORtime="time", cutoff = 0.8, title="Points inside (T)"+str(paramfileINDEX)) #########################################CHECK COVARIANCE STRUCTURE # assign these values to pdata pdata[where_out] = z_out ###############################~~TEMP #return() ##################################### # Bring in data. print '\tKriging to bring in data.' print '\tPreprocessing.' t1 = time.time() dev_posdef, xbi, ybi, dl_posdef = preprocess(C, data_locs, thin_grids, thin_x, n_blocks_x, n_blocks_y, tdata, pdata, relp, mean_ondata) t2 = time.time() print '\t\tDone in %f'%(t2-t1) ###############################~~TEMP #from IPython.Debugger import Pdb #Pdb(color_scheme='Linux').set_trace() ##################################### thin_row = np.empty(thin_grid_shape[:2], dtype=np.float32) print '\tKriging.' t1 = time.time() C1 = pm.gp.NearlyFullRankCovariance(C_straightfromtrace.eval_fun, **C_straightfromtrace.params) C2 = pm.gp.NearlyFullRankCovariance(C_straightfromtrace.eval_fun, **C_straightfromtrace.params) M1 = pm.gp.Mean(lambda x: np.zeros(x.shape[:-1])) M2 = pm.gp.Mean(lambda x: np.zeros(x.shape[:-1])) cholfac = C1.cholesky(data_locs, apply_pivot=False) U = cholfac['U'] piv = cholfac['pivots'] m = U.shape[0] w = np.linalg.solve(U[:m,:m],np.linalg.solve(U[:m,:m].T,(tdata-mean_ondata)[piv[:m]])) pm.gp.observe(M2,C2,data_locs,pdata) for i in xrange(grid_shape[2]-1,-1,-1): thin_row.fill(0.) thin_x[:,:,2] = axes[2][i] x[:,:,2] = axes[2][i] simkrige = M2(x) dkrige = covariate_mesh + M(x) + np.dot(C1(x,data_locs[piv[:m]]).view(ndarray), w).reshape(x.shape[:-1]) simsurf = grid_convert(out_arr[real_index,:,:,i], 'y-x+', 'x+y+') import pylab as pl pl.imshow(simsurf) pl.colorbar() pl.title('Simulated') pl.figure() pl.imshow(simkrige) pl.colorbar() pl.title('Sim krige') pl.figure() pl.imshow(simsurf-simkrige) pl.colorbar() pl.title('Adjusted sim') pl.figure() pl.imshow(dkrige) pl.colorbar() pl.title('Data krige') pl.figure() pl.imshow(simsurf-simkrige+dkrige) pl.colorbar() pl.title('Kriged sim') krige_month(C, i, dl_posdef, thin_grid_shape, n_blocks_x, n_blocks_y, xbi, ybi, thin_x, dev_posdef, thin_row, thin_mask) row = ndimage.map_coordinates(thin_row, mapgrid) row += covariate_mesh row += M(x) row += grid_convert(out_arr[real_index,:,:,i], 'y-x+', 'x+y+') pl.figure() pl.imshow(row) pl.colorbar() pl.title('Original method') from IPython.Debugger import Pdb Pdb(color_scheme='Linux').set_trace() # NaN the oceans to save storage row[np.where(1-mask)] = missing_val out_arr[real_index,:,:,i] = grid_convert(row, 'x+y+','y-x+') ####################################TEMP print 'variance of conditioned block month 6 = '+str(round(np.var(out_arr[:,:,:,6]),10)) print 'variance of conditioned block = '+str(round(np.var(out_arr),10)) examineRealization(outfile_root,0,6,15,None,None,conditioned=True,flipVertical="FALSE",SPACE=True,TIME=True) ######################################## print '\t\tDone in %f'%(time.time()-t1)
modis_res = (21600, 43200) lon = np.linspace(-180,180,modis_res[1]) lat = np.linspace(-90,90,modis_res[0]) af_lon, af_lat, af_data, af_type = map_utils.import_raster('africa','.') lon_min, lon_max, lat_min, lat_max = af_lon.min(), af_lon.max(), af_lat.min(), af_lat.max() pete_lon, pete_lat, pete_mask, junk = map_utils.import_raster('ls','.') # Subset the rasters lon_min_i, lon_max_i = ((np.array([lon_min, lon_max])+180.)/360.*modis_res[1]).astype('int') lat_min_i, lat_max_i = ((np.array([lat_min, lat_max])+90.)/180.*modis_res[0]).astype('int') pete_lat_min_i = np.argmin(np.abs(pete_lat - af_lat.min())) pete_lat_max_i = np.argmin(np.abs(pete_lat - af_lat.max())) clipped_pete_mask = grid_convert(grid_convert(pete_mask,'y-x+','x+y+')[lon_min_i/5.:lon_max_i/5.,pete_lat_min_i:pete_lat_max_i],'x+y+','y-x+') af_lon = af_lon[:-1] af_lat = af_lat[:-1] af_data = af_data[:-1,:-1] <<<<<<< Updated upstream af_data.mask = af_data.mask + clipped_pete_mask.mask map_utils.export_raster(af_lon,af_lat,af_data,'africa','5k-covariates','flt') ======= af_data.mask = af_data.mask + clipped_pete_mask map_utils.export_raster(af_lon,af_lat,af_data,'africa','5k-covariates','hdf5') >>>>>>> Stashed changes def subset_and_writeout(hf_in, fname, thin, maskval, binfn=lambda x:x): print 'Subsetting for %s'%fname