def make_rtree(self):
with self.dataset() as nc:
sg = from_nc_dataset(nc)
def rtree_generator_function():
c = 0
for i, axis in enumerate(sg.centers):
for j, (x, y) in enumerate(axis):
c += 1
yield (c, (x, y, x, y), (i, j))
logger.info(
"Building Faces (centers) Rtree Topology Cache for {0}".format(
self.name))
_, temp_file = tempfile.mkstemp(suffix='.face')
start = time.time()
p = index.Property()
p.filename = str(temp_file)
p.overwrite = True
p.storage = index.RT_Disk
p.dimension = 2
index.Index(p.filename.decode('utf-8'),
rtree_generator_function(),
properties=p,
overwrite=True,
interleaved=True)
logger.info(
"Built Faces (centers) Rtree Topology Cache in {0} seconds.".
format(time.time() - start))
shutil.move('{}.dat'.format(temp_file), self.face_tree_data_file)
shutil.move('{}.idx'.format(temp_file), self.face_tree_index_file)
def make_rtree(self):
p = rtree.index.Property()
p.overwrite = True
p.storage = rtree.index.RT_Disk
p.Dimension = 2
with self.dataset() as nc:
sg = from_nc_dataset(nc)
class FastRtree(rtree.Rtree):
def dumps(self, obj):
try:
import cPickle
return cPickle.dumps(obj, -1)
except ImportError:
super(FastRtree, self).dumps(obj)
def rtree_generator_function():
for i, axis in enumerate(sg.centers):
for j, (x, y) in enumerate(axis):
yield (i+j, (x, y, x, y), (i, j))
logger.info("Building Faces (centers) Rtree Topology Cache for {0}".format(self.name))
_, temp_file = tempfile.mkstemp(suffix='.face')
start = time.time()
FastRtree(temp_file,
rtree_generator_function(),
properties=p,
overwrite=True,
interleaved=True)
logger.info("Built Faces (centers) Rtree Topology Cache in {0} seconds.".format(time.time() - start))
shutil.move('{}.dat'.format(temp_file), self.face_tree_data_file)
shutil.move('{}.idx'.format(temp_file), self.face_tree_index_file)
def make_rtree(self):
with self.dataset() as nc:
sg = from_nc_dataset(nc)
def rtree_generator_function():
c = 0
for i, axis in enumerate(sg.centers):
for j, (x, y) in enumerate(axis):
c += 1
yield (c, (x, y, x, y), (i, j))
logger.info("Building Faces (centers) Rtree Topology Cache for {0}".format(self.name))
_, temp_file = tempfile.mkstemp(suffix='.face')
start = time.time()
p = index.Property()
p.filename = str(temp_file)
p.overwrite = True
p.storage = index.RT_Disk
p.dimension = 2
index.Index(p.filename.decode('utf-8'),
rtree_generator_function(),
properties=p,
overwrite=True,
interleaved=True)
logger.info("Built Faces (centers) Rtree Topology Cache in {0} seconds.".format(time.time() - start))
shutil.move('{}.dat'.format(temp_file), self.face_tree_data_file)
shutil.move('{}.idx'.format(temp_file), self.face_tree_index_file)
def update_cache(self, force=False):
with self.dataset() as nc:
sg = from_nc_dataset(nc)
sg.save_as_netcdf(self.topology_file)
if not os.path.exists(self.topology_file):
logger.error(
"Failed to create topology_file cache for Dataset '{}'".
format(self.dataset))
return
# add time to the cached topology
time_vars = nc.get_variables_by_attributes(standard_name='time')
time_dims = list(
itertools.chain.from_iterable(
[time_var.dimensions for time_var in time_vars]))
unique_time_dims = list(set(time_dims))
with EnhancedDataset(self.topology_file, mode='a') as cached_nc:
# create pertinent time dimensions if they aren't already present
for unique_time_dim in unique_time_dims:
dim_size = len(nc.dimensions[unique_time_dim])
try:
cached_nc.createDimension(unique_time_dim,
size=dim_size)
except RuntimeError:
continue
# support cases where there may be more than one variable with standard_name='time' in a dataset
for time_var in time_vars:
try:
time_var_obj = cached_nc.createVariable(
time_var.name, time_var.dtype, time_var.dimensions)
except RuntimeError:
time_var_obj = cached_nc.variables[time_var.name]
finally:
time_var_obj[:] = time_var[:]
time_var_obj.units = time_var.units
time_var_obj.standard_name = 'time'
# Now do the RTree index
self.make_rtree()
self.cache_last_updated = datetime.utcnow().replace(tzinfo=pytz.utc)
self.save()
def update_cache(self, force=False):
with self.dataset() as nc:
sg = from_nc_dataset(nc)
sg.save_as_netcdf(self.topology_file)
if not os.path.exists(self.topology_file):
logger.error("Failed to create topology_file cache for Dataset '{}'".format(self.dataset))
return
# add time to the cached topology
time_vars = nc.get_variables_by_attributes(standard_name='time')
time_dims = list(itertools.chain.from_iterable([time_var.dimensions for time_var in time_vars]))
unique_time_dims = list(set(time_dims))
with EnhancedDataset(self.topology_file, mode='a') as cached_nc:
# create pertinent time dimensions if they aren't already present
for unique_time_dim in unique_time_dims:
dim_size = len(nc.dimensions[unique_time_dim])
try:
cached_nc.createDimension(unique_time_dim, size=dim_size)
except RuntimeError:
continue
# support cases where there may be more than one variable with standard_name='time' in a dataset
for time_var in time_vars:
try:
time_var_obj = cached_nc.createVariable(time_var.name,
time_var.dtype,
time_var.dimensions)
except RuntimeError:
time_var_obj = cached_nc.variables[time_var.name]
finally:
time_var_obj[:] = time_var[:]
time_var_obj.units = time_var.units
time_var_obj.standard_name = 'time'
# Now do the RTree index
self.make_rtree()
self.cache_last_updated = datetime.utcnow().replace(tzinfo=pytz.utc)
self.save()
def make_rtree(self):
p = rtree.index.Property()
p.overwrite = True
p.storage = rtree.index.RT_Disk
p.Dimension = 2
with self.dataset() as nc:
sg = from_nc_dataset(nc)
class FastRtree(rtree.Rtree):
def dumps(self, obj):
try:
import cPickle
return cPickle.dumps(obj, -1)
except ImportError:
super(FastRtree, self).dumps(obj)
def rtree_generator_function():
for i, axis in enumerate(sg.centers):
for j, (x, y) in enumerate(axis):
yield (i + j, (x, y, x, y), (i, j))
logger.info(
"Building Faces (centers) Rtree Topology Cache for {0}".format(
self.name))
_, temp_file = tempfile.mkstemp(suffix='.face')
start = time.time()
FastRtree(temp_file,
rtree_generator_function(),
properties=p,
overwrite=True,
interleaved=True)
logger.info(
"Built Faces (centers) Rtree Topology Cache in {0} seconds.".
format(time.time() - start))
shutil.move('{}.dat'.format(temp_file), self.face_tree_data_file)
shutil.move('{}.idx'.format(temp_file), self.face_tree_index_file)
PADDING = 0.18
LAYER = 'temp'
os.environ['TCL_LIBRARY'] = 'C:/Python279/tcl/tcl8.5'
os.environ['TK_LIBRARY'] = 'C:/Python279/tcl/tk8.5'
t = datetime.datetime(2012, 6, 25, 2, 0)
z = -0.85
EPSG4326 = pyproj.Proj('+init=EPSG:4326')
cached_dataset = nc4.Dataset(CACHED_URL)
canon_dataset = nc4.Dataset(DATASET_URL)
time_idx, time_val = nearest_time(cached_dataset, t)
cached_sg = from_nc_dataset(cached_dataset)
lon_name, lat_name = cached_sg.face_coordinates
lon_obj = getattr(cached_sg, lon_name)
lat_obj = getattr(cached_sg, lat_name)
mask_rho_obj = cached_sg.mask_rho
raw_mask = canon_dataset.variables['mask_rho'][:, :]
trimed_mask = raw_mask[mask_rho_obj.center_slicing]
centers = cached_sg.centers
longitudes = centers[..., 0]
latitudes = centers[..., 1]
lon_trim = longitudes[lon_obj.center_slicing]
lat_trim = latitudes[lat_obj.center_slicing]
# get the indices of the longitude and latitude that fall within the specified bounds
subset_idx = lon_lat_subset_idx(lon_trim, lat_trim, LON_MIN, LAT_MIN, LON_MAX, LAT_MAX)
subset_lon = subset_data(lon_trim, subset_idx)
# In[1]: from netCDF4 import Dataset url = "http://geoport.whoi.edu/thredds/dodsC/clay/usgs/users/" "jcwarner/Projects/Sandy/triple_nest/00_dir_NYB05.ncml" nc = Dataset(url) # The pysgrid object can be instantiated from a `netCDF4-python` object or the file/URL. # In[2]: import pysgrid sgrid = pysgrid.from_nc_dataset(nc) # The `sgrid` object knows about the center of the grid and prepare a slicing object that can be used to get the data at the center of the grid. # In[3]: sgrid.u.center_slicing # In[4]: sgrid.v.center_slicing # Let's get some data. (Using integer indexes for convenience.)
url = ('http://geoport.whoi.edu/thredds/dodsC/clay/usgs/users/'
'jcwarner/Projects/Sandy/triple_nest/00_dir_NYB05.ncml')
nc = Dataset(url)
# <markdowncell>
# ### The sgrid object
# <codecell>
import pysgrid
# The object creation is a little bit slow. Can we defer some of the loading/computations?
sgrid = pysgrid.from_nc_dataset(nc)
sgrid # We need a better __repr__ and __str__ !!!
# <markdowncell>
# ### The object knows about sgrid conventions
# <codecell>
sgrid.edge1_coordinates, sgrid.edge1_dimensions, sgrid.edge1_padding
# <codecell>
u_var = sgrid.u
u_var.center_axis, u_var.node_axis
import netCDF4 as nc4 from pysgrid import from_nc_dataset WRF_TEST_FILE = 'http://geoport.whoi.edu/thredds/dodsC/usgs/data1/rsignell/models/wrf/sgrid.ncml' ds = nc4.Dataset(WRF_TEST_FILE) sg = from_nc_dataset(ds) centers = sg.centers xlat_u = ds.variables['XLAT_U'] xlong_u = ds.variables['XLONG_U'] xlat_v = ds.variables['XLAT_V'] xlong_v = ds.variables['XLONG_V']