def update_time_cache(self):
with self.dataset() as nc:
if nc is None:
logger.error("Failed update_time_cache, could not load dataset "
"as a netCDF4 object")
return
time_cache = {}
layer_cache = {}
time_vars = nc.get_variables_by_attributes(standard_name='time')
for time_var in time_vars:
time_cache[time_var.name] = nc4.num2date(
time_var[:],
time_var.units,
getattr(time_var, 'calendar', 'standard')
)
for ly in self.all_layers():
try:
layer_cache[ly.access_name] = find_appropriate_time(nc.variables[ly.access_name], time_vars)
except ValueError:
layer_cache[ly.access_name] = None
full_cache = {'times': time_cache, 'layers': layer_cache}
logger.info("Built time cache for {0}".format(self.name))
caches['time'].set(self.time_cache_file, full_cache, None)
return full_cache
def update_grid_cache(self, force=False):
with self.dataset() as nc:
if nc is None:
logger.error(
"Failed update_grid_cache, could not load dataset "
"as a netCDF4 object")
return
sg = load_grid(nc)
# Atomic write
tmphandle, tmpsave = tempfile.mkstemp()
try:
sg.save_as_netcdf(tmpsave)
finally:
os.close(tmphandle)
if os.path.isfile(tmpsave):
shutil.move(tmpsave, self.topology_file)
else:
logger.error(
"Failed to create topology_file cache for Dataset '{}'"
.format(self.dataset.name))
return
# Now do the RTree index
self.make_rtree()
def times(self, layer):
time_cache = caches['time'].get(self.time_cache_file, {'times': {}, 'layers': {}})
if layer.access_name not in time_cache['layers']:
logger.error("No layer ({}) in time cache, returning nothing".format(layer.access_name))
return []
ltv = time_cache['layers'][layer.access_name]
if ltv in time_cache['times']:
return time_cache['times'][ltv]
else:
logger.error("No time ({}) in time cache, returning nothing".format(ltv))
return []
def update_cache(self, force=False):
with self.dataset() as nc:
ug = UGrid.from_nc_dataset(nc=nc)
ug.save_as_netcdf(self.topology_file)
if not os.path.exists(self.topology_file):
logger.error(
"Failed to create topology_file cache for Dataset '{}'".
format(self.dataset))
return
time_vars = nc.get_variables_by_attributes(standard_name='time')
time_dims = list(
itertools.chain.from_iterable(
[time_var.dimensions for time_var in time_vars]))
unique_time_dims = list(set(time_dims))
with EnhancedDataset(self.topology_file, mode='a') as cached_nc:
# create pertinent time dimensions if they aren't already present
for unique_time_dim in unique_time_dims:
dim_size = len(nc.dimensions[unique_time_dim])
try:
cached_nc.createDimension(unique_time_dim,
size=dim_size)
except RuntimeError:
continue
# support cases where there may be more than one variable with standard_name='time' in a dataset
for time_var in time_vars:
try:
time_var_obj = cached_nc.createVariable(
time_var._name, time_var.dtype,
time_var.dimensions)
except RuntimeError:
time_var_obj = cached_nc.variables[time_var.name]
time_var_obj[:] = time_var[:]
time_var_obj.units = time_var.units
time_var_obj.standard_name = 'time'
# Now do the RTree index
self.make_rtree()
self.cache_last_updated = datetime.utcnow().replace(tzinfo=pytz.utc)
self.save()
def update_cache(self, force=False):
with self.dataset() as nc:
sg = from_nc_dataset(nc)
sg.save_as_netcdf(self.topology_file)
if not os.path.exists(self.topology_file):
logger.error("Failed to create topology_file cache for Dataset '{}'".format(self.dataset))
return
# add time to the cached topology
time_vars = nc.get_variables_by_attributes(standard_name='time')
time_dims = list(itertools.chain.from_iterable([time_var.dimensions for time_var in time_vars]))
unique_time_dims = list(set(time_dims))
with EnhancedDataset(self.topology_file, mode='a') as cached_nc:
# create pertinent time dimensions if they aren't already present
for unique_time_dim in unique_time_dims:
dim_size = len(nc.dimensions[unique_time_dim])
try:
cached_nc.createDimension(unique_time_dim, size=dim_size)
except RuntimeError:
continue
# support cases where there may be more than one variable with standard_name='time' in a dataset
for time_var in time_vars:
try:
time_var_obj = cached_nc.createVariable(time_var.name,
time_var.dtype,
time_var.dimensions)
except RuntimeError:
time_var_obj = cached_nc.variables[time_var.name]
finally:
time_var_obj[:] = time_var[:]
time_var_obj.units = time_var.units
time_var_obj.standard_name = 'time'
# Now do the RTree index
self.make_rtree()
self.cache_last_updated = datetime.utcnow().replace(tzinfo=pytz.utc)
self.save()
def times(self, layer):
time_cache = caches['time'].get(self.time_cache_file, {
'times': {},
'layers': {}
})
if layer.access_name not in time_cache['layers']:
logger.error(
"No layer ({}) in time cache, returning nothing".format(
layer.access_name))
return []
ltv = time_cache['layers'].get(layer.access_name)
if ltv is None:
# Legit this might not be a layer with time so just return empty list (no error message)
return []
if ltv in time_cache['times']:
return time_cache['times'][ltv]
else:
logger.error(
"No time ({}) in time cache, returning nothing".format(ltv))
return []
def update_cache(self, force=False):
with self.dataset() as nc:
ug = UGrid.from_nc_dataset(nc)
ug.save_as_netcdf(self.topology_file)
if not os.path.exists(self.topology_file):
logger.error(
"Failed to create topology_file cache for Dataset '{}'".
format(self.dataset))
return
uamp = nc.get_variables_by_attributes(
standard_name='eastward_sea_water_velocity_amplitude')[0]
vamp = nc.get_variables_by_attributes(
standard_name='northward_sea_water_velocity_amplitude')[0]
uphase = nc.get_variables_by_attributes(
standard_name='eastward_sea_water_velocity_phase')[0]
vphase = nc.get_variables_by_attributes(
standard_name='northward_sea_water_velocity_phase')[0]
tnames = nc.get_variables_by_attributes(
standard_name='tide_constituent')[0]
tfreqs = nc.get_variables_by_attributes(
standard_name='tide_frequency')[0]
with netCDF4.Dataset(self.topology_file, mode='a') as cnc:
ntides = uamp.shape[uamp.dimensions.index('ntides')]
nlocs = uamp.shape[uamp.dimensions.index(uamp.location)]
cnc.createDimension('ntides', ntides)
cnc.createDimension('maxStrlen64', 64)
vdims = ('ntides', '{}_num_{}'.format(uamp.mesh,
uamp.location))
# Swap ntides to always be the first dimension.. it can be the second in the source files!
transpose = False
if uamp.shape[0] > uamp.shape[1]:
logger.info(
"Found flipped dimensions in source file... fixing in local cache."
)
transpose = True
# We are changing the variable names to 'u' and 'v' from 'u_amp' and 'v_amp' so
# the layer.access_method can find the variable from the virtual layer 'u,v'
ua = cnc.createVariable('u',
uamp.dtype,
vdims,
zlib=True,
fill_value=uamp._FillValue,
chunksizes=[1, nlocs / 4])
for x in uamp.ncattrs():
if x != '_FillValue':
ua.setncattr(x, uamp.getncattr(x))
va = cnc.createVariable('v',
vamp.dtype,
vdims,
zlib=True,
fill_value=vamp._FillValue,
chunksizes=[1, nlocs / 4])
for x in vamp.ncattrs():
if x != '_FillValue':
va.setncattr(x, vamp.getncattr(x))
up = cnc.createVariable('u_phase',
uphase.dtype,
vdims,
zlib=True,
fill_value=uphase._FillValue,
chunksizes=[1, nlocs / 4])
for x in uphase.ncattrs():
if x != '_FillValue':
up.setncattr(x, uphase.getncattr(x))
vp = cnc.createVariable('v_phase',
vphase.dtype,
vdims,
zlib=True,
fill_value=vphase._FillValue,
chunksizes=[1, nlocs / 4])
for x in vphase.ncattrs():
if x != '_FillValue':
vp.setncattr(x, vphase.getncattr(x))
tc = cnc.createVariable('tidenames', tnames.dtype,
tnames.dimensions)
tc[:] = tnames[:]
for x in tnames.ncattrs():
if x != '_FillValue':
tc.setncattr(x, tnames.getncattr(x))
tf = cnc.createVariable('tidefreqs', tfreqs.dtype,
('ntides', ))
tf[:] = tfreqs[:]
for x in tfreqs.ncattrs():
if x != '_FillValue':
tf.setncattr(x, tfreqs.getncattr(x))
for r in range(ntides):
logger.info("Saving ntide {} into cache".format(r))
if transpose is True:
ua[r, :] = uamp[:, r].T
va[r, :] = vamp[:, r].T
up[r, :] = uphase[:, r].T
vp[r, :] = vphase[:, r].T
else:
ua[r, :] = uamp[r, :]
va[r, :] = vamp[r, :]
up[r, :] = uphase[r, :]
vp[r, :] = vphase[r, :]
# Now do the RTree index
self.make_rtree()
self.cache_last_updated = datetime.utcnow().replace(tzinfo=pytz.utc)
self.save()
def update_cache(self, force=False):
with self.dataset() as nc:
ug = UGrid.from_nc_dataset(nc)
ug.save_as_netcdf(self.topology_file)
if not os.path.exists(self.topology_file):
logger.error("Failed to create topology_file cache for Dataset '{}'".format(self.dataset))
return
uamp = nc.get_variables_by_attributes(standard_name='eastward_sea_water_velocity_amplitude')[0]
vamp = nc.get_variables_by_attributes(standard_name='northward_sea_water_velocity_amplitude')[0]
uphase = nc.get_variables_by_attributes(standard_name='eastward_sea_water_velocity_phase')[0]
vphase = nc.get_variables_by_attributes(standard_name='northward_sea_water_velocity_phase')[0]
tnames = nc.get_variables_by_attributes(standard_name='tide_constituent')[0]
tfreqs = nc.get_variables_by_attributes(standard_name='tide_frequency')[0]
with netCDF4.Dataset(self.topology_file, mode='a') as cnc:
ntides = uamp.shape[uamp.dimensions.index('ntides')]
nlocs = uamp.shape[uamp.dimensions.index(uamp.location)]
cnc.createDimension('ntides', ntides)
cnc.createDimension('maxStrlen64', 64)
vdims = ('ntides', '{}_num_{}'.format(uamp.mesh, uamp.location))
# Swap ntides to always be the first dimension.. it can be the second in the source files!
transpose = False
if uamp.shape[0] > uamp.shape[1]:
logger.info("Found flipped dimensions in source file... fixing in local cache.")
transpose = True
# We are changing the variable names to 'u' and 'v' from 'u_amp' and 'v_amp' so
# the layer.access_method can find the variable from the virtual layer 'u,v'
ua = cnc.createVariable('u', uamp.dtype, vdims, zlib=True, fill_value=uamp._FillValue, chunksizes=[1, nlocs/4])
for x in uamp.ncattrs():
if x != '_FillValue':
ua.setncattr(x, uamp.getncattr(x))
va = cnc.createVariable('v', vamp.dtype, vdims, zlib=True, fill_value=vamp._FillValue, chunksizes=[1, nlocs/4])
for x in vamp.ncattrs():
if x != '_FillValue':
va.setncattr(x, vamp.getncattr(x))
up = cnc.createVariable('u_phase', uphase.dtype, vdims, zlib=True, fill_value=uphase._FillValue, chunksizes=[1, nlocs/4])
for x in uphase.ncattrs():
if x != '_FillValue':
up.setncattr(x, uphase.getncattr(x))
vp = cnc.createVariable('v_phase', vphase.dtype, vdims, zlib=True, fill_value=vphase._FillValue, chunksizes=[1, nlocs/4])
for x in vphase.ncattrs():
if x != '_FillValue':
vp.setncattr(x, vphase.getncattr(x))
tc = cnc.createVariable('tidenames', tnames.dtype, tnames.dimensions)
tc[:] = tnames[:]
for x in tnames.ncattrs():
if x != '_FillValue':
tc.setncattr(x, tnames.getncattr(x))
tf = cnc.createVariable('tidefreqs', tfreqs.dtype, ('ntides',))
tf[:] = tfreqs[:]
for x in tfreqs.ncattrs():
if x != '_FillValue':
tf.setncattr(x, tfreqs.getncattr(x))
for r in range(ntides):
logger.info("Saving ntide {} into cache".format(r))
if transpose is True:
ua[r, :] = uamp[:, r].T
va[r, :] = vamp[:, r].T
up[r, :] = uphase[:, r].T
vp[r, :] = vphase[:, r].T
else:
ua[r, :] = uamp[r, :]
va[r, :] = vamp[r, :]
up[r, :] = uphase[r, :]
vp[r, :] = vphase[r, :]
# Now do the RTree index
self.make_rtree()
self.cache_last_updated = datetime.utcnow().replace(tzinfo=pytz.utc)
self.save()